diff --git a/scripts/__pycache__/scan_wins.cpython-311.pyc b/scripts/__pycache__/scan_wins.cpython-311.pyc new file mode 100644 index 00000000..7d1edd1d Binary files /dev/null and b/scripts/__pycache__/scan_wins.cpython-311.pyc differ diff --git a/scripts/__pycache__/training_orchestrator.cpython-311.pyc b/scripts/__pycache__/training_orchestrator.cpython-311.pyc new file mode 100644 index 00000000..fba29209 Binary files /dev/null and b/scripts/__pycache__/training_orchestrator.cpython-311.pyc differ diff --git a/scripts/scan_wins.py b/scripts/scan_wins.py index fb18f3db..ebf48513 100644 --- a/scripts/scan_wins.py +++ b/scripts/scan_wins.py @@ -64,4 +64,16 @@ def scan(csv_dir='training_data'): if __name__ == '__main__': d = sys.argv[1] if len(sys.argv) > 1 else 'training_data' + # 如果指定了目录不存在, 可能是编号的round目录 + if not os.path.isdir(d): + # 尝试 training_rounds 下最新 + rounds = sorted(glob.glob('training_rounds/r*')) + if rounds: + d = os.path.join(rounds[-1], 'training_data') + if not os.path.isdir(d): + print(f"No training_data in {rounds[-1]}") + sys.exit(1) + else: + print("No data found") + sys.exit(1) scan(d) \ No newline at end of file diff --git a/scripts/training_orchestrator.py b/scripts/training_orchestrator.py index 46c3a3a7..b4bee57c 100644 --- a/scripts/training_orchestrator.py +++ b/scripts/training_orchestrator.py @@ -1,30 +1,23 @@ #!/usr/bin/env python3 """ -持续训练 — 10k首轮 + 5k后续轮 → 循环到08:00 -每轮: dotnet自对弈 → V训练 → ONNX导出 +持续训练: 20k局/轮, 无限循环 +每轮: dotnet自对弈 → V训练 → ONNX导出 → 存档 +胜率趋势由 scan_wins.py 每小时 cron 独立采集 """ -import subprocess, os, sys, time, shutil, glob, json +import subprocess, os, sys, time, shutil, glob from datetime import datetime HJHA_DIR = '/home/xiaoou/projects/hjha-server' TRAINER_DIR = '/home/xiaoou/projects/paodekuai-trainer' -DEADLINE = '08:00' LOG = os.path.join(HJHA_DIR, 'training_loop.log') -def now(): return datetime.now().strftime('%H:%M:%S') +def now(): return datetime.now().strftime('%m%d %H:%M:%S') def log(msg): line = f"[{now()}] {msg}" print(line, flush=True) with open(LOG, 'a') as f: f.write(line + '\n') -def mins_left(): - dt = datetime.now() - h, m = map(int, DEADLINE.split(':')) - dl = dt.replace(hour=h, minute=m, second=0, microsecond=0) - return max(0, (dl - dt).total_seconds() / 60) - -def run(cmd, cwd=HJHA_DIR, to=14400): - """返回 (ok, stdout+stderr)""" +def run(cmd, cwd=HJHA_DIR, to=86400): try: r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, timeout=to) ok = r.returncode == 0 @@ -35,7 +28,6 @@ def run(cmd, cwd=HJHA_DIR, to=14400): log(f" EXCEPTION: {e}") return False, str(e) -# ── 自对弈 ── def selfplay(games): log(f"── 自对弈 {games}局 ──") td = os.path.join(HJHA_DIR, 'training_data') @@ -45,8 +37,8 @@ def selfplay(games): env = os.environ.copy() env['PATH'] = f"/usr/lib/dotnet:{env.get('PATH','')}" - ok, out = run(['/usr/lib/dotnet/dotnet', 'run', '--project', 'hjha-console', - '--', '--mix', str(games)], to=max(14400, games*2)) + ok, out = run(['/usr/lib/dotnet/dotnet', 'run', '--project', 'hjha-console', '-c', 'Release', + '--', '--mix', str(games)], to=max(36000, games*2)) csv = len(glob.glob(os.path.join(td, 'game_*.csv'))) sf = os.path.join(HJHA_DIR, 'stress_summary.txt') @@ -55,7 +47,6 @@ def selfplay(games): log(f" {csv} CSV") return csv -# ── V训练 ── def train_v(): log(f"── V训练 ──") @@ -63,7 +54,6 @@ def train_v(): import numpy as np, torch from models.network import PdkNet - # load data csv_dir = os.path.join(HJHA_DIR, 'training_data') states, results = [], [] for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')): @@ -71,6 +61,7 @@ def train_v(): with open(fp) as f: f.readline() for line in f: + if line.startswith('#'): continue p = line.strip().split('"') if len(p) < 3: continue sf = [float(x) for x in p[1].split(',')] @@ -88,15 +79,11 @@ def train_v(): results = np.array(results, dtype=np.float32) log(f" {len(states)} samples from {len(glob.glob(f'{csv_dir}/game_*.csv'))} games") - # load or init model — NOTE: old model.pt may be Q-network (14-output) - # V-training needs 1-output. Always start fresh for V-training. - mp = os.path.join(TRAINER_DIR, 'data/model.pt') model = PdkNet() - log(" 新V模型 (从头训练)") - opt = torch.optim.Adam(model.parameters(), lr=0.001) loss_fn = torch.nn.MSELoss() n = len(states) + t0 = datetime.now() for ep in range(20): perm = np.random.permutation(n) @@ -105,18 +92,19 @@ def train_v(): idx = perm[i:i+128] x = torch.from_numpy(states[idx]).float() y = torch.from_numpy(results[idx]).float().unsqueeze(1) - model.train() - pred = model(x) + model.train(); pred = model(x) loss = loss_fn(pred, y) - opt.zero_grad() - loss.backward() - opt.step() + opt.zero_grad(); loss.backward(); opt.step() losses.append(loss.item()) if ep % 5 == 0: - log(f" ep{ep+1}/20 loss={np.mean(losses):.4f}") - log(f" done. final loss={np.mean(losses):.4f}") + dt = (datetime.now() - t0).total_seconds() + log(f" ep{ep+1}/20 loss={np.mean(losses):.4f} ({dt:.0f}s)") - # save pytorch + elapsed = (datetime.now() - t0).total_seconds() + log(f" done {elapsed:.0f}s, final loss={np.mean(losses):.4f}") + + # save pt + mp = os.path.join(TRAINER_DIR, 'data/model.pt') model.save(mp) # export ONNX @@ -129,14 +117,12 @@ def train_v(): dynamic_axes={'state': {0: 'batch'}, 'v': {0: 'batch'}}, opset_version=15) - # copy to hjha-server (include .data if external storage) dest = os.path.join(HJHA_DIR, 'model.onnx') shutil.copy(onnx_path, dest) data_src = onnx_path + '.data' if os.path.exists(data_src): shutil.copy(data_src, dest + '.data') - # archive tag = datetime.now().strftime('%m%d_%H%M') rd = os.path.join(HJHA_DIR, f'training_rounds/r{tag}') os.makedirs(rd, exist_ok=True) @@ -148,60 +134,36 @@ def train_v(): shutil.move(csv_dir, os.path.join(rd, 'training_data')) os.makedirs(csv_dir, exist_ok=True) - sz = os.path.getsize(dest) / 1024 - log(f" ONNX → {dest} ({sz:.0f}KB), 存档 → {rd}") + sz = os.path.getsize(dest) + os.path.getsize(dest + '.data') if os.path.exists(dest + '.data') else os.path.getsize(dest) + log(f" ONNX → {dest} ({sz//1024}KB), 存档 → {rd}") except Exception as e: - log(f" ONNX失败: {e}") + log(f" ONNX failed: {e}") return False return True -# ── 主循环 ── def main(): log("=" * 50) - log(f"持续训练 启动 | 截止 {DEADLINE}") - log("策略: 首轮10k → 后续5k/轮") + log("持续训练: 20k局/轮 | 无限循环") log("=" * 50) rn, total = 0, 0 while True: - ml = mins_left() - if ml < 3: - log(f"只剩 {ml:.0f}min, 停止") - break - rn += 1 - games = 10000 if rn == 1 else 5000 - est = games/60 + 15 # estimate minutes - - if est > ml - 3: - # shrink - games = max(500, int((ml - 18) * 60)) - if games < 500: - log(f"不够时间跑完整轮 ({ml:.0f}min), 停止") - break - log(f"缩减为 {games} 局 (剩{ml:.0f}min)") - - log(f"\n── 第{rn}轮 {games}局 (剩{ml:.0f}min) ──") + games = 20000 + log(f"\n── 第{rn}轮: {games}局 ──") csv = selfplay(games) if csv == 0: - log("自对弈失败! 跳过训练, 继续下一轮") + log("自对弈失败! 跳过") continue total += games if not train_v(): - log("训练失败! 继续下一轮") + log("训练失败! 跳过") continue - - log(f"\n{'='*40}") - log(f"结束: {rn}轮, {total}局") - mp = os.path.join(HJHA_DIR, 'model.onnx') - if os.path.exists(mp): - log(f"最终模型: {mp} ({os.path.getsize(mp)/1024:.0f}KB)") - rd = os.path.join(HJHA_DIR, 'training_rounds') - if os.path.exists(rd): - log(f"存档: {len(os.listdir(rd))} 轮 → {rd}") + + log(f"累计: {rn}轮, {total}局") if __name__ == '__main__': main() \ No newline at end of file