feat: Neural-only training, experience replay, per-round eval

3个训练效率提升:
1. Neural filter: V训练只取player=2数据, 排除ISMCTS状态干扰
   旧格式(无player列)自动fallback全量数据
2. 经验回放: 保留最近3轮数据混入当前轮训练, 扩大分布覆盖
   存储到 paodekuai-trainer/data/replay_buffer.pkl
3. 每轮eval: 训练后跑200局--eval测胜率, 下降>10%自动回滚
   回滚到 model_prev.pt, 防止坏模型污染下轮数据

scripts/training_orchestrator.py

@@ -1,15 +1,18 @@
 #!/usr/bin/env python3
 """
 持续训练: 20k局/轮, 无限循环
-每轮: dotnet自对弈 → V训练 → ONNX导出 → 存档
+每轮: dotnet自对弈 → V训练(Neural-only+replay) → eval → ONNX导出 → 存档
-胜率趋势由 scan_wins.py 每小时 cron 独立采集
+改进: dropout正则化, 只训Neural数据, 经验回放, 每轮eval验证回滚
 """
-import subprocess, os, sys, time, shutil, glob
+import subprocess, os, sys, time, shutil, glob, pickle, re
 from datetime import datetime
 
 HJHA_DIR = '/home/xiaoou/projects/hjha-server'
 TRAINER_DIR = '/home/xiaoou/projects/paodekuai-trainer'
 LOG = os.path.join(HJHA_DIR, 'training_loop.log')
+REPLAY_PATH = os.path.join(TRAINER_DIR, 'data/replay_buffer.pkl')
+REPLAY_MAX_ROUNDS = 3  # keep last N rounds in buffer
+EVAL_GAMES = 200  # eval games per round
 
 def now(): return datetime.now().strftime('%m%d %H:%M:%S')
 def log(msg):
@@ -29,7 +32,7 @@ def run(cmd, cwd=HJHA_DIR, to=86400):
         return False, str(e)
 
 def selfplay(games):
-    log(f"── 自対弈 {games}局 ──")
+    log(f"── 自对弈 {games}局 ──")
     td = os.path.join(HJHA_DIR, 'training_data')
     if os.path.exists(td): shutil.rmtree(td)
     os.makedirs(td)
@@ -47,7 +50,37 @@ def selfplay(games):
     log(f"  {csv} CSV")
     return csv
 
-def train_v():
+def load_csv_states(csv_dir):
+    """Load states from CSV. If player column present, only keep Neural (player=2)."""
+    import numpy as np
+    states, results = [], []
+    neural_only = 0
+    for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
+        try:
+            with open(fp) as f:
+                header = f.readline().strip()
+                has_player = header.endswith(',player')
+                for line in f:
+                    if line.startswith('#'): continue
+                    p = line.strip().split('"')
+                    if len(p) < 3: continue
+                    sf = [float(x) for x in p[1].split(',')]
+                    rest = p[2].strip(',').split(',')
+                    if len(sf) != 364: continue
+                    if has_player and len(rest) >= 3:
+                        player = int(rest[2])
+                        if player != 2:  # only train on Neural states
+                            neural_only += 1
+                            continue
+                    if len(rest) < 2: continue
+                    states.append(sf)
+                    results.append(float(rest[1]))
+        except: pass
+    if neural_only:
+        log(f"  filtered out {neural_only} non-Neural states")
+    return np.array(states, dtype=np.float32) if states else None, np.array(results, dtype=np.float32) if results else None
+
+def train_v(prev_winrate=None):
     log(f"── V训练 ──")
     
     sys.path.insert(0, TRAINER_DIR)
@@ -55,29 +88,40 @@ def train_v():
     from models.network import PdkNet
     
     csv_dir = os.path.join(HJHA_DIR, 'training_data')
-    states, results = [], []
+    cur_states, cur_results = load_csv_states(csv_dir)
-    for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
-        try:
-            with open(fp) as f:
-                f.readline()
-                for line in f:
-                    if line.startswith('#'): continue
-                    p = line.strip().split('"')
-                    if len(p) < 3: continue
-                    sf = [float(x) for x in p[1].split(',')]
-                    rest = p[2].strip(',').split(',')
-                    if len(rest) < 2 or len(sf) != 364: continue
-                    states.append(sf)
-                    results.append(float(rest[1]))
-        except: pass
     
-    if not states:
+    if cur_states is None or len(cur_states) == 0:
         log("  0 samples!")
-        return False
+        return False, None
     
-    states = np.array(states, dtype=np.float32)
+    cur_games = len(glob.glob(f'{csv_dir}/game_*.csv'))
-    results = np.array(results, dtype=np.float32)
+    log(f"  current: {len(cur_states)} Neural samples from {cur_games} games")
-    log(f"  {len(states)} samples from {len(glob.glob(f'{csv_dir}/game_*.csv'))} games")
+    
+    # --- experience replay: load past rounds ---
+    replay_loaded = 0
+    if os.path.exists(REPLAY_PATH):
+        try:
+            with open(REPLAY_PATH, 'rb') as f:
+                replay_data = pickle.load(f)
+            # replay_data is list of (states, results) tuples
+            all_r = []
+            for rs, rr in replay_data[-REPLAY_MAX_ROUNDS:]:
+                if len(rs) > 0:
+                    all_r.append((rs, rr))
+                    replay_loaded += len(rs)
+            if all_r:
+                replay_states = np.concatenate([r[0] for r in all_r])
+                replay_results = np.concatenate([r[1] for r in all_r])
+                states = np.concatenate([cur_states, replay_states])
+                results = np.concatenate([cur_results, replay_results])
+                log(f"  replay: {replay_loaded} samples from {min(len(replay_data), REPLAY_MAX_ROUNDS)} past rounds → total {len(states)}")
+            else:
+                states, results = cur_states, cur_results
+        except Exception as e:
+            log(f"  replay load failed: {e}, using current only")
+            states, results = cur_states, cur_results
+    else:
+        states, results = cur_states, cur_results
     
     model = PdkNet()
     opt = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)
@@ -117,51 +161,107 @@ def train_v():
     elapsed = (datetime.now() - t0).total_seconds()
     log(f"  done {elapsed:.0f}s, final loss={avg_loss:.4f}")
     
-    # save pt
+    # --- eval before deploying ---
+    log(f"── eval {EVAL_GAMES}局 ──")
+    model.eval()
     mp = os.path.join(TRAINER_DIR, 'data/model.pt')
     model.save(mp)
+    # export ONNX for eval
+    onnx_tmp = os.path.join(TRAINER_DIR, 'data/model.onnx')
+    dummy = torch.randn(1, 364)
+    torch.onnx.export(model, dummy, onnx_tmp,
+        input_names=['state'], output_names=['v'],
+        dynamic_axes={'state': {0: 'batch'}, 'v': {0: 'batch'}},
+        opset_version=15)
+    # copy ONNX to hjha for eval
+    eval_dest = os.path.join(HJHA_DIR, 'model.onnx')
+    shutil.copy(onnx_tmp, eval_dest)
+    data_src = onnx_tmp + '.data'
+    if os.path.exists(data_src):
+        shutil.copy(data_src, eval_dest + '.data')
     
-    # export ONNX
+    winrate = 0.0
     try:
-        model.eval()
+        ok, out = run(['/usr/lib/dotnet/dotnet', 'run', '--project', 'hjha-console', '-c', 'Release',
-        dummy = torch.randn(1, 364)
+                       '--', '--eval', str(EVAL_GAMES)], to=600)
-        onnx_path = os.path.join(TRAINER_DIR, 'data/model.onnx')
+        sf = os.path.join(HJHA_DIR, 'stress_summary.txt')
-        torch.onnx.export(model, dummy, onnx_path,
+        if os.path.exists(sf):
-            input_names=['state'], output_names=['v'],
+            with open(sf) as f:
-            dynamic_axes={'state': {0: 'batch'}, 'v': {0: 'batch'}},
+                summary = f.read()
-            opset_version=15)
+            log(f"  {summary.strip()}")
-        
+            # parse Neural win rate from summary
-        dest = os.path.join(HJHA_DIR, 'model.onnx')
+            m = re.search(r'NEURAL.*?(\d+)\s*wins?.*?(\d+)\s*games?', summary, re.IGNORECASE)
-        shutil.copy(onnx_path, dest)
+            if not m:
-        data_src = onnx_path + '.data'
+                m = re.search(r'position.*?(\d+).*?wins?.*?(\d+)', summary, re.IGNORECASE)
-        if os.path.exists(data_src):
+            if m:
-            shutil.copy(data_src, dest + '.data')
+                wins, games = int(m.group(1)), int(m.group(2))
-        
+                winrate = wins / games * 100 if games > 0 else 0.0
-        tag = datetime.now().strftime('%m%d_%H%M')
+        log(f"  Neural winrate: {winrate:.1f}%")
-        rd = os.path.join(HJHA_DIR, f'training_rounds/r{tag}')
-        os.makedirs(rd, exist_ok=True)
-        shutil.copy(onnx_path, os.path.join(rd, 'model.onnx'))
-        if os.path.exists(data_src):
-            shutil.copy(data_src, os.path.join(rd, 'model.onnx.data'))
-        shutil.copy(mp, os.path.join(rd, 'model.pt'))
-        if os.path.exists(csv_dir):
-            shutil.move(csv_dir, os.path.join(rd, 'training_data'))
-        os.makedirs(csv_dir, exist_ok=True)
-        
-        sz = os.path.getsize(dest) + os.path.getsize(dest + '.data') if os.path.exists(dest + '.data') else os.path.getsize(dest)
-        log(f"  ONNX → {dest} ({sz//1024}KB), 存档 → {rd}")
     except Exception as e:
-        log(f"  ONNX failed: {e}")
+        log(f"  eval failed: {e}")
-        return False
     
-    return True
+    # --- rollback if worse ---
+    if prev_winrate is not None and winrate < prev_winrate * 0.9:
+        log(f"  ⚠️ winrate dropped {prev_winrate:.1f}% → {winrate:.1f}%, rolling back model")
+        # restore previous model.pt and ONNX
+        prev_pt = os.path.join(TRAINER_DIR, 'data/model_prev.pt')
+        if os.path.exists(prev_pt):
+            shutil.copy(prev_pt, mp)
+            log(f"  restored model.pt from backup")
+        return False, prev_winrate
+    
+    # --- save and deploy ---
+    # backup current as prev
+    if os.path.exists(mp):
+        shutil.copy(mp, os.path.join(TRAINER_DIR, 'data/model_prev.pt'))
+    
+    # final ONNX copy to hjha (already done above for eval, but ensure)
+    dest = os.path.join(HJHA_DIR, 'model.onnx')
+    shutil.copy(onnx_tmp, dest)
+    if os.path.exists(data_src):
+        shutil.copy(data_src, dest + '.data')
+    
+    # archive round
+    tag = datetime.now().strftime('%m%d_%H%M')
+    rd = os.path.join(HJHA_DIR, f'training_rounds/r{tag}')
+    os.makedirs(rd, exist_ok=True)
+    shutil.copy(onnx_tmp, os.path.join(rd, 'model.onnx'))
+    if os.path.exists(data_src):
+        shutil.copy(data_src, os.path.join(rd, 'model.onnx.data'))
+    shutil.copy(mp, os.path.join(rd, 'model.pt'))
+    if os.path.exists(csv_dir):
+        shutil.move(csv_dir, os.path.join(rd, 'training_data'))
+    os.makedirs(csv_dir, exist_ok=True)
+    
+    sz = os.path.getsize(dest) + (os.path.getsize(dest + '.data') if os.path.exists(dest + '.data') else 0)
+    log(f"  ONNX → {dest} ({sz//1024}KB), 存档 → {rd}")
+    
+    # --- update replay buffer ---
+    try:
+        if os.path.exists(REPLAY_PATH):
+            with open(REPLAY_PATH, 'rb') as f:
+                replay_data = pickle.load(f)
+        else:
+            replay_data = []
+        replay_data.append((cur_states, cur_results))
+        if len(replay_data) > REPLAY_MAX_ROUNDS * 2:
+            replay_data = replay_data[-REPLAY_MAX_ROUNDS * 2:]
+        with open(REPLAY_PATH, 'wb') as f:
+            pickle.dump(replay_data, f)
+        log(f"  replay buffer: {len(replay_data)} rounds stored")
+    except Exception as e:
+        log(f"  replay save failed: {e}")
+    
+    return True, winrate
 
 def main():
     log("=" * 50)
-    log("持续训练: 20k局/轮 | 无限循环")
+    log("持续训练: 20k局/轮 | Neural-only + replay + eval")
     log("=" * 50)
     
     rn, total = 0, 0
+    winrate = None  # track best winrate for rollback
+    
     while True:
         rn += 1
         games = 20000
@@ -169,15 +269,18 @@ def main():
         
         csv = selfplay(games)
         if csv == 0:
-            log("自対弈失败! 跳过")
+            log("自对弈失败! 跳过")
             continue
         total += games
         
-        if not train_v():
+        ok, wr = train_v(prev_winrate=winrate)
-            log("训练失败! 跳过")
+        if not ok:
+            log("训练/eval失败! 跳过")
             continue
+        if wr is not None:
+            winrate = wr
         
-        log(f"累计: {rn}轮, {total}局")
+        log(f"累计: {rn}轮, {total}局, best wr={winrate:.1f}%" if winrate else f"累计: {rn}轮, {total}局")
 
 if __name__ == '__main__':
     main()