fix: backup before selfplay, never rmtree — crash-safe

rmtree(training_data) → mv to training_data_prev
训练成功后再清理backup, 崩了数据不丢

PdkFriendServer/Logic/IsmctsBot.cs

@@ -392,7 +392,7 @@ namespace PdkFriendServer.Logic
                 int totalSims = allCandidates.Count * fastSims
                     + scoredCandidates.Count(c => c.winRate > 0.3) * (simsPerCandidate - fastSims);
                 Console.WriteLine($"[ISMCTS pos{view.MyPos}] {totalSims}sims => {desc}");
-                TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1);
+                TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1, (float)best.winRate);
                 return best.play;
             }
 

PdkFriendServer/Logic/TrainingCollector.cs

@@ -7,8 +7,8 @@ using GameMessage.PaoDeKuaiF;
 namespace PdkFriendServer.Logic
 {
     /// <summary>
-    /// 训练数据收集器 — 记录 (state, action, result) 三元组供 Python 训练。
-    /// 状态编码与 Python trainer 完全一致：13rank×4suit×7ch=364 float。
+    /// 训练数据收集器 — 记录 (state, action, q_value, player) 四元组供 Python 训练。
+    /// q_value: ISMCTS 对选出动作的胜率评估 (0~1), NeuralBot 填游戏结果(-1/0/1)
     /// </summary>
     public static class TrainingCollector
     {
@@ -25,14 +25,15 @@ namespace PdkFriendServer.Logic
         {
             public float[] State;
             public int Action;   // 0-12 = rank index, 13 = pass
+            public float QValue; // ISMCTS 评估值 (0~1) 或 NeuralBot 用游戏结果
             public int Player;
         }
 
-        public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player)
+        public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player, float qValue = 0f)
         {
             if (!Enabled) return;
             var state = EncodeState(view, tracker);
-            _records.Add(new TrainingRecord { State = state, Action = actionIdx, Player = player });
+            _records.Add(new TrainingRecord { State = state, Action = actionIdx, QValue = qValue, Player = player });
         }
 
         /// <summary>
@@ -45,12 +46,11 @@ namespace PdkFriendServer.Logic
             var path = $"training_data/game_{gameId:D5}.csv";
             System.IO.Directory.CreateDirectory("training_data");
             using var w = new System.IO.StreamWriter(path);
-            w.WriteLine("state,action,reward,player");
+            w.WriteLine("state,action,q_value,player");
             foreach (var r in _records)
             {
-                float reward = r.Player >= 0 && r.Player < rewards.Length ? rewards[r.Player] : 0;
                 var stateStr = string.Join(",", r.State.Select(f => f.ToString("F6")));
-                w.WriteLine($"\"{stateStr}\",{r.Action},{reward:F3},{r.Player}");
+                w.WriteLine($"\"{stateStr}\",{r.Action},{r.QValue:F3},{r.Player}");
             }
             // 游戏结果摘要 — 方便不中断训练即可评估模型进化
             w.WriteLine($"#RESULT,winner={winnerPos},bots={string.Join(",", botTypes ?? new[]{"ISMCTS","ISMCTS","ISMCTS"})}");

hjha-console/Program.cs

@@ -7,10 +7,18 @@ namespace hjha_console
 {
     class Program
     {
+        public static int RotateInterval = 0; // 0=disabled, N=rotate every N rounds
+        private static readonly string[][] Rotations = new[] {    // 3种排列
+            new[] { "ISMCTS", "ISMCTS", "NEURAL" },
+            new[] { "NEURAL", "ISMCTS", "ISMCTS" },
+            new[] { "ISMCTS", "NEURAL", "ISMCTS" }
+        };
+
         static void Main(string[] args)
         {
             if (args.Length > 0 && args[0] == "--eval") { PdkGameMain.BotTypes = new[] { "ISMCTS", "ISMCTS", "NEURAL" }; TrainingCollector.Enabled = false; args = new[] { "--stress", args.Length > 1 ? args[1] : "10" }; }
             if (args.Length > 0 && args[0] == "--mix") { PdkGameMain.BotTypes = new[] { "ISMCTS", "ISMCTS", "NEURAL" }; TrainingCollector.Enabled = true; args = new[] { "--stress", args.Length > 1 ? args[1] : "10" }; }
+            if (args.Length > 0 && args[0] == "--mix-rotate") { PdkGameMain.BotTypes = new[] { "ISMCTS", "ISMCTS", "NEURAL" }; TrainingCollector.Enabled = true; Program.RotateInterval = 10; args = new[] { "--stress", args.Length > 1 ? args[1] : "10" }; }
             if (args.Length > 0 && args[0] == "--mix-ismcts") { PdkGameMain.BotTypes = new[] { "ISMCTS", "ISMCTS", "ISMCTS" }; TrainingCollector.Enabled = true; args = new[] { "--stress", args.Length > 1 ? args[1] : "10" }; }
             if (args.Length > 0 && args[0] == "--bots") { PdkGameMain.BotTypes = args[1].Split(','); TrainingCollector.Enabled = true; args = args.Length > 2 ? new[] { "--stress", args[2] } : new[] { "--stress", "10" }; }
             if (args.Length > 0 && args[0] == "--stress")
@@ -52,6 +60,13 @@ namespace hjha_console
 
             for (int r = 1; r <= totalRounds; r++)
             {
+                // 位置轮换: 每N局切换排列
+                if (RotateInterval > 0 && (r - 1) % RotateInterval == 0)
+                {
+                    int ri = ((r - 1) / RotateInterval) % Rotations.Length;
+                    PdkGameMain.BotTypes = Rotations[ri];
+                }
+
                 var roundSw = Stopwatch.StartNew();
                 try
                 {

scripts/training_orchestrator.py

@@ -1,15 +1,18 @@
 #!/usr/bin/env python3
 """
 持续训练: 20k局/轮, 无限循环
-每轮: dotnet自对弈 → V训练 → ONNX导出 → 存档
-胜率趋势由 scan_wins.py 每小时 cron 独立采集
+每轮: dotnet自对弈 → V训练(Neural-only+replay) → eval → ONNX导出 → 存档
+改进: dropout正则化, 只训Neural数据, 经验回放, 每轮eval验证回滚
 """
-import subprocess, os, sys, time, shutil, glob
+import subprocess, os, sys, time, shutil, glob, pickle, re
 from datetime import datetime
 
 HJHA_DIR = '/home/xiaoou/projects/hjha-server'
 TRAINER_DIR = '/home/xiaoou/projects/paodekuai-trainer'
 LOG = os.path.join(HJHA_DIR, 'training_loop.log')
+REPLAY_PATH = os.path.join(TRAINER_DIR, 'data/replay_buffer.pkl')
+REPLAY_MAX_ROUNDS = 3  # keep last N rounds in buffer
+EVAL_GAMES = 200  # eval games per round
 
 def now(): return datetime.now().strftime('%m%d %H:%M:%S')
 def log(msg):
@@ -31,14 +34,18 @@ def run(cmd, cwd=HJHA_DIR, to=86400):
 def selfplay(games):
     log(f"── 自对弈 {games}局 ──")
     td = os.path.join(HJHA_DIR, 'training_data')
-    if os.path.exists(td): shutil.rmtree(td)
+    # 先备份旧数据, 崩了也不丢
+    if os.path.exists(td):
+        backup = f"{td}_prev"
+        if os.path.exists(backup): shutil.rmtree(backup)
+        shutil.move(td, backup)
     os.makedirs(td)
     
     env = os.environ.copy()
     env['PATH'] = f"/usr/lib/dotnet:{env.get('PATH','')}"
     
     ok, out = run(['/usr/lib/dotnet/dotnet', 'run', '--project', 'hjha-console', '-c', 'Release',
-                   '--', '--mix', str(games)], to=max(36000, games*2))
+                   '--', '--mix-rotate', str(games)], to=max(36000, games*2))
     
     csv = len(glob.glob(os.path.join(td, 'game_*.csv')))
     sf = os.path.join(HJHA_DIR, 'stress_summary.txt')
@@ -47,7 +54,53 @@ def selfplay(games):
     log(f"  {csv} CSV")
     return csv
 
-def train_v():
+def load_csv_states(csv_dir):
+    """Load states from CSV.
+    New format (q_value): prefer ISMCTS states (player!=2) with ISMCTS evaluation as target.
+    Old format (reward): fall back to Neural-only with game result."""
+    import numpy as np
+    states, results = [], []
+    filtered = 0
+    ismcts_samples = 0
+    for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
+        try:
+            with open(fp) as f:
+                header = f.readline().strip()
+                has_qvalue = 'q_value' in header
+                has_player = header.endswith(',player')
+                for line in f:
+                    if line.startswith('#'): continue
+                    p = line.strip().split('"')
+                    if len(p) < 3: continue
+                    sf = [float(x) for x in p[1].split(',')]
+                    rest = p[2].strip(',').split(',')
+                    if len(sf) != 364: continue
+                    if len(rest) < 2: continue
+                    label = float(rest[1])  # q_value or reward
+                    if has_qvalue and has_player and len(rest) >= 3:
+                        player = int(rest[2])
+                        if player == 2 and abs(label) < 0.01:
+                            # Neural state with no q_value (0.0) → skip
+                            filtered += 1
+                            continue
+                        if player != 2:
+                            ismcts_samples += 1
+                    elif not has_qvalue and has_player and len(rest) >= 3:
+                        # Old format: Neural-only filter
+                        player = int(rest[2])
+                        if player != 2:
+                            filtered += 1
+                            continue
+                    states.append(sf)
+                    results.append(label)
+        except: pass
+    if ismcts_samples:
+        log(f"  {ismcts_samples} ISMCTS q-value samples")
+    if filtered:
+        log(f"  filtered {filtered} unusable states")
+    return np.array(states, dtype=np.float32) if states else None, np.array(results, dtype=np.float32) if results else None
+
+def train_v(prev_winrate=None):
     log(f"── V训练 ──")
     
     sys.path.insert(0, TRAINER_DIR)
@@ -55,37 +108,51 @@ def train_v():
     from models.network import PdkNet
     
     csv_dir = os.path.join(HJHA_DIR, 'training_data')
-    states, results = [], []
-    for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
-        try:
-            with open(fp) as f:
-                f.readline()
-                for line in f:
-                    if line.startswith('#'): continue
-                    p = line.strip().split('"')
-                    if len(p) < 3: continue
-                    sf = [float(x) for x in p[1].split(',')]
-                    rest = p[2].strip(',').split(',')
-                    if len(rest) < 2 or len(sf) != 364: continue
-                    states.append(sf)
-                    results.append(float(rest[1]))
-        except: pass
+    cur_states, cur_results = load_csv_states(csv_dir)
     
-    if not states:
+    if cur_states is None or len(cur_states) == 0:
         log("  0 samples!")
-        return False
+        return False, None
     
-    states = np.array(states, dtype=np.float32)
-    results = np.array(results, dtype=np.float32)
-    log(f"  {len(states)} samples from {len(glob.glob(f'{csv_dir}/game_*.csv'))} games")
+    cur_games = len(glob.glob(f'{csv_dir}/game_*.csv'))
+    log(f"  current: {len(cur_states)} Neural samples from {cur_games} games")
+    
+    # --- experience replay: load past rounds ---
+    replay_loaded = 0
+    if os.path.exists(REPLAY_PATH):
+        try:
+            with open(REPLAY_PATH, 'rb') as f:
+                replay_data = pickle.load(f)
+            # replay_data is list of (states, results) tuples
+            all_r = []
+            for rs, rr in replay_data[-REPLAY_MAX_ROUNDS:]:
+                if len(rs) > 0:
+                    all_r.append((rs, rr))
+                    replay_loaded += len(rs)
+            if all_r:
+                replay_states = np.concatenate([r[0] for r in all_r])
+                replay_results = np.concatenate([r[1] for r in all_r])
+                states = np.concatenate([cur_states, replay_states])
+                results = np.concatenate([cur_results, replay_results])
+                log(f"  replay: {replay_loaded} samples from {min(len(replay_data), REPLAY_MAX_ROUNDS)} past rounds → total {len(states)}")
+            else:
+                states, results = cur_states, cur_results
+        except Exception as e:
+            log(f"  replay load failed: {e}, using current only")
+            states, results = cur_states, cur_results
+    else:
+        states, results = cur_states, cur_results
     
     model = PdkNet()
-    opt = torch.optim.Adam(model.parameters(), lr=0.001)
+    opt = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)
+    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(opt, T_max=50, eta_min=1e-5)
     loss_fn = torch.nn.MSELoss()
     n = len(states)
     t0 = datetime.now()
+    best_loss = float('inf')
+    no_improve = 0
     
-    for ep in range(20):
+    for ep in range(50):
         perm = np.random.permutation(n)
         losses = []
         for i in range(0, n, 128):
@@ -96,58 +163,125 @@ def train_v():
             loss = loss_fn(pred, y)
             opt.zero_grad(); loss.backward(); opt.step()
             losses.append(loss.item())
-        if ep % 5 == 0:
+        scheduler.step()
+        avg_loss = np.mean(losses)
+        if avg_loss < best_loss * 0.999:
+            best_loss = avg_loss
+            no_improve = 0
+        else:
+            no_improve += 1
+        if ep % 5 == 0 or ep == 49:
             dt = (datetime.now() - t0).total_seconds()
-            log(f"  ep{ep+1}/20 loss={np.mean(losses):.4f} ({dt:.0f}s)")
+            lr = scheduler.get_last_lr()[0]
+            log(f"  ep{ep+1}/50 loss={avg_loss:.4f} lr={lr:.6f} ({dt:.0f}s)")
+        if no_improve >= 10:
+            log(f"  early stop ep{ep+1}, plateau 10 epochs")
+            break
     
     elapsed = (datetime.now() - t0).total_seconds()
-    log(f"  done {elapsed:.0f}s, final loss={np.mean(losses):.4f}")
+    log(f"  done {elapsed:.0f}s, final loss={avg_loss:.4f}")
     
-    # save pt
+    # --- eval before deploying ---
+    log(f"── eval {EVAL_GAMES}局 ──")
+    model.eval()
     mp = os.path.join(TRAINER_DIR, 'data/model.pt')
     model.save(mp)
+    # export ONNX for eval
+    onnx_tmp = os.path.join(TRAINER_DIR, 'data/model.onnx')
+    dummy = torch.randn(1, 364)
+    torch.onnx.export(model, dummy, onnx_tmp,
+        input_names=['state'], output_names=['v'],
+        dynamic_axes={'state': {0: 'batch'}, 'v': {0: 'batch'}},
+        opset_version=15)
+    # copy ONNX to hjha for eval
+    eval_dest = os.path.join(HJHA_DIR, 'model.onnx')
+    shutil.copy(onnx_tmp, eval_dest)
+    data_src = onnx_tmp + '.data'
+    if os.path.exists(data_src):
+        shutil.copy(data_src, eval_dest + '.data')
     
-    # export ONNX
+    winrate = 0.0
     try:
-        model.eval()
-        dummy = torch.randn(1, 364)
-        onnx_path = os.path.join(TRAINER_DIR, 'data/model.onnx')
-        torch.onnx.export(model, dummy, onnx_path,
-            input_names=['state'], output_names=['v'],
-            dynamic_axes={'state': {0: 'batch'}, 'v': {0: 'batch'}},
-            opset_version=15)
-        
-        dest = os.path.join(HJHA_DIR, 'model.onnx')
-        shutil.copy(onnx_path, dest)
-        data_src = onnx_path + '.data'
-        if os.path.exists(data_src):
-            shutil.copy(data_src, dest + '.data')
-        
-        tag = datetime.now().strftime('%m%d_%H%M')
-        rd = os.path.join(HJHA_DIR, f'training_rounds/r{tag}')
-        os.makedirs(rd, exist_ok=True)
-        shutil.copy(onnx_path, os.path.join(rd, 'model.onnx'))
-        if os.path.exists(data_src):
-            shutil.copy(data_src, os.path.join(rd, 'model.onnx.data'))
-        shutil.copy(mp, os.path.join(rd, 'model.pt'))
-        if os.path.exists(csv_dir):
-            shutil.move(csv_dir, os.path.join(rd, 'training_data'))
-        os.makedirs(csv_dir, exist_ok=True)
-        
-        sz = os.path.getsize(dest) + os.path.getsize(dest + '.data') if os.path.exists(dest + '.data') else os.path.getsize(dest)
-        log(f"  ONNX → {dest} ({sz//1024}KB), 存档 → {rd}")
+        ok, out = run(['/usr/lib/dotnet/dotnet', 'run', '--project', 'hjha-console', '-c', 'Release',
+                       '--', '--eval', str(EVAL_GAMES)], to=600)
+        sf = os.path.join(HJHA_DIR, 'stress_summary.txt')
+        if os.path.exists(sf):
+            with open(sf) as f:
+                summary = f.read()
+            log(f"  {summary.strip()}")
+            # parse Neural win rate from summary
+            m = re.search(r'NEURAL.*?(\d+)\s*wins?.*?(\d+)\s*games?', summary, re.IGNORECASE)
+            if not m:
+                m = re.search(r'position.*?(\d+).*?wins?.*?(\d+)', summary, re.IGNORECASE)
+            if m:
+                wins, games = int(m.group(1)), int(m.group(2))
+                winrate = wins / games * 100 if games > 0 else 0.0
+        log(f"  Neural winrate: {winrate:.1f}%")
     except Exception as e:
-        log(f"  ONNX failed: {e}")
-        return False
+        log(f"  eval failed: {e}")
     
-    return True
+    # --- rollback if worse ---
+    if prev_winrate is not None and winrate < prev_winrate * 0.9:
+        log(f"  ⚠️ winrate dropped {prev_winrate:.1f}% → {winrate:.1f}%, rolling back model")
+        # restore previous model.pt and ONNX
+        prev_pt = os.path.join(TRAINER_DIR, 'data/model_prev.pt')
+        if os.path.exists(prev_pt):
+            shutil.copy(prev_pt, mp)
+            log(f"  restored model.pt from backup")
+        return False, prev_winrate
+    
+    # --- save and deploy ---
+    # backup current as prev
+    if os.path.exists(mp):
+        shutil.copy(mp, os.path.join(TRAINER_DIR, 'data/model_prev.pt'))
+    
+    # final ONNX copy to hjha (already done above for eval, but ensure)
+    dest = os.path.join(HJHA_DIR, 'model.onnx')
+    shutil.copy(onnx_tmp, dest)
+    if os.path.exists(data_src):
+        shutil.copy(data_src, dest + '.data')
+    
+    # archive round
+    tag = datetime.now().strftime('%m%d_%H%M')
+    rd = os.path.join(HJHA_DIR, f'training_rounds/r{tag}')
+    os.makedirs(rd, exist_ok=True)
+    shutil.copy(onnx_tmp, os.path.join(rd, 'model.onnx'))
+    if os.path.exists(data_src):
+        shutil.copy(data_src, os.path.join(rd, 'model.onnx.data'))
+    shutil.copy(mp, os.path.join(rd, 'model.pt'))
+    if os.path.exists(csv_dir):
+        shutil.move(csv_dir, os.path.join(rd, 'training_data'))
+    os.makedirs(csv_dir, exist_ok=True)
+    
+    sz = os.path.getsize(dest) + (os.path.getsize(dest + '.data') if os.path.exists(dest + '.data') else 0)
+    log(f"  ONNX → {dest} ({sz//1024}KB), 存档 → {rd}")
+    
+    # --- update replay buffer ---
+    try:
+        if os.path.exists(REPLAY_PATH):
+            with open(REPLAY_PATH, 'rb') as f:
+                replay_data = pickle.load(f)
+        else:
+            replay_data = []
+        replay_data.append((cur_states, cur_results))
+        if len(replay_data) > REPLAY_MAX_ROUNDS * 2:
+            replay_data = replay_data[-REPLAY_MAX_ROUNDS * 2:]
+        with open(REPLAY_PATH, 'wb') as f:
+            pickle.dump(replay_data, f)
+        log(f"  replay buffer: {len(replay_data)} rounds stored")
+    except Exception as e:
+        log(f"  replay save failed: {e}")
+    
+    return True, winrate
 
 def main():
     log("=" * 50)
-    log("持续训练: 20k局/轮 | 无限循环")
+    log("持续训练: 20k局/轮 | Neural-only + replay + eval")
     log("=" * 50)
     
     rn, total = 0, 0
+    winrate = None  # track best winrate for rollback
+    
     while True:
         rn += 1
         games = 20000
@@ -159,11 +293,17 @@ def main():
             continue
         total += games
         
-        if not train_v():
-            log("训练失败! 跳过")
+        ok, wr = train_v(prev_winrate=winrate)
+        if not ok:
+            log("训练/eval失败! 跳过")
             continue
+        # 训练成功, 清理备份
+        backup = os.path.join(HJHA_DIR, 'training_data_prev')
+        if os.path.exists(backup): shutil.rmtree(backup)
+        if wr is not None:
+            winrate = wr
         
-        log(f"累计: {rn}轮, {total}局")
+        log(f"累计: {rn}轮, {total}局, best wr={winrate:.1f}%" if winrate else f"累计: {rn}轮, {total}局")
 
 if __name__ == '__main__':
-    main()
+    main()