feat: ISMCTS q-value as training label

C#: TrainingCollector新增q_value列, IsmctsBot传入winRate
CSV: reward→q_value, ISMCTS状态带真实评估值, Neural填0
Python: 优先取ISMCTS状态+评估值训练, 无q_value的Neural状态跳过
旧CSV格式(reward)自动fallback到Neural-only过滤

信号从'这局谁赢'改为'ISMCTS评估这步有多好'

PdkFriendServer/Logic/IsmctsBot.cs

@@ -392,7 +392,7 @@ namespace PdkFriendServer.Logic
                 int totalSims = allCandidates.Count * fastSims
                     + scoredCandidates.Count(c => c.winRate > 0.3) * (simsPerCandidate - fastSims);
                 Console.WriteLine($"[ISMCTS pos{view.MyPos}] {totalSims}sims => {desc}");
-                TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1);
+                TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1, (float)best.winRate);
                 return best.play;
             }
 

PdkFriendServer/Logic/TrainingCollector.cs

@@ -7,8 +7,8 @@ using GameMessage.PaoDeKuaiF;
 namespace PdkFriendServer.Logic
 {
     /// <summary>
-    /// 训练数据收集器 — 记录 (state, action, result) 三元组供 Python 训练。
-    /// 状态编码与 Python trainer 完全一致：13rank×4suit×7ch=364 float。
+    /// 训练数据收集器 — 记录 (state, action, q_value, player) 四元组供 Python 训练。
+    /// q_value: ISMCTS 对选出动作的胜率评估 (0~1), NeuralBot 填游戏结果(-1/0/1)
     /// </summary>
     public static class TrainingCollector
     {
@@ -25,14 +25,15 @@ namespace PdkFriendServer.Logic
         {
             public float[] State;
             public int Action;   // 0-12 = rank index, 13 = pass
+            public float QValue; // ISMCTS 评估值 (0~1) 或 NeuralBot 用游戏结果
             public int Player;
         }
 
-        public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player)
+        public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player, float qValue = 0f)
         {
             if (!Enabled) return;
             var state = EncodeState(view, tracker);
-            _records.Add(new TrainingRecord { State = state, Action = actionIdx, Player = player });
+            _records.Add(new TrainingRecord { State = state, Action = actionIdx, QValue = qValue, Player = player });
         }
 
         /// <summary>
@@ -45,12 +46,11 @@ namespace PdkFriendServer.Logic
             var path = $"training_data/game_{gameId:D5}.csv";
             System.IO.Directory.CreateDirectory("training_data");
             using var w = new System.IO.StreamWriter(path);
-            w.WriteLine("state,action,reward,player");
+            w.WriteLine("state,action,q_value,player");
             foreach (var r in _records)
             {
-                float reward = r.Player >= 0 && r.Player < rewards.Length ? rewards[r.Player] : 0;
                 var stateStr = string.Join(",", r.State.Select(f => f.ToString("F6")));
-                w.WriteLine($"\"{stateStr}\",{r.Action},{reward:F3},{r.Player}");
+                w.WriteLine($"\"{stateStr}\",{r.Action},{r.QValue:F3},{r.Player}");
             }
             // 游戏结果摘要 — 方便不中断训练即可评估模型进化
             w.WriteLine($"#RESULT,winner={winnerPos},bots={string.Join(",", botTypes ?? new[]{"ISMCTS","ISMCTS","ISMCTS"})}");

scripts/training_orchestrator.py

@@ -51,14 +51,18 @@ def selfplay(games):
     return csv
 
 def load_csv_states(csv_dir):
-    """Load states from CSV. If player column present, only keep Neural (player=2)."""
+    """Load states from CSV.
+    New format (q_value): prefer ISMCTS states (player!=2) with ISMCTS evaluation as target.
+    Old format (reward): fall back to Neural-only with game result."""
     import numpy as np
     states, results = [], []
-    neural_only = 0
+    filtered = 0
+    ismcts_samples = 0
     for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
         try:
             with open(fp) as f:
                 header = f.readline().strip()
+                has_qvalue = 'q_value' in header
                 has_player = header.endswith(',player')
                 for line in f:
                     if line.startswith('#'): continue
@@ -67,17 +71,29 @@ def load_csv_states(csv_dir):
                     sf = [float(x) for x in p[1].split(',')]
                     rest = p[2].strip(',').split(',')
                     if len(sf) != 364: continue
-                    if has_player and len(rest) >= 3:
-                        player = int(rest[2])
-                        if player != 2:  # only train on Neural states
-                            neural_only += 1
-                            continue
                     if len(rest) < 2: continue
+                    label = float(rest[1])  # q_value or reward
+                    if has_qvalue and has_player and len(rest) >= 3:
+                        player = int(rest[2])
+                        if player == 2 and abs(label) < 0.01:
+                            # Neural state with no q_value (0.0) → skip
+                            filtered += 1
+                            continue
+                        if player != 2:
+                            ismcts_samples += 1
+                    elif not has_qvalue and has_player and len(rest) >= 3:
+                        # Old format: Neural-only filter
+                        player = int(rest[2])
+                        if player != 2:
+                            filtered += 1
+                            continue
                     states.append(sf)
-                    results.append(float(rest[1]))
+                    results.append(label)
         except: pass
-    if neural_only:
-        log(f"  filtered out {neural_only} non-Neural states")
+    if ismcts_samples:
+        log(f"  {ismcts_samples} ISMCTS q-value samples")
+    if filtered:
+        log(f"  filtered {filtered} unusable states")
     return np.array(states, dtype=np.float32) if states else None, np.array(results, dtype=np.float32) if results else None
 
 def train_v(prev_winrate=None):