diff --git a/PdkFriendServer/Logic/IsmctsBot.cs b/PdkFriendServer/Logic/IsmctsBot.cs index 9ccd2444..6b159f28 100644 --- a/PdkFriendServer/Logic/IsmctsBot.cs +++ b/PdkFriendServer/Logic/IsmctsBot.cs @@ -392,7 +392,7 @@ namespace PdkFriendServer.Logic int totalSims = allCandidates.Count * fastSims + scoredCandidates.Count(c => c.winRate > 0.3) * (simsPerCandidate - fastSims); Console.WriteLine($"[ISMCTS pos{view.MyPos}] {totalSims}sims => {desc}"); - TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1); + TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1, (float)best.winRate); return best.play; } diff --git a/PdkFriendServer/Logic/TrainingCollector.cs b/PdkFriendServer/Logic/TrainingCollector.cs index 4356c556..0dd608f3 100644 --- a/PdkFriendServer/Logic/TrainingCollector.cs +++ b/PdkFriendServer/Logic/TrainingCollector.cs @@ -7,8 +7,8 @@ using GameMessage.PaoDeKuaiF; namespace PdkFriendServer.Logic { /// - /// 训练数据收集器 — 记录 (state, action, result) 三元组供 Python 训练。 - /// 状态编码与 Python trainer 完全一致:13rank×4suit×7ch=364 float。 + /// 训练数据收集器 — 记录 (state, action, q_value, player) 四元组供 Python 训练。 + /// q_value: ISMCTS 对选出动作的胜率评估 (0~1), NeuralBot 填游戏结果(-1/0/1) /// public static class TrainingCollector { @@ -25,14 +25,15 @@ namespace PdkFriendServer.Logic { public float[] State; public int Action; // 0-12 = rank index, 13 = pass + public float QValue; // ISMCTS 评估值 (0~1) 或 NeuralBot 用游戏结果 public int Player; } - public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player) + public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player, float qValue = 0f) { if (!Enabled) return; var state = EncodeState(view, tracker); - _records.Add(new TrainingRecord { State = state, Action = actionIdx, Player = player }); + _records.Add(new TrainingRecord { State = state, Action = actionIdx, QValue = qValue, Player = player }); } /// @@ -45,12 +46,11 @@ namespace PdkFriendServer.Logic var path = $"training_data/game_{gameId:D5}.csv"; System.IO.Directory.CreateDirectory("training_data"); using var w = new System.IO.StreamWriter(path); - w.WriteLine("state,action,reward,player"); + w.WriteLine("state,action,q_value,player"); foreach (var r in _records) { - float reward = r.Player >= 0 && r.Player < rewards.Length ? rewards[r.Player] : 0; var stateStr = string.Join(",", r.State.Select(f => f.ToString("F6"))); - w.WriteLine($"\"{stateStr}\",{r.Action},{reward:F3},{r.Player}"); + w.WriteLine($"\"{stateStr}\",{r.Action},{r.QValue:F3},{r.Player}"); } // 游戏结果摘要 — 方便不中断训练即可评估模型进化 w.WriteLine($"#RESULT,winner={winnerPos},bots={string.Join(",", botTypes ?? new[]{"ISMCTS","ISMCTS","ISMCTS"})}"); diff --git a/scripts/training_orchestrator.py b/scripts/training_orchestrator.py index 0f31862b..d7b89ecd 100644 --- a/scripts/training_orchestrator.py +++ b/scripts/training_orchestrator.py @@ -51,14 +51,18 @@ def selfplay(games): return csv def load_csv_states(csv_dir): - """Load states from CSV. If player column present, only keep Neural (player=2).""" + """Load states from CSV. + New format (q_value): prefer ISMCTS states (player!=2) with ISMCTS evaluation as target. + Old format (reward): fall back to Neural-only with game result.""" import numpy as np states, results = [], [] - neural_only = 0 + filtered = 0 + ismcts_samples = 0 for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')): try: with open(fp) as f: header = f.readline().strip() + has_qvalue = 'q_value' in header has_player = header.endswith(',player') for line in f: if line.startswith('#'): continue @@ -67,17 +71,29 @@ def load_csv_states(csv_dir): sf = [float(x) for x in p[1].split(',')] rest = p[2].strip(',').split(',') if len(sf) != 364: continue - if has_player and len(rest) >= 3: - player = int(rest[2]) - if player != 2: # only train on Neural states - neural_only += 1 - continue if len(rest) < 2: continue + label = float(rest[1]) # q_value or reward + if has_qvalue and has_player and len(rest) >= 3: + player = int(rest[2]) + if player == 2 and abs(label) < 0.01: + # Neural state with no q_value (0.0) → skip + filtered += 1 + continue + if player != 2: + ismcts_samples += 1 + elif not has_qvalue and has_player and len(rest) >= 3: + # Old format: Neural-only filter + player = int(rest[2]) + if player != 2: + filtered += 1 + continue states.append(sf) - results.append(float(rest[1])) + results.append(label) except: pass - if neural_only: - log(f" filtered out {neural_only} non-Neural states") + if ismcts_samples: + log(f" {ismcts_samples} ISMCTS q-value samples") + if filtered: + log(f" filtered {filtered} unusable states") return np.array(states, dtype=np.float32) if states else None, np.array(results, dtype=np.float32) if results else None def train_v(prev_winrate=None):