diff --git a/PdkFriendServer/Logic/IsmctsBot.cs b/PdkFriendServer/Logic/IsmctsBot.cs
index 9ccd2444..6b159f28 100644
--- a/PdkFriendServer/Logic/IsmctsBot.cs
+++ b/PdkFriendServer/Logic/IsmctsBot.cs
@@ -392,7 +392,7 @@ namespace PdkFriendServer.Logic
int totalSims = allCandidates.Count * fastSims
+ scoredCandidates.Count(c => c.winRate > 0.3) * (simsPerCandidate - fastSims);
Console.WriteLine($"[ISMCTS pos{view.MyPos}] {totalSims}sims => {desc}");
- TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1);
+ TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1, (float)best.winRate);
return best.play;
}
diff --git a/PdkFriendServer/Logic/TrainingCollector.cs b/PdkFriendServer/Logic/TrainingCollector.cs
index 4356c556..0dd608f3 100644
--- a/PdkFriendServer/Logic/TrainingCollector.cs
+++ b/PdkFriendServer/Logic/TrainingCollector.cs
@@ -7,8 +7,8 @@ using GameMessage.PaoDeKuaiF;
namespace PdkFriendServer.Logic
{
///
- /// 训练数据收集器 — 记录 (state, action, result) 三元组供 Python 训练。
- /// 状态编码与 Python trainer 完全一致:13rank×4suit×7ch=364 float。
+ /// 训练数据收集器 — 记录 (state, action, q_value, player) 四元组供 Python 训练。
+ /// q_value: ISMCTS 对选出动作的胜率评估 (0~1), NeuralBot 填游戏结果(-1/0/1)
///
public static class TrainingCollector
{
@@ -25,14 +25,15 @@ namespace PdkFriendServer.Logic
{
public float[] State;
public int Action; // 0-12 = rank index, 13 = pass
+ public float QValue; // ISMCTS 评估值 (0~1) 或 NeuralBot 用游戏结果
public int Player;
}
- public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player)
+ public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player, float qValue = 0f)
{
if (!Enabled) return;
var state = EncodeState(view, tracker);
- _records.Add(new TrainingRecord { State = state, Action = actionIdx, Player = player });
+ _records.Add(new TrainingRecord { State = state, Action = actionIdx, QValue = qValue, Player = player });
}
///
@@ -45,12 +46,11 @@ namespace PdkFriendServer.Logic
var path = $"training_data/game_{gameId:D5}.csv";
System.IO.Directory.CreateDirectory("training_data");
using var w = new System.IO.StreamWriter(path);
- w.WriteLine("state,action,reward,player");
+ w.WriteLine("state,action,q_value,player");
foreach (var r in _records)
{
- float reward = r.Player >= 0 && r.Player < rewards.Length ? rewards[r.Player] : 0;
var stateStr = string.Join(",", r.State.Select(f => f.ToString("F6")));
- w.WriteLine($"\"{stateStr}\",{r.Action},{reward:F3},{r.Player}");
+ w.WriteLine($"\"{stateStr}\",{r.Action},{r.QValue:F3},{r.Player}");
}
// 游戏结果摘要 — 方便不中断训练即可评估模型进化
w.WriteLine($"#RESULT,winner={winnerPos},bots={string.Join(",", botTypes ?? new[]{"ISMCTS","ISMCTS","ISMCTS"})}");
diff --git a/scripts/training_orchestrator.py b/scripts/training_orchestrator.py
index 0f31862b..d7b89ecd 100644
--- a/scripts/training_orchestrator.py
+++ b/scripts/training_orchestrator.py
@@ -51,14 +51,18 @@ def selfplay(games):
return csv
def load_csv_states(csv_dir):
- """Load states from CSV. If player column present, only keep Neural (player=2)."""
+ """Load states from CSV.
+ New format (q_value): prefer ISMCTS states (player!=2) with ISMCTS evaluation as target.
+ Old format (reward): fall back to Neural-only with game result."""
import numpy as np
states, results = [], []
- neural_only = 0
+ filtered = 0
+ ismcts_samples = 0
for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
try:
with open(fp) as f:
header = f.readline().strip()
+ has_qvalue = 'q_value' in header
has_player = header.endswith(',player')
for line in f:
if line.startswith('#'): continue
@@ -67,17 +71,29 @@ def load_csv_states(csv_dir):
sf = [float(x) for x in p[1].split(',')]
rest = p[2].strip(',').split(',')
if len(sf) != 364: continue
- if has_player and len(rest) >= 3:
- player = int(rest[2])
- if player != 2: # only train on Neural states
- neural_only += 1
- continue
if len(rest) < 2: continue
+ label = float(rest[1]) # q_value or reward
+ if has_qvalue and has_player and len(rest) >= 3:
+ player = int(rest[2])
+ if player == 2 and abs(label) < 0.01:
+ # Neural state with no q_value (0.0) → skip
+ filtered += 1
+ continue
+ if player != 2:
+ ismcts_samples += 1
+ elif not has_qvalue and has_player and len(rest) >= 3:
+ # Old format: Neural-only filter
+ player = int(rest[2])
+ if player != 2:
+ filtered += 1
+ continue
states.append(sf)
- results.append(float(rest[1]))
+ results.append(label)
except: pass
- if neural_only:
- log(f" filtered out {neural_only} non-Neural states")
+ if ismcts_samples:
+ log(f" {ismcts_samples} ISMCTS q-value samples")
+ if filtered:
+ log(f" filtered {filtered} unusable states")
return np.array(states, dtype=np.float32) if states else None, np.array(results, dtype=np.float32) if results else None
def train_v(prev_winrate=None):