feat: V-learning架构 — 改Q→V
Q-learning问题: pass action Q值稳定偏高 → 永远pass V-learning修复: - 网络输出 14(Q-action)→1(V-state): P(win|state) - NeuralBot: 对每个候选编码出牌后state → 选V值最高的 - pass和出牌都是候选, 公平竞争(不是固定action slot) TrainingCollector: - 新增 EncodeAfterPlay(view, tracker, afterHand, playedIds) - EncodeState 支持 extraSeen 参数(模拟出牌时标记已见) Python: - train_v.py: V-learning训练器(load CSV→predict win/loss) - network.py: OUTPUT_DIM=14→1 验证: 200局0错误, V-model loss 0.81→0.0001 需更多数据覆盖状态空间
This commit is contained in:
@ -60,8 +60,9 @@ namespace PdkFriendServer.Logic
|
||||
/// BotTypes: "ISMCTS" / "NEURAL" / null=默认ISMCTS
|
||||
/// 训练模式: ["NEURAL","NEURAL","ISMCTS"] = 2个模型bot + 1个ISMCTS</summary>
|
||||
public static string[] BotTypes = null;
|
||||
private static int _flushGameCounter = 0;
|
||||
|
||||
private void InitBots()
|
||||
private void InitBots()
|
||||
{
|
||||
_bots = new IPdkBot[3];
|
||||
for (int i = 0; i < 3; i++)
|
||||
@ -485,10 +486,23 @@ namespace PdkFriendServer.Logic
|
||||
for (int i = 0; i < GamePack.Info.PlayNum; i++)
|
||||
{
|
||||
Log($"玩家:{(i + 1)} ,输赢:{GamePack.Info.Score[i]}");
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
if (DeskGameDo != null)
|
||||
// 训练数据收集:输出本局所有 ISMCTS 决策
|
||||
if (TrainingCollector.Enabled)
|
||||
{
|
||||
float[] rewards = new float[GamePack.Info.PlayNum];
|
||||
for (int i = 0; i < GamePack.Info.PlayNum; i++)
|
||||
{
|
||||
// 赢的分 > 0 → reward=1; 输的分 < 0 → reward=-1; 否则 0
|
||||
float s = GamePack.Info.Score[i];
|
||||
rewards[i] = s > 0 ? 1f : s < 0 ? -1f : 0f;
|
||||
}
|
||||
TrainingCollector.FlushGame(_flushGameCounter++, rewards);
|
||||
}
|
||||
|
||||
if (DeskGameDo != null)
|
||||
{
|
||||
DeskGameDo.GameOver();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user