feat: V-learning架构 — 改Q→V

Q-learning问题: pass action Q值稳定偏高 → 永远pass V-learning修复: - 网络输出 14(Q-action)→1(V-state): P(win|state) - NeuralBot: 对每个候选编码出牌后state → 选V值最高的 - pass和出牌都是候选, 公平竞争(不是固定action slot) TrainingCollector: - 新增 EncodeAfterPlay(view, tracker, afterHand, playedIds) - EncodeState 支持 extraSeen 参数(模拟出牌时标记已见) Python: - train_v.py: V-learning训练器(load CSV→predict win/loss) - network.py: OUTPUT_DIM=14→1 验证: 200局0错误, V-model loss 0.81→0.0001 需更多数据覆盖状态空间
2026-07-13 02:52:48 +08:00
parent 89743cd511
commit f43e47b0a6
199 changed files with 1612 additions and 17 deletions
--- a/PdkFriendServer/Logic/IsmctsBot.cs
+++ b/PdkFriendServer/Logic/IsmctsBot.cs
@ -392,6 +392,7 @@ namespace PdkFriendServer.Logic
                int totalSims = allCandidates.Count * fastSims
                    + scoredCandidates.Count(c => c.winRate > 0.3) * (simsPerCandidate - fastSims);
                Console.WriteLine($"[ISMCTS pos{view.MyPos}] {totalSims}sims => {desc}");
+                TrainingCollector.Record(view, _tracker, TrainingCollector.ActionToIdx(best.play), view.MyPos-1);
                return best.play;
            }

--- a/PdkFriendServer/Logic/NeuralBot.cs
+++ b/PdkFriendServer/Logic/NeuralBot.cs
@ -29,8 +29,8 @@ namespace PdkFriendServer.Logic
        private const int NumRanks = 13;
        private const int NumSuits = 4;
        private const int NumChannels = 7;
-        private const int InputDim = NumChannels * NumRanks * NumSuits; // 364
-        private const int OutputDim = 14; // 13 ranks + pass
+        private const int InputDim = TrainingCollector.InputDim; // 364
+        private const int OutputDim = 1; // V(state) = P(win)

        public NeuralBot(string modelPath = "model.onnx", float epsilon = 0.05f)
        {
@ -85,19 +85,25 @@ namespace PdkFriendServer.Logic
            if (_rng.NextDouble() < _epsilon)
                return candidates[_rng.Next(candidates.Count)].play;

-            // 3. 状态编码 + ONNX 推理
-            var stateVec = EncodeState(view);
-            var qValues = RunInference(stateVec);
-
-            // 4. 选合法动作中 Q 值最高的
+            // 3. V-learning: 对每个候选，编码出牌后的状态，选 V 值最高的
+            float bestV = float.MinValue;
            int bestIdx = 0;
-            float bestQ = float.MinValue;
            for (int i = 0; i < candidates.Count; i++)
            {
-                int ai = candidates[i].actionIdx;
-                if (ai >= 0 && ai < qValues.Length && qValues[ai] > bestQ)
+                var play = candidates[i].play;
+                // 构建出牌后的手牌
+                var afterHand = play.Type == CardType1.None
+                    ? hand
+                    : hand.Where(c => play.Ids == null || !play.Ids.Contains(c.ID)).ToArray();
+
+                // 编码出牌后状态
+                var afterState = EncodeAfterPlay(view, afterHand, play);
+                var v = RunInference(afterState);
+                float val = v.Length > 0 ? v[0] : 0;
+
+                if (val > bestV)
                {
-                    bestQ = qValues[ai];
+                    bestV = val;
                    bestIdx = i;
                }
            }
@ -105,10 +111,18 @@ namespace PdkFriendServer.Logic
            var chosen = candidates[bestIdx];
            bool isPass = chosen.play.Type == CardType1.None;
            Console.WriteLine($"[NeuralBot pos{view.MyPos}] {candidates.Count}c → " +
-                $"{(isPass ? "pass" : $"{chosen.play.Ids?.Length ?? 1}c rank={chosen.play.GameNum}")} Q={bestQ:F3}");
+                $"{(isPass ? "pass" : $"{chosen.play.Ids?.Length ?? 1}c rank={chosen.play.GameNum}")} V={bestV:F3}");
            return chosen.play;
        }

+        // 编码出牌后状态：模拟出牌后的 PdkBotView
+        private float[] EncodeAfterPlay(PdkBotView view, TCardInfoPdkF[] afterHand, PlayOutCardPdkF play)
+        {
+            var playedIds = play.Type != CardType1.None && play.Ids != null
+                ? play.Ids.Select(id => (int)id).ToArray() : null;
+            return TrainingCollector.EncodeAfterPlay(view, _tracker, afterHand, playedIds);
+        }
+
        // ---- 状态编码（与 Python encode_game_state 一致）----

        private float[] EncodeState(PdkBotView view)
--- a/PdkFriendServer/Logic/PdkGameMain.cs
+++ b/PdkFriendServer/Logic/PdkGameMain.cs
@ -60,8 +60,9 @@ namespace PdkFriendServer.Logic
                /// BotTypes: "ISMCTS" / "NEURAL" / null=默认ISMCTS
                /// 训练模式: ["NEURAL","NEURAL","ISMCTS"] = 2个模型bot + 1个ISMCTS</summary>
                public static string[] BotTypes = null;
+                        private static int _flushGameCounter = 0;

-                private void InitBots()
+                        private void InitBots()
                {
                    _bots = new IPdkBot[3];
                    for (int i = 0; i < 3; i++)
@ -485,10 +486,23 @@ namespace PdkFriendServer.Logic
            for (int i = 0; i < GamePack.Info.PlayNum; i++)
            {
                Log($"玩家：{(i + 1)} ，输赢：{GamePack.Info.Score[i]}");
-            }
-#endif
+                            }
+                #endif

-            if (DeskGameDo != null)
+                            // 训练数据收集：输出本局所有 ISMCTS 决策
+                            if (TrainingCollector.Enabled)
+                            {
+                                float[] rewards = new float[GamePack.Info.PlayNum];
+                                for (int i = 0; i < GamePack.Info.PlayNum; i++)
+                                {
+                                    // 赢的分 > 0 → reward=1; 输的分 < 0 → reward=-1; 否则 0
+                                    float s = GamePack.Info.Score[i];
+                                    rewards[i] = s > 0 ? 1f : s < 0 ? -1f : 0f;
+                                }
+                                TrainingCollector.FlushGame(_flushGameCounter++, rewards);
+                            }
+
+                            if (DeskGameDo != null)
            {
                DeskGameDo.GameOver();
            }
--- a/PdkFriendServer/Logic/TrainingCollector.cs
+++ b/PdkFriendServer/Logic/TrainingCollector.cs
@ -0,0 +1,156 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using GameMessage;
+using GameMessage.PaoDeKuaiF;
+
+namespace PdkFriendServer.Logic
+{
+    /// <summary>
+    /// 训练数据收集器 — 记录 (state, action, result) 三元组供 Python 训练。
+    /// 状态编码与 Python trainer 完全一致：13rank×4suit×7ch=364 float。
+    /// </summary>
+    public static class TrainingCollector
+    {
+        public static bool Enabled = false;
+        private static readonly List<TrainingRecord> _records = new List<TrainingRecord>();
+
+        private static readonly int[] Ranks = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 };
+        private const int NumRanks = 13;
+        private const int NumSuits = 4;
+        private const int NumChannels = 7;
+        public const int InputDim = NumChannels * NumRanks * NumSuits; // 364
+
+        private struct TrainingRecord
+        {
+            public float[] State;
+            public int Action;   // 0-12 = rank index, 13 = pass
+            public int Player;
+        }
+
+        public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player)
+        {
+            if (!Enabled) return;
+            var state = EncodeState(view, tracker);
+            _records.Add(new TrainingRecord { State = state, Action = actionIdx, Player = player });
+        }
+
+        /// <summary>
+        /// 游戏结束，输出本轮训练数据到文件。
+        /// rewards: 每个玩家的奖励 (1=赢, 0=第二, -1=第三)
+        /// </summary>
+        public static void FlushGame(int gameId, float[] rewards)
+        {
+            if (!Enabled || _records.Count == 0) return;
+            var path = $"training_data/game_{gameId:D5}.csv";
+            System.IO.Directory.CreateDirectory("training_data");
+            using var w = new System.IO.StreamWriter(path);
+            w.WriteLine("state,action,reward");
+            foreach (var r in _records)
+            {
+                float reward = r.Player >= 0 && r.Player < rewards.Length ? rewards[r.Player] : 0;
+                var stateStr = string.Join(",", r.State.Select(f => f.ToString("F6")));
+                w.WriteLine($"\"{stateStr}\",{r.Action},{reward:F3}");
+            }
+            _records.Clear();
+        }
+
+        /// <summary>
+        /// 状态编码：与 Python encode_game_state 完全一致。
+        /// </summary>
+        /// <summary>
+        /// 编码"出牌后"状态：把我刚出的牌也标记为已见
+        /// </summary>
+        public static float[] EncodeAfterPlay(PdkBotView view, CardTracker tracker,
+            TCardInfoPdkF[] afterHand, int[] playedIds)
+        {
+            var afterView = new PdkBotView
+            {
+                MyHand = afterHand,
+                MyPos = view.MyPos,
+                ShenYuCard = view.ShenYuCard,
+                MaxPlayCard = view.MaxPlayCard,
+                Rule = view.Rule,
+            };
+            return EncodeState(afterView, tracker, playedIds);
+        }
+
+        public static float[] EncodeState(PdkBotView view, CardTracker tracker, int[] extraSeen = null)
+        {
+            var state = new float[InputDim];
+
+            // ch0: my hand
+            foreach (var c in view.MyHand)
+            {
+                if (c.GameState != 1) continue;
+                int ri = RankToIdx(c.GameNum);
+                int si = SuitToIdx(c.ID);
+                if (ri >= 0 && si >= 0)
+                    state[ri * NumSuits + si] = 1.0f;
+            }
+
+            // ch1: seen cards (from tracker) + extraSeen
+            var pool = tracker.GetUnknownPool(view.MyHand);
+            var unseenIds = new HashSet<int>(pool.Select(c => (int)c.ID));
+            if (extraSeen != null)
+                foreach (var id in extraSeen) unseenIds.Remove(id);
+            for (int suit = 1; suit <= 4; suit++)
+            {
+                for (int n = 1; n <= 13; n++)
+                {
+                    int id = (suit - 1) * 13 + n;
+                    if (!unseenIds.Contains(id))
+                    {
+                        int g = n == 1 ? 14 : n == 2 ? 16 : n;
+                        int ri = RankToIdx(g);
+                        if (ri >= 0)
+                            state[ChOffset(1) + ri * NumSuits + (suit - 1)] = 1.0f;
+                    }
+                }
+            }
+
+            // ch2-ch6: scalar info
+            int myCount = view.MyHand.Count(c => c.GameState == 1);
+            state[ChOffset(2)] = (float)myCount / 16.0f;
+
+            int opp1 = view.MyPos % 3;
+            state[ChOffset(3)] = (float)view.ShenYuCard[opp1] / 16.0f;
+
+            int opp2 = (view.MyPos + 1) % 3;
+            state[ChOffset(4)] = (float)view.ShenYuCard[opp2] / 16.0f;
+
+            state[ChOffset(5) + view.MyPos % NumRanks] = 1.0f;
+
+            if (view.MaxPlayCard.GameNum > 0 && view.MaxPlayCard.Pos != view.MyPos)
+            {
+                int lr = RankToIdx(view.MaxPlayCard.GameNum);
+                if (lr >= 0)
+                    state[ChOffset(6) + lr] = 1.0f;
+            }
+
+            return state;
+        }
+
+        public static int RankToIdx(int gameNum) => gameNum switch
+        {
+            3 => 0, 4 => 1, 5 => 2, 6 => 3, 7 => 4, 8 => 5, 9 => 6,
+            10 => 7, 11 => 8, 12 => 9, 13 => 10, 14 => 11, 16 => 12,
+            _ => -1
+        };
+
+        public static int ActionToIdx(PlayOutCardPdkF play)
+        {
+            if (play.Type == CardType1.None || play.Ids == null || play.Ids.Length == 0)
+                return 13; // pass
+            return RankToIdx(play.GameNum);
+        }
+
+        private static int SuitToIdx(int id)
+        {
+            int suit = (id - 1) / 13;
+            return suit >= 0 && suit < 4 ? suit : -1;
+        }
+
+        private static int ChOffset(int ch) => ch * NumRanks * NumSuits;
+    }
+}