hjha-server/PdkFriendServer/Logic/TrainingCollector.cs

using System;
using System.Collections.Generic;
using System.Linq;
using GameMessage;
using GameMessage.PaoDeKuaiF;

namespace PdkFriendServer.Logic
{
    /// <summary>
    /// 训练数据收集器 — 记录 (state, action, q_value, player) 四元组供 Python 训练。
    /// q_value: ISMCTS 对选出动作的胜率评估 (0~1), NeuralBot 填游戏结果(-1/0/1)
    /// </summary>
    public static class TrainingCollector
    {
        public static bool Enabled = false;
        private static readonly List<TrainingRecord> _records = new List<TrainingRecord>();
        private static readonly string _sessionPrefix = DateTime.Now.ToString("yyyyMMdd_HHmm");

        private static readonly int[] Ranks = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 };
        private const int NumRanks = 13;
        private const int NumSuits = 4;
        private const int NumChannels = 7;
        public const int InputDim = NumChannels * NumRanks * NumSuits; // 364

        private struct TrainingRecord
        {
            public float[] State;
            public int Action;   // 0-12 = rank index, 13 = pass
            public float QValue; // ISMCTS 评估值 (0~1) 或 NeuralBot 用游戏结果
            public int Player;
        }

        public static void Record(PdkBotView view, CardTracker tracker, int actionIdx, int player, float qValue = 0f)
        {
            if (!Enabled) return;
            var state = EncodeState(view, tracker);
            _records.Add(new TrainingRecord { State = state, Action = actionIdx, QValue = qValue, Player = player });
        }

        /// <summary>
        /// 游戏结束，输出本轮训练数据到文件。
        /// rewards: 每个玩家的奖励 (1=赢, 0=第二, -1=第三)
        /// </summary>
        public static void FlushGame(int gameId, float[] rewards, int winnerPos, string[] botTypes)
        {
            if (!Enabled || _records.Count == 0) return;
            var path = $"training_data/game_{_sessionPrefix}_{gameId:D6}.csv";
            System.IO.Directory.CreateDirectory("training_data");
            using var w = new System.IO.StreamWriter(path);
            w.WriteLine("state,action,q_value,player");
            foreach (var r in _records)
            {
                var stateStr = string.Join(",", r.State.Select(f => f.ToString("F6")));
                w.WriteLine($"\"{stateStr}\",{r.Action},{r.QValue:F3},{r.Player}");
            }
            // 游戏结果摘要 — 方便不中断训练即可评估模型进化
            w.WriteLine($"#RESULT,winner={winnerPos},bots={string.Join(",", botTypes ?? new[]{"ISMCTS","ISMCTS","ISMCTS"})}");
            _records.Clear();
        }

        /// <summary>
        /// 状态编码：与 Python encode_game_state 完全一致。
        /// </summary>
        /// <summary>
        /// 编码"出牌后"状态：把我刚出的牌也标记为已见
        /// </summary>
        public static float[] EncodeAfterPlay(PdkBotView view, CardTracker tracker,
            TCardInfoPdkF[] afterHand, int[] playedIds)
        {
            // 更新剩余张数：我打出了牌
            int playedCount = playedIds?.Length ?? 0;
            byte[] afterShenYu = new byte[view.ShenYuCard.Length];
            Array.Copy(view.ShenYuCard, afterShenYu, view.ShenYuCard.Length);
            afterShenYu[view.MyPos - 1] = (byte)Math.Max(0, afterShenYu[view.MyPos - 1] - playedCount);

            var afterView = new PdkBotView
            {
                MyHand = afterHand,
                MyPos = view.MyPos,
                ShenYuCard = afterShenYu,
                // 出牌后：如果我不是pass，MaxPlayCard改成我出的牌
                MaxPlayCard = playedIds != null && playedIds.Length > 0
                    ? new PlayOutCardPdkF { Pos = (byte)view.MyPos,
                        GameNum = view.MyHand.FirstOrDefault(c => playedIds.Contains(c.ID)).GameNum,
                        Type = CardType1.DanZhang, Ids = playedIds.Select(id => (byte)id).ToArray() }
                    : view.MaxPlayCard,
                Rule = view.Rule,
            };
            return EncodeState(afterView, tracker, playedIds);
        }

        public static float[] EncodeState(PdkBotView view, CardTracker tracker, int[] extraSeen = null)
        {
            var state = new float[InputDim];

            // ch0: my hand
            foreach (var c in view.MyHand)
            {
                if (c.GameState != 1) continue;
                int ri = RankToIdx(c.GameNum);
                int si = SuitToIdx(c.ID);
                if (ri >= 0 && si >= 0)
                    state[ri * NumSuits + si] = 1.0f;
            }

            // ch1: seen cards (from tracker) + extraSeen
            var pool = tracker.GetUnknownPool(view.MyHand);
            var unseenIds = new HashSet<int>(pool.Select(c => (int)c.ID));
            if (extraSeen != null)
                foreach (var id in extraSeen) unseenIds.Remove(id);
            for (int suit = 1; suit <= 4; suit++)
            {
                for (int n = 1; n <= 13; n++)
                {
                    int id = (suit - 1) * 13 + n;
                    if (!unseenIds.Contains(id))
                    {
                        int g = n == 1 ? 14 : n == 2 ? 16 : n;
                        int ri = RankToIdx(g);
                        if (ri >= 0)
                            state[ChOffset(1) + ri * NumSuits + (suit - 1)] = 1.0f;
                    }
                }
            }

            // ch2-ch6: scalar info
            int myCount = view.MyHand.Count(c => c.GameState == 1);
            state[ChOffset(2)] = (float)myCount / 16.0f;

            int opp1 = view.MyPos % 3;
            state[ChOffset(3)] = (float)view.ShenYuCard[opp1] / 16.0f;

            int opp2 = (view.MyPos + 1) % 3;
            state[ChOffset(4)] = (float)view.ShenYuCard[opp2] / 16.0f;

            state[ChOffset(5) + view.MyPos % NumRanks] = 1.0f;

            if (view.MaxPlayCard.GameNum > 0 && view.MaxPlayCard.Pos != view.MyPos)
            {
                int lr = RankToIdx(view.MaxPlayCard.GameNum);
                if (lr >= 0)
                    state[ChOffset(6) + lr] = 1.0f;
            }

            return state;
        }

        public static int RankToIdx(int gameNum) => gameNum switch
        {
            3 => 0, 4 => 1, 5 => 2, 6 => 3, 7 => 4, 8 => 5, 9 => 6,
            10 => 7, 11 => 8, 12 => 9, 13 => 10, 14 => 11, 16 => 12,
            _ => -1
        };

        public static int ActionToIdx(PlayOutCardPdkF play)
        {
            if (play.Type == CardType1.None || play.Ids == null || play.Ids.Length == 0)
                return 13; // pass
            return RankToIdx(play.GameNum);
        }

        private static int SuitToIdx(int id)
        {
            int suit = (id - 1) / 13;
            return suit >= 0 && suit < 4 ? suit : -1;
        }

        private static int ChOffset(int ch) => ch * NumRanks * NumSuits;
    }
}