xiaoou
2b108c4d41
B: NeuralBot也调用TrainingCollector.Record → 3个bot全部记录(之前只有ISMCTS) → 9样本/局(之前6, +50%) A: parallel_train.py 并行调度器 python3 scripts/parallel_train.py --workers 3 --games 333 → 串行3个worker各跑333局 → 合并数据到training_data/ 效率: 1000局 ≈ 12分钟, 9000样本 之前: 1000局 ≈ 12分钟, 6000样本
293 lines
12 KiB
C#
293 lines
12 KiB
C#
using System;
|
||
using System.Collections.Generic;
|
||
using System.Linq;
|
||
using GameFix.PaoDeKuaiF;
|
||
using GameFix.Poker;
|
||
using GameMessage;
|
||
using GameMessage.PaoDeKuaiF;
|
||
using Microsoft.ML.OnnxRuntime;
|
||
using Microsoft.ML.OnnxRuntime.Tensors;
|
||
|
||
namespace PdkFriendServer.Logic
|
||
{
|
||
/// <summary>
|
||
/// NeuralBot v1 — 加载 ONNX 模型做决策的 Bot。
|
||
/// 状态编码与 Python trainer 完全一致：15 rank × 4 suit × 7 channel → 420 float。
|
||
///
|
||
/// 用法：PdkGameMain.InitBots() 中替换部分 bot 为 NeuralBot。
|
||
/// 训练：Python trainer → 导出 ONNX → 放 model.onnx → C# 加载。
|
||
/// </summary>
|
||
public class NeuralBot : IPdkBot
|
||
{
|
||
private readonly InferenceSession _session;
|
||
private readonly Random _rng;
|
||
private readonly CardTracker _tracker = new CardTracker();
|
||
private readonly float _epsilon; // 探索率
|
||
|
||
// 状态编码常量
|
||
private static readonly int[] Ranks = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 };
|
||
private const int NumRanks = 13;
|
||
private const int NumSuits = 4;
|
||
private const int NumChannels = 7;
|
||
private const int InputDim = TrainingCollector.InputDim; // 364
|
||
private const int OutputDim = 1; // V(state) = P(win)
|
||
|
||
public NeuralBot(string modelPath = "model.onnx", float epsilon = 0.05f)
|
||
{
|
||
var options = new SessionOptions();
|
||
options.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR;
|
||
_session = new InferenceSession(modelPath, options);
|
||
_rng = new Random();
|
||
_epsilon = epsilon;
|
||
}
|
||
|
||
public bool DecideBaoZhuang(PdkBotView view)
|
||
{
|
||
// Neural model 暂不参与包庄决策，沿用 ISMCTS 逻辑
|
||
_tracker.Update(view);
|
||
int minPlays = HandOptimizer.MinPlays(view.MyHand);
|
||
if (minPlays >= 7) return false;
|
||
return minPlays <= 3; // 极简：手牌极好才包
|
||
}
|
||
|
||
public PlayOutCardPdkF DecidePlay(PdkBotView view)
|
||
{
|
||
_tracker.Update(view);
|
||
var hand = view.MyHand;
|
||
if (hand.Length == 0) return Pass(view);
|
||
|
||
// 1. 获取合法出牌
|
||
bool isFirstPlay = view.MaxPlayCard.GameNum <= 0;
|
||
var tips = PdkCardAlgorithm.GetTipCard(view.MaxPlayCard, hand, null, view.Rule.AAAIsZhaDan);
|
||
var candidates = new List<(PlayOutCardPdkF play, int actionIdx)>();
|
||
|
||
if (tips != null)
|
||
{
|
||
foreach (var tip in tips)
|
||
{
|
||
int rank = tip[0].GameNum;
|
||
int idx = RankToIdx(rank);
|
||
candidates.Add((MakePlay(tip, view.MyPos), idx));
|
||
}
|
||
}
|
||
|
||
// 新轮次补多牌方案
|
||
if (isFirstPlay)
|
||
AddMultiCardCandidates(hand, view.MyPos, candidates);
|
||
|
||
if (!isFirstPlay)
|
||
candidates.Add((Pass(view), OutputDim - 1)); // pass = last index
|
||
|
||
if (candidates.Count == 0) return Pass(view);
|
||
if (candidates.Count == 1) return candidates[0].play;
|
||
|
||
// 2. ε-greedy 探索
|
||
if (_rng.NextDouble() < _epsilon)
|
||
return candidates[_rng.Next(candidates.Count)].play;
|
||
|
||
// 3. V-learning: 对每个候选，编码出牌后的状态，选 V 值最高的
|
||
float bestV = float.MinValue;
|
||
int bestIdx = 0;
|
||
var debugVals = new System.Text.StringBuilder();
|
||
for (int i = 0; i < Math.Min(candidates.Count, 8); i++)
|
||
{
|
||
var play = candidates[i].play;
|
||
var afterHand = play.Ids != null && play.Ids.Length > 0
|
||
? hand.Where(c => !play.Ids.Contains(c.ID)).ToArray()
|
||
: hand;
|
||
var afterState = EncodeAfterPlay(view, afterHand, play);
|
||
var v = RunInference(afterState);
|
||
float val = v.Length > 0 ? v[0] : 0;
|
||
|
||
string label = play.Ids != null && play.Ids.Length > 0
|
||
? $"{play.Ids.Length}c r{play.GameNum}" : "pass";
|
||
debugVals.Append($" {label}={val:F2}");
|
||
|
||
if (val > bestV) { bestV = val; bestIdx = i; }
|
||
}
|
||
// 继续评估剩余候选（不输出日志）
|
||
for (int i = 8; i < candidates.Count; i++)
|
||
{
|
||
var play = candidates[i].play;
|
||
var afterHand = play.Ids != null && play.Ids.Length > 0
|
||
? hand.Where(c => !play.Ids.Contains(c.ID)).ToArray()
|
||
: hand;
|
||
|
||
// 编码出牌后状态
|
||
var afterState = EncodeAfterPlay(view, afterHand, play);
|
||
var v = RunInference(afterState);
|
||
float val = v.Length > 0 ? v[0] : 0;
|
||
|
||
if (val > bestV)
|
||
{
|
||
bestV = val;
|
||
bestIdx = i;
|
||
}
|
||
}
|
||
|
||
var chosen = candidates[bestIdx];
|
||
bool isPass = chosen.play.Ids == null || chosen.play.Ids.Length == 0;
|
||
Console.WriteLine($"[NeuralBot pos{view.MyPos}] {candidates.Count}c → "
|
||
+ $"{(isPass ? "pass" : $"{chosen.play.Ids?.Length ?? 1}c r{chosen.play.GameNum}")} V={bestV:F3}"
|
||
+ (debugVals.Length > 0 ? $" [{debugVals}]" : ""));
|
||
TrainingCollector.Record(view, _tracker,
|
||
isPass ? 13 : TrainingCollector.RankToIdx(chosen.play.GameNum), view.MyPos - 1);
|
||
return chosen.play;
|
||
}
|
||
|
||
// 编码出牌后状态：模拟出牌后的 PdkBotView
|
||
private float[] EncodeAfterPlay(PdkBotView view, TCardInfoPdkF[] afterHand, PlayOutCardPdkF play)
|
||
{
|
||
var playedIds = play.Type != CardType1.None && play.Ids != null
|
||
? play.Ids.Select(id => (int)id).ToArray() : null;
|
||
return TrainingCollector.EncodeAfterPlay(view, _tracker, afterHand, playedIds);
|
||
}
|
||
|
||
// ---- 状态编码（与 Python encode_game_state 一致）----
|
||
|
||
private float[] EncodeState(PdkBotView view)
|
||
{
|
||
var state = new float[InputDim]; // 7 × 13 × 4 = 364
|
||
|
||
// ch0: 我的手牌
|
||
foreach (var c in view.MyHand)
|
||
{
|
||
if (c.GameState != 1) continue;
|
||
int ri = RankToIdx(c.GameNum);
|
||
int si = SuitToIdx(c.ID);
|
||
if (ri >= 0 && si >= 0)
|
||
state[ChOffset(0) + ri * NumSuits + si] = 1.0f;
|
||
}
|
||
|
||
// ch1: 已出牌 — 通过 card tracker 的 seen cards
|
||
var seenIds = GetSeenCardIds(view);
|
||
foreach (var id in seenIds)
|
||
{
|
||
int ri = RankToIdx(IdToGameNum(id));
|
||
int si = SuitToIdx(id);
|
||
if (ri >= 0 && si >= 0)
|
||
state[ChOffset(1) + ri * NumSuits + si] = 1.0f;
|
||
}
|
||
|
||
// ch2: 我的手牌数量
|
||
int myCount = view.MyHand.Count(c => c.GameState == 1);
|
||
int c2Slot = myCount % NumRanks;
|
||
state[ChOffset(2) + c2Slot * NumSuits + 0] = (float)myCount / 16.0f;
|
||
|
||
// ch3: 下家剩余
|
||
int opp1 = view.MyPos % 3;
|
||
int c1 = view.ShenYuCard[opp1];
|
||
state[ChOffset(3) + (c1 % NumRanks) * NumSuits + 0] = (float)c1 / 16.0f;
|
||
|
||
// ch4: 上家剩余
|
||
int opp2 = (view.MyPos + 1) % 3;
|
||
int c2 = view.ShenYuCard[opp2];
|
||
state[ChOffset(4) + (c2 % NumRanks) * NumSuits + 0] = (float)c2 / 16.0f;
|
||
|
||
// ch5: 当前位置
|
||
state[ChOffset(5) + (view.MyPos % NumRanks) * NumSuits + 0] = 1.0f;
|
||
|
||
// ch6: last_play rank
|
||
if (view.MaxPlayCard.GameNum > 0 && view.MaxPlayCard.Pos != view.MyPos)
|
||
{
|
||
int lr = RankToIdx(view.MaxPlayCard.GameNum);
|
||
if (lr >= 0)
|
||
state[ChOffset(6) + lr * NumSuits + 0] = 1.0f;
|
||
}
|
||
|
||
return state;
|
||
}
|
||
|
||
private float[] RunInference(float[] input)
|
||
{
|
||
var tensor = new DenseTensor<float>(input, new[] { 1, InputDim });
|
||
var inputs = new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor("state", tensor) };
|
||
using var results = _session.Run(inputs);
|
||
var output = results.First().AsTensor<float>();
|
||
return output.ToArray();
|
||
}
|
||
|
||
// ---- 辅助 ----
|
||
|
||
private static int RankToIdx(int gameNum) => gameNum switch
|
||
{
|
||
3 => 0, 4 => 1, 5 => 2, 6 => 3, 7 => 4, 8 => 5, 9 => 6,
|
||
10 => 7, 11 => 8, 12 => 9, 13 => 10, 14 => 11, 16 => 12,
|
||
_ => -1
|
||
};
|
||
|
||
private static int SuitToIdx(int id)
|
||
{
|
||
// ID = (suit-1)*13 + n, suit 1-4
|
||
int suit = (id - 1) / 13; // 0-3
|
||
return suit >= 0 && suit < 4 ? suit : -1;
|
||
}
|
||
|
||
private static int IdToGameNum(int id)
|
||
{
|
||
int n = (id - 1) % 13 + 1; // 1-13
|
||
return n == 1 ? 14 : n == 2 ? 16 : n;
|
||
}
|
||
|
||
private HashSet<int> GetSeenCardIds(PdkBotView view)
|
||
{
|
||
var seen = new HashSet<int>();
|
||
// Track played cards through view: all hands that have appeared on table
|
||
if (view.MaxPlayCard.Ids != null)
|
||
foreach (var id in view.MaxPlayCard.Ids) seen.Add(id);
|
||
// Also track from our own played cards (cards not in my hand anymore)
|
||
return seen;
|
||
}
|
||
|
||
private static int ChOffset(int ch) => ch * NumRanks * NumSuits;
|
||
|
||
// ---- 多牌候选（精简版，复用 IsmctsBot 逻辑）----
|
||
|
||
private void AddMultiCardCandidates(TCardInfoPdkF[] hand, int pos,
|
||
List<(PlayOutCardPdkF play, int actionIdx)> candidates)
|
||
{
|
||
// 对子
|
||
foreach (var g in hand.Where(c => c.GameState == 1).GroupBy(c => c.GameNum).Where(g => g.Count() >= 2))
|
||
{
|
||
var pair = g.Take(2).ToList();
|
||
candidates.Add((MakePlay(pair, pos), RankToIdx(pair[0].GameNum)));
|
||
}
|
||
// 三张
|
||
foreach (var g in hand.Where(c => c.GameState == 1).GroupBy(c => c.GameNum).Where(g => g.Count() >= 3))
|
||
{
|
||
var trip = g.Take(3).ToList();
|
||
var rem = hand.Where(c => c.GameState == 1 && c.GameNum != g.Key).ToList();
|
||
if (rem.Count >= 2)
|
||
{
|
||
// 优先带对子
|
||
var pair = rem.GroupBy(c => c.GameNum).Where(x => x.Count() >= 2).OrderBy(x => x.Key).FirstOrDefault();
|
||
var attach = pair != null ? pair.Take(2).ToList() : rem.OrderBy(c => c.GameNum).Take(2).ToList();
|
||
candidates.Add((MakePlay(trip.Concat(attach).ToList(), pos), RankToIdx(trip[0].GameNum)));
|
||
}
|
||
}
|
||
// 顺子
|
||
var straights = PokerLogic.GetShunZi<TCardInfoPdkF>(hand);
|
||
if (straights != null)
|
||
foreach (var s in straights)
|
||
{
|
||
var cards = s.ToList();
|
||
candidates.Add((MakePlay(cards, pos), RankToIdx(cards[0].GameNum)));
|
||
}
|
||
// 炸弹
|
||
if (PokerLogic.GetPlayZhaDans<TCardInfoPdkF>(hand, out var bombs) && bombs != null)
|
||
foreach (var b in bombs)
|
||
{
|
||
var cards = b.ToList();
|
||
candidates.Add((MakePlay(cards, pos), RankToIdx(cards[0].GameNum)));
|
||
}
|
||
}
|
||
|
||
private static PlayOutCardPdkF MakePlay(List<TCardInfoPdkF> cards, int pos)
|
||
=> new PlayOutCardPdkF { Pos = (byte)pos, GameNum = cards[0].GameNum, Ids = cards.Select(c => c.ID).ToArray(), Type = CardType1.None };
|
||
|
||
private static PlayOutCardPdkF Pass(PdkBotView v)
|
||
=> new PlayOutCardPdkF { Pos = (byte)v.MyPos, Type = CardType1.None };
|
||
|
||
public void Dispose() => _session?.Dispose();
|
||
}
|
||
} |