feat: NeuralBot — ONNX模型驱动的Bot + 混合Bot训练架构
NeuralBot.cs: - 加载ONNX模型，状态编码与Python trainer完全一致(13rank×4suit×7ch=364) - ε-greedy探索(10%) + ONNX推理选Q值最高动作 - 多牌候选: 对子/三带二(优先带对子)/顺子/炸弹 PdkGameMain: - BotTypes静态配置: ["NEURAL","NEURAL","ISMCTS"] = 2模型+1ISMCTS - --mix参数: dotnet run -- --mix 100 hjha-console: - --mix参数自动设置BotTypes并转调--stress 验证: 3局0错误, 1.3s/局, NeuralBot出牌正常
This commit is contained in:
259
PdkFriendServer/Logic/NeuralBot.cs
Normal file
259
PdkFriendServer/Logic/NeuralBot.cs
Normal file
@ -0,0 +1,259 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using GameFix.PaoDeKuaiF;
|
||||
using GameFix.Poker;
|
||||
using GameMessage;
|
||||
using GameMessage.PaoDeKuaiF;
|
||||
using Microsoft.ML.OnnxRuntime;
|
||||
using Microsoft.ML.OnnxRuntime.Tensors;
|
||||
|
||||
namespace PdkFriendServer.Logic
|
||||
{
|
||||
/// <summary>
|
||||
/// NeuralBot v1 — 加载 ONNX 模型做决策的 Bot。
|
||||
/// 状态编码与 Python trainer 完全一致：15 rank × 4 suit × 7 channel → 420 float。
|
||||
///
|
||||
/// 用法：PdkGameMain.InitBots() 中替换部分 bot 为 NeuralBot。
|
||||
/// 训练：Python trainer → 导出 ONNX → 放 model.onnx → C# 加载。
|
||||
/// </summary>
|
||||
public class NeuralBot : IPdkBot
|
||||
{
|
||||
private readonly InferenceSession _session;
|
||||
private readonly Random _rng;
|
||||
private readonly CardTracker _tracker = new CardTracker();
|
||||
private readonly float _epsilon; // 探索率
|
||||
|
||||
// 状态编码常量
|
||||
private static readonly int[] Ranks = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16 };
|
||||
private const int NumRanks = 13;
|
||||
private const int NumSuits = 4;
|
||||
private const int NumChannels = 7;
|
||||
private const int InputDim = NumChannels * NumRanks * NumSuits; // 364
|
||||
private const int OutputDim = 14; // 13 ranks + pass
|
||||
|
||||
public NeuralBot(string modelPath = "model.onnx", float epsilon = 0.05f)
|
||||
{
|
||||
var options = new SessionOptions();
|
||||
options.LogSeverityLevel = OrtLoggingLevel.ORT_LOGGING_LEVEL_ERROR;
|
||||
_session = new InferenceSession(modelPath, options);
|
||||
_rng = new Random();
|
||||
_epsilon = epsilon;
|
||||
}
|
||||
|
||||
public bool DecideBaoZhuang(PdkBotView view)
|
||||
{
|
||||
// Neural model 暂不参与包庄决策，沿用 ISMCTS 逻辑
|
||||
_tracker.Update(view);
|
||||
int minPlays = HandOptimizer.MinPlays(view.MyHand);
|
||||
if (minPlays >= 7) return false;
|
||||
return minPlays <= 3; // 极简：手牌极好才包
|
||||
}
|
||||
|
||||
public PlayOutCardPdkF DecidePlay(PdkBotView view)
|
||||
{
|
||||
_tracker.Update(view);
|
||||
var hand = view.MyHand;
|
||||
if (hand.Length == 0) return Pass(view);
|
||||
|
||||
// 1. 获取合法出牌
|
||||
bool isFirstPlay = view.MaxPlayCard.GameNum <= 0;
|
||||
var tips = PdkCardAlgorithm.GetTipCard(view.MaxPlayCard, hand, null, view.Rule.AAAIsZhaDan);
|
||||
var candidates = new List<(PlayOutCardPdkF play, int actionIdx)>();
|
||||
|
||||
if (tips != null)
|
||||
{
|
||||
foreach (var tip in tips)
|
||||
{
|
||||
int rank = tip[0].GameNum;
|
||||
int idx = RankToIdx(rank);
|
||||
candidates.Add((MakePlay(tip, view.MyPos), idx));
|
||||
}
|
||||
}
|
||||
|
||||
// 新轮次补多牌方案
|
||||
if (isFirstPlay)
|
||||
AddMultiCardCandidates(hand, view.MyPos, candidates);
|
||||
|
||||
if (!isFirstPlay)
|
||||
candidates.Add((Pass(view), OutputDim - 1)); // pass = last index
|
||||
|
||||
if (candidates.Count == 0) return Pass(view);
|
||||
if (candidates.Count == 1) return candidates[0].play;
|
||||
|
||||
// 2. ε-greedy 探索
|
||||
if (_rng.NextDouble() < _epsilon)
|
||||
return candidates[_rng.Next(candidates.Count)].play;
|
||||
|
||||
// 3. 状态编码 + ONNX 推理
|
||||
var stateVec = EncodeState(view);
|
||||
var qValues = RunInference(stateVec);
|
||||
|
||||
// 4. 选合法动作中 Q 值最高的
|
||||
int bestIdx = 0;
|
||||
float bestQ = float.MinValue;
|
||||
for (int i = 0; i < candidates.Count; i++)
|
||||
{
|
||||
int ai = candidates[i].actionIdx;
|
||||
if (ai >= 0 && ai < qValues.Length && qValues[ai] > bestQ)
|
||||
{
|
||||
bestQ = qValues[ai];
|
||||
bestIdx = i;
|
||||
}
|
||||
}
|
||||
|
||||
var chosen = candidates[bestIdx];
|
||||
bool isPass = chosen.play.Type == CardType1.None;
|
||||
Console.WriteLine($"[NeuralBot pos{view.MyPos}] {candidates.Count}c → " +
|
||||
$"{(isPass ? "pass" : $"{chosen.play.Ids?.Length ?? 1}c rank={chosen.play.GameNum}")} Q={bestQ:F3}");
|
||||
return chosen.play;
|
||||
}
|
||||
|
||||
// ---- 状态编码（与 Python encode_game_state 一致）----
|
||||
|
||||
private float[] EncodeState(PdkBotView view)
|
||||
{
|
||||
var state = new float[InputDim]; // 7 × 13 × 4 = 364
|
||||
|
||||
// ch0: 我的手牌
|
||||
foreach (var c in view.MyHand)
|
||||
{
|
||||
if (c.GameState != 1) continue;
|
||||
int ri = RankToIdx(c.GameNum);
|
||||
int si = SuitToIdx(c.ID);
|
||||
if (ri >= 0 && si >= 0)
|
||||
state[ChOffset(0) + ri * NumSuits + si] = 1.0f;
|
||||
}
|
||||
|
||||
// ch1: 已出牌 — 通过 card tracker 的 seen cards
|
||||
var seenIds = GetSeenCardIds(view);
|
||||
foreach (var id in seenIds)
|
||||
{
|
||||
int ri = RankToIdx(IdToGameNum(id));
|
||||
int si = SuitToIdx(id);
|
||||
if (ri >= 0 && si >= 0)
|
||||
state[ChOffset(1) + ri * NumSuits + si] = 1.0f;
|
||||
}
|
||||
|
||||
// ch2: 我的手牌数量
|
||||
int myCount = view.MyHand.Count(c => c.GameState == 1);
|
||||
int c2Slot = myCount % NumRanks;
|
||||
state[ChOffset(2) + c2Slot * NumSuits + 0] = (float)myCount / 16.0f;
|
||||
|
||||
// ch3: 下家剩余
|
||||
int opp1 = view.MyPos % 3;
|
||||
int c1 = view.ShenYuCard[opp1];
|
||||
state[ChOffset(3) + (c1 % NumRanks) * NumSuits + 0] = (float)c1 / 16.0f;
|
||||
|
||||
// ch4: 上家剩余
|
||||
int opp2 = (view.MyPos + 1) % 3;
|
||||
int c2 = view.ShenYuCard[opp2];
|
||||
state[ChOffset(4) + (c2 % NumRanks) * NumSuits + 0] = (float)c2 / 16.0f;
|
||||
|
||||
// ch5: 当前位置
|
||||
state[ChOffset(5) + (view.MyPos % NumRanks) * NumSuits + 0] = 1.0f;
|
||||
|
||||
// ch6: last_play rank
|
||||
if (view.MaxPlayCard.GameNum > 0 && view.MaxPlayCard.Pos != view.MyPos)
|
||||
{
|
||||
int lr = RankToIdx(view.MaxPlayCard.GameNum);
|
||||
if (lr >= 0)
|
||||
state[ChOffset(6) + lr * NumSuits + 0] = 1.0f;
|
||||
}
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
private float[] RunInference(float[] input)
|
||||
{
|
||||
var tensor = new DenseTensor<float>(input, new[] { 1, InputDim });
|
||||
var inputs = new List<NamedOnnxValue> { NamedOnnxValue.CreateFromTensor("state", tensor) };
|
||||
using var results = _session.Run(inputs);
|
||||
var output = results.First().AsTensor<float>();
|
||||
return output.ToArray();
|
||||
}
|
||||
|
||||
// ---- 辅助 ----
|
||||
|
||||
private static int RankToIdx(int gameNum) => gameNum switch
|
||||
{
|
||||
3 => 0, 4 => 1, 5 => 2, 6 => 3, 7 => 4, 8 => 5, 9 => 6,
|
||||
10 => 7, 11 => 8, 12 => 9, 13 => 10, 14 => 11, 16 => 12,
|
||||
_ => -1
|
||||
};
|
||||
|
||||
private static int SuitToIdx(int id)
|
||||
{
|
||||
// ID = (suit-1)*13 + n, suit 1-4
|
||||
int suit = (id - 1) / 13; // 0-3
|
||||
return suit >= 0 && suit < 4 ? suit : -1;
|
||||
}
|
||||
|
||||
private static int IdToGameNum(int id)
|
||||
{
|
||||
int n = (id - 1) % 13 + 1; // 1-13
|
||||
return n == 1 ? 14 : n == 2 ? 16 : n;
|
||||
}
|
||||
|
||||
private HashSet<int> GetSeenCardIds(PdkBotView view)
|
||||
{
|
||||
var seen = new HashSet<int>();
|
||||
// Track played cards through view: all hands that have appeared on table
|
||||
if (view.MaxPlayCard.Ids != null)
|
||||
foreach (var id in view.MaxPlayCard.Ids) seen.Add(id);
|
||||
// Also track from our own played cards (cards not in my hand anymore)
|
||||
return seen;
|
||||
}
|
||||
|
||||
private static int ChOffset(int ch) => ch * NumRanks * NumSuits;
|
||||
|
||||
// ---- 多牌候选（精简版，复用 IsmctsBot 逻辑）----
|
||||
|
||||
private void AddMultiCardCandidates(TCardInfoPdkF[] hand, int pos,
|
||||
List<(PlayOutCardPdkF play, int actionIdx)> candidates)
|
||||
{
|
||||
// 对子
|
||||
foreach (var g in hand.Where(c => c.GameState == 1).GroupBy(c => c.GameNum).Where(g => g.Count() >= 2))
|
||||
{
|
||||
var pair = g.Take(2).ToList();
|
||||
candidates.Add((MakePlay(pair, pos), RankToIdx(pair[0].GameNum)));
|
||||
}
|
||||
// 三张
|
||||
foreach (var g in hand.Where(c => c.GameState == 1).GroupBy(c => c.GameNum).Where(g => g.Count() >= 3))
|
||||
{
|
||||
var trip = g.Take(3).ToList();
|
||||
var rem = hand.Where(c => c.GameState == 1 && c.GameNum != g.Key).ToList();
|
||||
if (rem.Count >= 2)
|
||||
{
|
||||
// 优先带对子
|
||||
var pair = rem.GroupBy(c => c.GameNum).Where(x => x.Count() >= 2).OrderBy(x => x.Key).FirstOrDefault();
|
||||
var attach = pair != null ? pair.Take(2).ToList() : rem.OrderBy(c => c.GameNum).Take(2).ToList();
|
||||
candidates.Add((MakePlay(trip.Concat(attach).ToList(), pos), RankToIdx(trip[0].GameNum)));
|
||||
}
|
||||
}
|
||||
// 顺子
|
||||
var straights = PokerLogic.GetShunZi<TCardInfoPdkF>(hand);
|
||||
if (straights != null)
|
||||
foreach (var s in straights)
|
||||
{
|
||||
var cards = s.ToList();
|
||||
candidates.Add((MakePlay(cards, pos), RankToIdx(cards[0].GameNum)));
|
||||
}
|
||||
// 炸弹
|
||||
if (PokerLogic.GetPlayZhaDans<TCardInfoPdkF>(hand, out var bombs) && bombs != null)
|
||||
foreach (var b in bombs)
|
||||
{
|
||||
var cards = b.ToList();
|
||||
candidates.Add((MakePlay(cards, pos), RankToIdx(cards[0].GameNum)));
|
||||
}
|
||||
}
|
||||
|
||||
private static PlayOutCardPdkF MakePlay(List<TCardInfoPdkF> cards, int pos)
|
||||
=> new PlayOutCardPdkF { Pos = (byte)pos, GameNum = cards[0].GameNum, Ids = cards.Select(c => c.ID).ToArray(), Type = CardType1.None };
|
||||
|
||||
private static PlayOutCardPdkF Pass(PdkBotView v)
|
||||
=> new PlayOutCardPdkF { Pos = (byte)v.MyPos, Type = CardType1.None };
|
||||
|
||||
public void Dispose() => _session?.Dispose();
|
||||
}
|
||||
}
|
||||
@ -56,12 +56,22 @@ namespace PdkFriendServer.Logic
|
||||
this.GamePack = new PdkGamePack();
|
||||
}
|
||||
|
||||
/// <summary>懒初始化3个bot（BotMode==true时首次WaitWanJiaShuRu调用）</summary>
|
||||
private void InitBots()
|
||||
/// <summary>懒初始化3个bot（BotMode==true时首次WaitWanJiaShuRu调用）
|
||||
/// BotTypes: "ISMCTS" / "NEURAL" / null=默认ISMCTS
|
||||
/// 训练模式: ["NEURAL","NEURAL","ISMCTS"] = 2个模型bot + 1个ISMCTS</summary>
|
||||
public static string[] BotTypes = null;
|
||||
|
||||
private void InitBots()
|
||||
{
|
||||
_bots = new IPdkBot[3];
|
||||
for (int i = 0; i < 3; i++)
|
||||
_bots[i] = new IsmctsBot();
|
||||
{
|
||||
string type = BotTypes != null && BotTypes.Length > i ? BotTypes[i] : null;
|
||||
if (type == "NEURAL")
|
||||
_bots[i] = new NeuralBot("model.onnx", 0.1f);
|
||||
else
|
||||
_bots[i] = new IsmctsBot();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>构建当前玩家的可见视图——只看自己手牌+公开信息</summary>
|
||||
|
||||
@ -43,6 +43,8 @@
|
||||
<ItemGroup>
|
||||
<PackageReference Include="MessagePack.Annotations" />
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
|
||||
<PackageReference Include="Microsoft.ML.OnnxRuntime" />
|
||||
<PackageReference Include="Microsoft.ML.OnnxRuntime.Managed" />
|
||||
<PackageReference Include="Microsoft.NET.StringTools" />
|
||||
<PackageReference Include="System.Buffers" />
|
||||
<PackageReference Include="System.Memory" />
|
||||
|
||||
Reference in New Issue
Block a user