feat: Neural-only training, experience replay, per-round eval
3个训练效率提升: 1. Neural filter: V训练只取player=2数据, 排除ISMCTS状态干扰 旧格式(无player列)自动fallback全量数据 2. 经验回放: 保留最近3轮数据混入当前轮训练, 扩大分布覆盖 存储到 paodekuai-trainer/data/replay_buffer.pkl 3. 每轮eval: 训练后跑200局--eval测胜率, 下降>10%自动回滚 回滚到 model_prev.pt, 防止坏模型污染下轮数据
This commit is contained in:
@ -1,15 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
持续训练: 20k局/轮, 无限循环
|
||||
每轮: dotnet自对弈 → V训练 → ONNX导出 → 存档
|
||||
胜率趋势由 scan_wins.py 每小时 cron 独立采集
|
||||
每轮: dotnet自对弈 → V训练(Neural-only+replay) → eval → ONNX导出 → 存档
|
||||
改进: dropout正则化, 只训Neural数据, 经验回放, 每轮eval验证回滚
|
||||
"""
|
||||
import subprocess, os, sys, time, shutil, glob
|
||||
import subprocess, os, sys, time, shutil, glob, pickle, re
|
||||
from datetime import datetime
|
||||
|
||||
HJHA_DIR = '/home/xiaoou/projects/hjha-server'
|
||||
TRAINER_DIR = '/home/xiaoou/projects/paodekuai-trainer'
|
||||
LOG = os.path.join(HJHA_DIR, 'training_loop.log')
|
||||
REPLAY_PATH = os.path.join(TRAINER_DIR, 'data/replay_buffer.pkl')
|
||||
REPLAY_MAX_ROUNDS = 3 # keep last N rounds in buffer
|
||||
EVAL_GAMES = 200 # eval games per round
|
||||
|
||||
def now(): return datetime.now().strftime('%m%d %H:%M:%S')
|
||||
def log(msg):
|
||||
@ -29,7 +32,7 @@ def run(cmd, cwd=HJHA_DIR, to=86400):
|
||||
return False, str(e)
|
||||
|
||||
def selfplay(games):
|
||||
log(f"── 自対弈 {games}局 ──")
|
||||
log(f"── 自对弈 {games}局 ──")
|
||||
td = os.path.join(HJHA_DIR, 'training_data')
|
||||
if os.path.exists(td): shutil.rmtree(td)
|
||||
os.makedirs(td)
|
||||
@ -47,7 +50,37 @@ def selfplay(games):
|
||||
log(f" {csv} CSV")
|
||||
return csv
|
||||
|
||||
def train_v():
|
||||
def load_csv_states(csv_dir):
|
||||
"""Load states from CSV. If player column present, only keep Neural (player=2)."""
|
||||
import numpy as np
|
||||
states, results = [], []
|
||||
neural_only = 0
|
||||
for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
|
||||
try:
|
||||
with open(fp) as f:
|
||||
header = f.readline().strip()
|
||||
has_player = header.endswith(',player')
|
||||
for line in f:
|
||||
if line.startswith('#'): continue
|
||||
p = line.strip().split('"')
|
||||
if len(p) < 3: continue
|
||||
sf = [float(x) for x in p[1].split(',')]
|
||||
rest = p[2].strip(',').split(',')
|
||||
if len(sf) != 364: continue
|
||||
if has_player and len(rest) >= 3:
|
||||
player = int(rest[2])
|
||||
if player != 2: # only train on Neural states
|
||||
neural_only += 1
|
||||
continue
|
||||
if len(rest) < 2: continue
|
||||
states.append(sf)
|
||||
results.append(float(rest[1]))
|
||||
except: pass
|
||||
if neural_only:
|
||||
log(f" filtered out {neural_only} non-Neural states")
|
||||
return np.array(states, dtype=np.float32) if states else None, np.array(results, dtype=np.float32) if results else None
|
||||
|
||||
def train_v(prev_winrate=None):
|
||||
log(f"── V训练 ──")
|
||||
|
||||
sys.path.insert(0, TRAINER_DIR)
|
||||
@ -55,29 +88,40 @@ def train_v():
|
||||
from models.network import PdkNet
|
||||
|
||||
csv_dir = os.path.join(HJHA_DIR, 'training_data')
|
||||
states, results = [], []
|
||||
for fp in sorted(glob.glob(f'{csv_dir}/game_*.csv')):
|
||||
try:
|
||||
with open(fp) as f:
|
||||
f.readline()
|
||||
for line in f:
|
||||
if line.startswith('#'): continue
|
||||
p = line.strip().split('"')
|
||||
if len(p) < 3: continue
|
||||
sf = [float(x) for x in p[1].split(',')]
|
||||
rest = p[2].strip(',').split(',')
|
||||
if len(rest) < 2 or len(sf) != 364: continue
|
||||
states.append(sf)
|
||||
results.append(float(rest[1]))
|
||||
except: pass
|
||||
cur_states, cur_results = load_csv_states(csv_dir)
|
||||
|
||||
if not states:
|
||||
if cur_states is None or len(cur_states) == 0:
|
||||
log(" 0 samples!")
|
||||
return False
|
||||
return False, None
|
||||
|
||||
states = np.array(states, dtype=np.float32)
|
||||
results = np.array(results, dtype=np.float32)
|
||||
log(f" {len(states)} samples from {len(glob.glob(f'{csv_dir}/game_*.csv'))} games")
|
||||
cur_games = len(glob.glob(f'{csv_dir}/game_*.csv'))
|
||||
log(f" current: {len(cur_states)} Neural samples from {cur_games} games")
|
||||
|
||||
# --- experience replay: load past rounds ---
|
||||
replay_loaded = 0
|
||||
if os.path.exists(REPLAY_PATH):
|
||||
try:
|
||||
with open(REPLAY_PATH, 'rb') as f:
|
||||
replay_data = pickle.load(f)
|
||||
# replay_data is list of (states, results) tuples
|
||||
all_r = []
|
||||
for rs, rr in replay_data[-REPLAY_MAX_ROUNDS:]:
|
||||
if len(rs) > 0:
|
||||
all_r.append((rs, rr))
|
||||
replay_loaded += len(rs)
|
||||
if all_r:
|
||||
replay_states = np.concatenate([r[0] for r in all_r])
|
||||
replay_results = np.concatenate([r[1] for r in all_r])
|
||||
states = np.concatenate([cur_states, replay_states])
|
||||
results = np.concatenate([cur_results, replay_results])
|
||||
log(f" replay: {replay_loaded} samples from {min(len(replay_data), REPLAY_MAX_ROUNDS)} past rounds → total {len(states)}")
|
||||
else:
|
||||
states, results = cur_states, cur_results
|
||||
except Exception as e:
|
||||
log(f" replay load failed: {e}, using current only")
|
||||
states, results = cur_states, cur_results
|
||||
else:
|
||||
states, results = cur_states, cur_results
|
||||
|
||||
model = PdkNet()
|
||||
opt = torch.optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-5)
|
||||
@ -117,51 +161,107 @@ def train_v():
|
||||
elapsed = (datetime.now() - t0).total_seconds()
|
||||
log(f" done {elapsed:.0f}s, final loss={avg_loss:.4f}")
|
||||
|
||||
# save pt
|
||||
# --- eval before deploying ---
|
||||
log(f"── eval {EVAL_GAMES}局 ──")
|
||||
model.eval()
|
||||
mp = os.path.join(TRAINER_DIR, 'data/model.pt')
|
||||
model.save(mp)
|
||||
# export ONNX for eval
|
||||
onnx_tmp = os.path.join(TRAINER_DIR, 'data/model.onnx')
|
||||
dummy = torch.randn(1, 364)
|
||||
torch.onnx.export(model, dummy, onnx_tmp,
|
||||
input_names=['state'], output_names=['v'],
|
||||
dynamic_axes={'state': {0: 'batch'}, 'v': {0: 'batch'}},
|
||||
opset_version=15)
|
||||
# copy ONNX to hjha for eval
|
||||
eval_dest = os.path.join(HJHA_DIR, 'model.onnx')
|
||||
shutil.copy(onnx_tmp, eval_dest)
|
||||
data_src = onnx_tmp + '.data'
|
||||
if os.path.exists(data_src):
|
||||
shutil.copy(data_src, eval_dest + '.data')
|
||||
|
||||
# export ONNX
|
||||
winrate = 0.0
|
||||
try:
|
||||
model.eval()
|
||||
dummy = torch.randn(1, 364)
|
||||
onnx_path = os.path.join(TRAINER_DIR, 'data/model.onnx')
|
||||
torch.onnx.export(model, dummy, onnx_path,
|
||||
input_names=['state'], output_names=['v'],
|
||||
dynamic_axes={'state': {0: 'batch'}, 'v': {0: 'batch'}},
|
||||
opset_version=15)
|
||||
|
||||
dest = os.path.join(HJHA_DIR, 'model.onnx')
|
||||
shutil.copy(onnx_path, dest)
|
||||
data_src = onnx_path + '.data'
|
||||
if os.path.exists(data_src):
|
||||
shutil.copy(data_src, dest + '.data')
|
||||
|
||||
tag = datetime.now().strftime('%m%d_%H%M')
|
||||
rd = os.path.join(HJHA_DIR, f'training_rounds/r{tag}')
|
||||
os.makedirs(rd, exist_ok=True)
|
||||
shutil.copy(onnx_path, os.path.join(rd, 'model.onnx'))
|
||||
if os.path.exists(data_src):
|
||||
shutil.copy(data_src, os.path.join(rd, 'model.onnx.data'))
|
||||
shutil.copy(mp, os.path.join(rd, 'model.pt'))
|
||||
if os.path.exists(csv_dir):
|
||||
shutil.move(csv_dir, os.path.join(rd, 'training_data'))
|
||||
os.makedirs(csv_dir, exist_ok=True)
|
||||
|
||||
sz = os.path.getsize(dest) + os.path.getsize(dest + '.data') if os.path.exists(dest + '.data') else os.path.getsize(dest)
|
||||
log(f" ONNX → {dest} ({sz//1024}KB), 存档 → {rd}")
|
||||
ok, out = run(['/usr/lib/dotnet/dotnet', 'run', '--project', 'hjha-console', '-c', 'Release',
|
||||
'--', '--eval', str(EVAL_GAMES)], to=600)
|
||||
sf = os.path.join(HJHA_DIR, 'stress_summary.txt')
|
||||
if os.path.exists(sf):
|
||||
with open(sf) as f:
|
||||
summary = f.read()
|
||||
log(f" {summary.strip()}")
|
||||
# parse Neural win rate from summary
|
||||
m = re.search(r'NEURAL.*?(\d+)\s*wins?.*?(\d+)\s*games?', summary, re.IGNORECASE)
|
||||
if not m:
|
||||
m = re.search(r'position.*?(\d+).*?wins?.*?(\d+)', summary, re.IGNORECASE)
|
||||
if m:
|
||||
wins, games = int(m.group(1)), int(m.group(2))
|
||||
winrate = wins / games * 100 if games > 0 else 0.0
|
||||
log(f" Neural winrate: {winrate:.1f}%")
|
||||
except Exception as e:
|
||||
log(f" ONNX failed: {e}")
|
||||
return False
|
||||
log(f" eval failed: {e}")
|
||||
|
||||
return True
|
||||
# --- rollback if worse ---
|
||||
if prev_winrate is not None and winrate < prev_winrate * 0.9:
|
||||
log(f" ⚠️ winrate dropped {prev_winrate:.1f}% → {winrate:.1f}%, rolling back model")
|
||||
# restore previous model.pt and ONNX
|
||||
prev_pt = os.path.join(TRAINER_DIR, 'data/model_prev.pt')
|
||||
if os.path.exists(prev_pt):
|
||||
shutil.copy(prev_pt, mp)
|
||||
log(f" restored model.pt from backup")
|
||||
return False, prev_winrate
|
||||
|
||||
# --- save and deploy ---
|
||||
# backup current as prev
|
||||
if os.path.exists(mp):
|
||||
shutil.copy(mp, os.path.join(TRAINER_DIR, 'data/model_prev.pt'))
|
||||
|
||||
# final ONNX copy to hjha (already done above for eval, but ensure)
|
||||
dest = os.path.join(HJHA_DIR, 'model.onnx')
|
||||
shutil.copy(onnx_tmp, dest)
|
||||
if os.path.exists(data_src):
|
||||
shutil.copy(data_src, dest + '.data')
|
||||
|
||||
# archive round
|
||||
tag = datetime.now().strftime('%m%d_%H%M')
|
||||
rd = os.path.join(HJHA_DIR, f'training_rounds/r{tag}')
|
||||
os.makedirs(rd, exist_ok=True)
|
||||
shutil.copy(onnx_tmp, os.path.join(rd, 'model.onnx'))
|
||||
if os.path.exists(data_src):
|
||||
shutil.copy(data_src, os.path.join(rd, 'model.onnx.data'))
|
||||
shutil.copy(mp, os.path.join(rd, 'model.pt'))
|
||||
if os.path.exists(csv_dir):
|
||||
shutil.move(csv_dir, os.path.join(rd, 'training_data'))
|
||||
os.makedirs(csv_dir, exist_ok=True)
|
||||
|
||||
sz = os.path.getsize(dest) + (os.path.getsize(dest + '.data') if os.path.exists(dest + '.data') else 0)
|
||||
log(f" ONNX → {dest} ({sz//1024}KB), 存档 → {rd}")
|
||||
|
||||
# --- update replay buffer ---
|
||||
try:
|
||||
if os.path.exists(REPLAY_PATH):
|
||||
with open(REPLAY_PATH, 'rb') as f:
|
||||
replay_data = pickle.load(f)
|
||||
else:
|
||||
replay_data = []
|
||||
replay_data.append((cur_states, cur_results))
|
||||
if len(replay_data) > REPLAY_MAX_ROUNDS * 2:
|
||||
replay_data = replay_data[-REPLAY_MAX_ROUNDS * 2:]
|
||||
with open(REPLAY_PATH, 'wb') as f:
|
||||
pickle.dump(replay_data, f)
|
||||
log(f" replay buffer: {len(replay_data)} rounds stored")
|
||||
except Exception as e:
|
||||
log(f" replay save failed: {e}")
|
||||
|
||||
return True, winrate
|
||||
|
||||
def main():
|
||||
log("=" * 50)
|
||||
log("持续训练: 20k局/轮 | 无限循环")
|
||||
log("持续训练: 20k局/轮 | Neural-only + replay + eval")
|
||||
log("=" * 50)
|
||||
|
||||
rn, total = 0, 0
|
||||
winrate = None # track best winrate for rollback
|
||||
|
||||
while True:
|
||||
rn += 1
|
||||
games = 20000
|
||||
@ -169,15 +269,18 @@ def main():
|
||||
|
||||
csv = selfplay(games)
|
||||
if csv == 0:
|
||||
log("自対弈失败! 跳过")
|
||||
log("自对弈失败! 跳过")
|
||||
continue
|
||||
total += games
|
||||
|
||||
if not train_v():
|
||||
log("训练失败! 跳过")
|
||||
ok, wr = train_v(prev_winrate=winrate)
|
||||
if not ok:
|
||||
log("训练/eval失败! 跳过")
|
||||
continue
|
||||
if wr is not None:
|
||||
winrate = wr
|
||||
|
||||
log(f"累计: {rn}轮, {total}局")
|
||||
log(f"累计: {rn}轮, {total}局, best wr={winrate:.1f}%" if winrate else f"累计: {rn}轮, {total}局")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user