From a1e2ad9c2dbe890c68b948750ddb0f388fe71182 Mon Sep 17 00:00:00 2001 From: Josh-Heaps Date: Sat, 13 Jun 2026 18:44:14 -0600 Subject: [PATCH] Bug fixes --- JoshHeaps.Net/Controllers/ChessController.cs | 20 +++++ JoshHeaps.Net/Pages/Watch.cshtml | 5 ++ JoshHeaps.Net/Program.cs | 1 + .../Implementations/AutoTrainingService.cs | 80 ++++++++++++------- .../Implementations/AutoTrainingSettings.cs | 31 +++++++ .../Implementations/ChessEngineFactory.cs | 7 ++ .../Implementations/SelfPlayCoordinator.cs | 45 +++++++++-- .../wwwroot/js/ChessScripts/Spectate.js | 31 +++++++ 8 files changed, 183 insertions(+), 37 deletions(-) create mode 100644 JoshHeaps.Net/Services/Implementations/AutoTrainingSettings.cs diff --git a/JoshHeaps.Net/Controllers/ChessController.cs b/JoshHeaps.Net/Controllers/ChessController.cs index 9f5f7c1..5344ae0 100644 --- a/JoshHeaps.Net/Controllers/ChessController.cs +++ b/JoshHeaps.Net/Controllers/ChessController.cs @@ -17,6 +17,7 @@ public class ChessController( ILearnedWeightsStore weightsStore, IGameStore gameStore, ISelfPlayCoordinator selfPlay, + AutoTrainingSettings autoTraining, IHubContext chessHub) : ControllerBase { private static readonly TimeSpan _computerGameTimeout = TimeSpan.FromHours(1); @@ -106,6 +107,25 @@ public class ChessController( _ => ChessEngineKind.Custom }; + /// + /// Current number of background auto-training games (and the allowed maximum). Auto-training + /// itself runs only outside Development; this reflects the target the service is keeping. + /// + [HttpGet("autotrain")] + public ActionResult GetAutoTrain() => + Ok(new { count = autoTraining.GameCount, max = AutoTrainingSettings.MaxGames }); + + /// + /// Set how many auto-training games run concurrently (clamped to 0..max; 0 pauses training). + /// Takes effect live — the background service tops up or drains toward the new count. + /// + [HttpPost("autotrain")] + public ActionResult SetAutoTrain([FromQuery] int count) + { + autoTraining.GameCount = count; // clamped inside the setter + return Ok(new { count = autoTraining.GameCount, max = AutoTrainingSettings.MaxGames }); + } + /// /// Joins the "pool" of chess players. /// Test code expects to receive a GUID for the player diff --git a/JoshHeaps.Net/Pages/Watch.cshtml b/JoshHeaps.Net/Pages/Watch.cshtml index 3e04add..b4718e9 100644 --- a/JoshHeaps.Net/Pages/Watch.cshtml +++ b/JoshHeaps.Net/Pages/Watch.cshtml @@ -38,6 +38,11 @@ +
+ Auto-train games + + +
← Play a game View learned weights → diff --git a/JoshHeaps.Net/Program.cs b/JoshHeaps.Net/Program.cs index 936efe1..cd006d7 100644 --- a/JoshHeaps.Net/Program.cs +++ b/JoshHeaps.Net/Program.cs @@ -28,6 +28,7 @@ builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); if (!builder.Environment.IsDevelopment()) { diff --git a/JoshHeaps.Net/Services/Implementations/AutoTrainingService.cs b/JoshHeaps.Net/Services/Implementations/AutoTrainingService.cs index 6b5c2f3..607aeec 100644 --- a/JoshHeaps.Net/Services/Implementations/AutoTrainingService.cs +++ b/JoshHeaps.Net/Services/Implementations/AutoTrainingService.cs @@ -3,53 +3,71 @@ using JoshHeaps.Net.Services.Interfaces; namespace JoshHeaps.Net.Services.Implementations; /// -/// Continuously trains the learned engine in the background: two self-play games run in -/// parallel, the learned engine (skill 6) against Stockfish (skill 20), one with Stockfish as -/// black and one as white. Each slot is independent — when its game finishes it immediately -/// starts another under the same conditions, without waiting on the other slot. Registered -/// only outside Development and gated by the ChessEngine:AutoTrain config flag. +/// Continuously trains the learned engine in the background by keeping a configurable number of +/// self-play games running — the learned engine (skill 6) against Stockfish (skill 20), alternating +/// which color Stockfish takes so the model trains on both. The target count is read live from +/// (adjustable from the website): when a game finishes another +/// starts to refill the pool, raising the count starts more, and lowering it lets the surplus drain +/// as games finish (0 pauses training). Registered only outside Development and gated by the +/// ChessEngine:AutoTrain config flag. /// public sealed class AutoTrainingService( ISelfPlayCoordinator coordinator, + AutoTrainingSettings settings, ILogger logger) : BackgroundService { private const int LearnedSkill = 6; private const int StockfishSkill = 20; private static readonly TimeSpan _restartBackoff = TimeSpan.FromSeconds(5); + private static readonly TimeSpan _pollInterval = TimeSpan.FromSeconds(2); - protected override Task ExecuteAsync(CancellationToken stoppingToken) + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - // Learned plays both colors so the model trains symmetrically; each slot is its own loop. - var stockfishBlack = RunSlot( - new SelfPlayConfig(ChessEngineKind.CustomLearned, LearnedSkill, ChessEngineKind.Stockfish, StockfishSkill), - stoppingToken); - var stockfishWhite = RunSlot( - new SelfPlayConfig(ChessEngineKind.Stockfish, StockfishSkill, ChessEngineKind.CustomLearned, LearnedSkill), - stoppingToken); + var running = new List(); + int started = 0; - return Task.WhenAll(stockfishBlack, stockfishWhite); - } - - private async Task RunSlot(SelfPlayConfig config, CancellationToken stoppingToken) - { while (!stoppingToken.IsCancellationRequested) { + running.RemoveAll(t => t.IsCompleted); + + int desired = settings.GameCount; + bool startFailed = false; + + while (running.Count < desired && !stoppingToken.IsCancellationRequested) + { + // Alternate Stockfish's color so the learned engine trains as both white and black. + var config = started++ % 2 == 0 + ? new SelfPlayConfig(ChessEngineKind.CustomLearned, LearnedSkill, ChessEngineKind.Stockfish, StockfishSkill) + : new SelfPlayConfig(ChessEngineKind.Stockfish, StockfishSkill, ChessEngineKind.CustomLearned, LearnedSkill); + + try + { + var (_, completion) = coordinator.StartGame(config, stoppingToken); + running.Add(completion); + } + catch (Exception ex) + { + // Most likely an engine failing to start (e.g. Stockfish). Back off so a + // persistent failure doesn't spin a tight loop, then try again. + logger.LogError(ex, "Failed to start an auto-training game; retrying after backoff."); + startFailed = true; + break; + } + } + try { - await coordinator.StartGame(config, stoppingToken).Completion; - } - catch (OperationCanceledException) - { - break; - } - catch (Exception ex) - { - // Most likely an engine failing to start (e.g. Stockfish). Back off so a - // persistent failure doesn't spin a tight loop, then try again. - logger.LogError(ex, "Auto-training game failed to start; retrying after backoff."); - try { await Task.Delay(_restartBackoff, stoppingToken); } - catch (OperationCanceledException) { break; } + if (startFailed) + await Task.Delay(_restartBackoff, stoppingToken); + else if (running.Count > 0) + // Wake when any game finishes (to refill) or after a short poll (to pick up a + // count increase promptly). + await Task.WhenAny(Task.WhenAny(running), Task.Delay(_pollInterval, stoppingToken)); + else + // Pool is empty (count is 0) — just poll for the count to change. + await Task.Delay(_pollInterval, stoppingToken); } + catch (OperationCanceledException) { break; } } } } diff --git a/JoshHeaps.Net/Services/Implementations/AutoTrainingSettings.cs b/JoshHeaps.Net/Services/Implementations/AutoTrainingSettings.cs new file mode 100644 index 0000000..16e605f --- /dev/null +++ b/JoshHeaps.Net/Services/Implementations/AutoTrainingSettings.cs @@ -0,0 +1,31 @@ +using Microsoft.Extensions.Options; + +namespace JoshHeaps.Net.Services.Implementations; + +/// +/// Runtime-adjustable auto-training settings. Singleton so the value set from the website (via the +/// chess controller) is seen live by the background . Seeded from +/// and clamped to a sane range. +/// +public sealed class AutoTrainingSettings +{ + /// Upper bound on concurrent auto-training games (each spawns a Stockfish + a learned engine). + public const int MaxGames = 16; + + private int _gameCount; + + public AutoTrainingSettings(IOptions options) + => _gameCount = Clamp(options.Value.AutoTrainGameCount); + + /// + /// Number of auto-training games to keep running concurrently. 0 pauses auto-training. + /// Reads/writes are atomic; the background service reads this every cycle. + /// + public int GameCount + { + get => Volatile.Read(ref _gameCount); + set => Volatile.Write(ref _gameCount, Clamp(value)); + } + + private static int Clamp(int n) => Math.Clamp(n, 0, MaxGames); +} diff --git a/JoshHeaps.Net/Services/Implementations/ChessEngineFactory.cs b/JoshHeaps.Net/Services/Implementations/ChessEngineFactory.cs index 8fcad4c..d0833a8 100644 --- a/JoshHeaps.Net/Services/Implementations/ChessEngineFactory.cs +++ b/JoshHeaps.Net/Services/Implementations/ChessEngineFactory.cs @@ -36,6 +36,13 @@ public sealed class ChessEngineOptions /// redeploy. Override via the ChessEngine__AutoTrain environment variable. /// public bool AutoTrain { get; set; } = true; + + /// + /// How many auto-training games run concurrently (when is on). This is + /// the starting value; it can be changed at runtime from the website. Override the default via + /// the ChessEngine__AutoTrainGameCount environment variable. + /// + public int AutoTrainGameCount { get; set; } = 2; } /// Creates the configured per game. diff --git a/JoshHeaps.Net/Services/Implementations/SelfPlayCoordinator.cs b/JoshHeaps.Net/Services/Implementations/SelfPlayCoordinator.cs index 6cca76f..af6ce84 100644 --- a/JoshHeaps.Net/Services/Implementations/SelfPlayCoordinator.cs +++ b/JoshHeaps.Net/Services/Implementations/SelfPlayCoordinator.cs @@ -18,6 +18,10 @@ public sealed class SelfPlayCoordinator( private static readonly TimeSpan _selfPlayMoveDelay = TimeSpan.FromSeconds(1); private static readonly TimeSpan _selfPlayResultTimeout = TimeSpan.FromSeconds(30); + // Abort a game if a single engine move takes longer than this — a stopgap for engines + // (usually Stockfish) that occasionally freeze and would otherwise hang the game. + private static readonly TimeSpan _moveTimeout = TimeSpan.FromSeconds(60); + // Plies of random legal moves at the start of a training game, so self-play and // engine-vs-engine games explore different lines instead of replaying one game. private const int _openingRandomPlies = 4; @@ -61,11 +65,13 @@ public sealed class SelfPlayCoordinator( } /// - /// Drives the game to completion, then trains from it. Never throws — a failure just ends - /// the game (and the trainer is always freed), so callers can safely await or ignore it. + /// Drives the game to completion, then trains from it. Never throws — a failure (or a frozen + /// engine) just ends the game and the trainer is always freed, so callers can await or ignore. /// private async Task RunAsync(GameState gameState, CancellationToken cancellationToken) { + bool aborted = false; + try { // Give spectators a moment to join the SignalR group before the first move. @@ -81,21 +87,48 @@ public sealed class SelfPlayCoordinator( while (IsLive(gameState, cancellationToken)) { - await orchestrator.PlayAsync(gameState); + await PlayMoveWithTimeoutAsync(gameState, cancellationToken); await Task.Delay(_selfPlayMoveDelay, cancellationToken); } } - catch (OperationCanceledException) { /* service shutting down */ } + catch (OperationCanceledException) { aborted = true; /* service shutting down */ } + catch (TimeoutException) + { + aborted = true; + Console.WriteLine($"Self-play game {gameState.GameId} aborted: a move took over " + + $"{_moveTimeout.TotalSeconds:n0}s (likely a frozen engine)."); + } catch (Exception ex) { + aborted = true; Console.WriteLine($"Self-play game {gameState.GameId} stopped: {ex.Message}"); } ApplyLearning(gameState); - // Leave the finished game in place briefly so spectators can see the result. + // A clean finish lingers briefly so spectators see the result; an aborted/hung game is + // torn down immediately so its engines (and any frozen Stockfish process) are released. if (gameStore.Contains(gameState.GameId)) - gameStore.ScheduleRemove(gameState.GameId, _selfPlayResultTimeout); + gameStore.ScheduleRemove(gameState.GameId, aborted ? TimeSpan.Zero : _selfPlayResultTimeout); + } + + /// + /// Plays one engine move, abandoning it if it exceeds (throwing + /// ). The abandoned move's eventual fault — it errors once the + /// game's engines are disposed — is observed so it isn't an unobserved task exception. + /// + private async Task PlayMoveWithTimeoutAsync(GameState gameState, CancellationToken cancellationToken) + { + var play = orchestrator.PlayAsync(gameState); + try + { + await play.WaitAsync(_moveTimeout, cancellationToken); + } + catch (TimeoutException) + { + _ = play.ContinueWith(static t => { _ = t.Exception; }, TaskScheduler.Default); + throw; + } } private bool IsLive(GameState gameState, CancellationToken cancellationToken) => diff --git a/JoshHeaps.Net/wwwroot/js/ChessScripts/Spectate.js b/JoshHeaps.Net/wwwroot/js/ChessScripts/Spectate.js index d576798..35fa15e 100644 --- a/JoshHeaps.Net/wwwroot/js/ChessScripts/Spectate.js +++ b/JoshHeaps.Net/wwwroot/js/ChessScripts/Spectate.js @@ -23,9 +23,40 @@ const Spectate = { } await this.refreshGames(); + await this.loadAutoTrainCount(); setInterval(() => this.refreshGames(), 5000); }, + // Auto-training runs server-side; show its target count and let it be changed here. + async loadAutoTrainCount() { + const input = document.getElementById("autoTrainCount"); + if (!input) return; + + try { + const response = await fetch("/api/chess/autotrain"); + const data = await response.json(); + input.max = data.max; + // Don't clobber the value while the user is editing it. + if (document.activeElement !== input) + input.value = data.count; + } catch { + // Leave the control as-is if auto-training status can't be read. + } + }, + + async setAutoTrainCount() { + const input = document.getElementById("autoTrainCount"); + const count = Math.max(0, parseInt(input.value, 10) || 0); + + try { + const response = await fetch(`/api/chess/autotrain?count=${count}`, { method: "POST" }); + const data = await response.json(); + input.value = data.count; + } catch (err) { + console.error("❌ Could not set the auto-training game count.", err); + } + }, + async startCpuGame() { const params = new URLSearchParams({ whiteEngine: document.getElementById("whiteEngine").value,