Bug fixes

JoshHeaps.Net/Controllers/ChessController.cs

@@ -17,6 +17,7 @@ public class ChessController(
     ILearnedWeightsStore weightsStore,
     IGameStore gameStore,
     ISelfPlayCoordinator selfPlay,
+    AutoTrainingSettings autoTraining,
     IHubContext<ChessHub> chessHub) : ControllerBase
 {
     private static readonly TimeSpan _computerGameTimeout = TimeSpan.FromHours(1);
@@ -106,6 +107,25 @@ public class ChessController(
         _ => ChessEngineKind.Custom
     };
 
+    /// <summary>
+    /// Current number of background auto-training games (and the allowed maximum). Auto-training
+    /// itself runs only outside Development; this reflects the target the service is keeping.
+    /// </summary>
+    [HttpGet("autotrain")]
+    public ActionResult GetAutoTrain() =>
+        Ok(new { count = autoTraining.GameCount, max = AutoTrainingSettings.MaxGames });
+
+    /// <summary>
+    /// Set how many auto-training games run concurrently (clamped to 0..max; 0 pauses training).
+    /// Takes effect live — the background service tops up or drains toward the new count.
+    /// </summary>
+    [HttpPost("autotrain")]
+    public ActionResult SetAutoTrain([FromQuery] int count)
+    {
+        autoTraining.GameCount = count;   // clamped inside the setter
+        return Ok(new { count = autoTraining.GameCount, max = AutoTrainingSettings.MaxGames });
+    }
+
     /// <summary>
     /// Joins the "pool" of chess players.
     /// Test code expects to receive a GUID for the player

JoshHeaps.Net/Pages/Watch.cshtml

@@ -38,6 +38,11 @@
             </select>
         </fieldset>
         <button id="startCpuVsCpu" onclick="Spectate.startCpuGame()">Watch CPU vs CPU</button>
+        <fieldset class="enginePicker">
+            <legend>Auto-train games</legend>
+            <input id="autoTrainCount" type="number" min="0" max="16" step="1" />
+            <button id="applyAutoTrain" onclick="Spectate.setAutoTrainCount()">Apply</button>
+        </fieldset>
     </div>
     <a id="backToPlay" href="/chess">← Play a game</a>
     <a id="viewWeights" href="/weights">View learned weights →</a>

JoshHeaps.Net/Program.cs

@@ -28,6 +28,7 @@ builder.Services.AddSingleton<IChessEngineFactory, ChessEngineFactory>();
 builder.Services.AddSingleton<IComputerMoveOrchestrator, ComputerMoveOrchestrator>();
 builder.Services.AddSingleton<IGameStore, GameStore>();
 builder.Services.AddSingleton<ISelfPlayCoordinator, SelfPlayCoordinator>();
+builder.Services.AddSingleton<AutoTrainingSettings>();
 
 if (!builder.Environment.IsDevelopment())
 {

JoshHeaps.Net/Services/Implementations/AutoTrainingService.cs

@@ -3,53 +3,71 @@ using JoshHeaps.Net.Services.Interfaces;
 namespace JoshHeaps.Net.Services.Implementations;
 
 /// <summary>
-/// Continuously trains the learned engine in the background: two self-play games run in
-/// parallel, the learned engine (skill 6) against Stockfish (skill 20), one with Stockfish as
-/// black and one as white. Each slot is independent — when its game finishes it immediately
-/// starts another under the same conditions, without waiting on the other slot. Registered
-/// only outside Development and gated by the ChessEngine:AutoTrain config flag.
+/// Continuously trains the learned engine in the background by keeping a configurable number of
+/// self-play games running — the learned engine (skill 6) against Stockfish (skill 20), alternating
+/// which color Stockfish takes so the model trains on both. The target count is read live from
+/// <see cref="AutoTrainingSettings"/> (adjustable from the website): when a game finishes another
+/// starts to refill the pool, raising the count starts more, and lowering it lets the surplus drain
+/// as games finish (0 pauses training). Registered only outside Development and gated by the
+/// ChessEngine:AutoTrain config flag.
 /// </summary>
 public sealed class AutoTrainingService(
     ISelfPlayCoordinator coordinator,
+    AutoTrainingSettings settings,
     ILogger<AutoTrainingService> logger) : BackgroundService
 {
     private const int LearnedSkill = 6;
     private const int StockfishSkill = 20;
     private static readonly TimeSpan _restartBackoff = TimeSpan.FromSeconds(5);
+    private static readonly TimeSpan _pollInterval = TimeSpan.FromSeconds(2);
 
-    protected override Task ExecuteAsync(CancellationToken stoppingToken)
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
-        // Learned plays both colors so the model trains symmetrically; each slot is its own loop.
-        var stockfishBlack = RunSlot(
-            new SelfPlayConfig(ChessEngineKind.CustomLearned, LearnedSkill, ChessEngineKind.Stockfish, StockfishSkill),
-            stoppingToken);
-        var stockfishWhite = RunSlot(
-            new SelfPlayConfig(ChessEngineKind.Stockfish, StockfishSkill, ChessEngineKind.CustomLearned, LearnedSkill),
-            stoppingToken);
+        var running = new List<Task>();
+        int started = 0;
 
-        return Task.WhenAll(stockfishBlack, stockfishWhite);
-    }
-
-    private async Task RunSlot(SelfPlayConfig config, CancellationToken stoppingToken)
-    {
         while (!stoppingToken.IsCancellationRequested)
         {
+            running.RemoveAll(t => t.IsCompleted);
+
+            int desired = settings.GameCount;
+            bool startFailed = false;
+
+            while (running.Count < desired && !stoppingToken.IsCancellationRequested)
+            {
+                // Alternate Stockfish's color so the learned engine trains as both white and black.
+                var config = started++ % 2 == 0
+                    ? new SelfPlayConfig(ChessEngineKind.CustomLearned, LearnedSkill, ChessEngineKind.Stockfish, StockfishSkill)
+                    : new SelfPlayConfig(ChessEngineKind.Stockfish, StockfishSkill, ChessEngineKind.CustomLearned, LearnedSkill);
+
                 try
                 {
-                await coordinator.StartGame(config, stoppingToken).Completion;
-            }
-            catch (OperationCanceledException)
-            {
-                break;
+                    var (_, completion) = coordinator.StartGame(config, stoppingToken);
+                    running.Add(completion);
                 }
                 catch (Exception ex)
                 {
                     // Most likely an engine failing to start (e.g. Stockfish). Back off so a
                     // persistent failure doesn't spin a tight loop, then try again.
-                logger.LogError(ex, "Auto-training game failed to start; retrying after backoff.");
-                try { await Task.Delay(_restartBackoff, stoppingToken); }
+                    logger.LogError(ex, "Failed to start an auto-training game; retrying after backoff.");
+                    startFailed = true;
+                    break;
+                }
+            }
+
+            try
+            {
+                if (startFailed)
+                    await Task.Delay(_restartBackoff, stoppingToken);
+                else if (running.Count > 0)
+                    // Wake when any game finishes (to refill) or after a short poll (to pick up a
+                    // count increase promptly).
+                    await Task.WhenAny(Task.WhenAny(running), Task.Delay(_pollInterval, stoppingToken));
+                else
+                    // Pool is empty (count is 0) — just poll for the count to change.
+                    await Task.Delay(_pollInterval, stoppingToken);
+            }
             catch (OperationCanceledException) { break; }
         }
     }
-    }
 }

JoshHeaps.Net/Services/Implementations/AutoTrainingSettings.cs

@@ -0,0 +1,31 @@
+using Microsoft.Extensions.Options;
+
+namespace JoshHeaps.Net.Services.Implementations;
+
+/// <summary>
+/// Runtime-adjustable auto-training settings. Singleton so the value set from the website (via the
+/// chess controller) is seen live by the background <see cref="AutoTrainingService"/>. Seeded from
+/// <see cref="ChessEngineOptions.AutoTrainGameCount"/> and clamped to a sane range.
+/// </summary>
+public sealed class AutoTrainingSettings
+{
+    /// <summary>Upper bound on concurrent auto-training games (each spawns a Stockfish + a learned engine).</summary>
+    public const int MaxGames = 16;
+
+    private int _gameCount;
+
+    public AutoTrainingSettings(IOptions<ChessEngineOptions> options)
+        => _gameCount = Clamp(options.Value.AutoTrainGameCount);
+
+    /// <summary>
+    /// Number of auto-training games to keep running concurrently. 0 pauses auto-training.
+    /// Reads/writes are atomic; the background service reads this every cycle.
+    /// </summary>
+    public int GameCount
+    {
+        get => Volatile.Read(ref _gameCount);
+        set => Volatile.Write(ref _gameCount, Clamp(value));
+    }
+
+    private static int Clamp(int n) => Math.Clamp(n, 0, MaxGames);
+}

JoshHeaps.Net/Services/Implementations/ChessEngineFactory.cs

@@ -36,6 +36,13 @@ public sealed class ChessEngineOptions
     /// redeploy. Override via the <c>ChessEngine__AutoTrain</c> environment variable.
     /// </summary>
     public bool AutoTrain { get; set; } = true;
+
+    /// <summary>
+    /// How many auto-training games run concurrently (when <see cref="AutoTrain"/> is on). This is
+    /// the starting value; it can be changed at runtime from the website. Override the default via
+    /// the <c>ChessEngine__AutoTrainGameCount</c> environment variable.
+    /// </summary>
+    public int AutoTrainGameCount { get; set; } = 2;
 }
 
 /// <summary>Creates the configured <see cref="IChessEngine"/> per game.</summary>

JoshHeaps.Net/Services/Implementations/SelfPlayCoordinator.cs

@@ -18,6 +18,10 @@ public sealed class SelfPlayCoordinator(
     private static readonly TimeSpan _selfPlayMoveDelay = TimeSpan.FromSeconds(1);
     private static readonly TimeSpan _selfPlayResultTimeout = TimeSpan.FromSeconds(30);
 
+    // Abort a game if a single engine move takes longer than this — a stopgap for engines
+    // (usually Stockfish) that occasionally freeze and would otherwise hang the game.
+    private static readonly TimeSpan _moveTimeout = TimeSpan.FromSeconds(60);
+
     // Plies of random legal moves at the start of a training game, so self-play and
     // engine-vs-engine games explore different lines instead of replaying one game.
     private const int _openingRandomPlies = 4;
@@ -61,11 +65,13 @@ public sealed class SelfPlayCoordinator(
     }
 
     /// <summary>
-    /// Drives the game to completion, then trains from it. Never throws — a failure just ends
-    /// the game (and the trainer is always freed), so callers can safely await or ignore it.
+    /// Drives the game to completion, then trains from it. Never throws — a failure (or a frozen
+    /// engine) just ends the game and the trainer is always freed, so callers can await or ignore.
     /// </summary>
     private async Task RunAsync(GameState gameState, CancellationToken cancellationToken)
     {
+        bool aborted = false;
+
         try
         {
             // Give spectators a moment to join the SignalR group before the first move.
@@ -81,21 +87,48 @@ public sealed class SelfPlayCoordinator(
 
             while (IsLive(gameState, cancellationToken))
             {
-                await orchestrator.PlayAsync(gameState);
+                await PlayMoveWithTimeoutAsync(gameState, cancellationToken);
                 await Task.Delay(_selfPlayMoveDelay, cancellationToken);
             }
         }
-        catch (OperationCanceledException) { /* service shutting down */ }
+        catch (OperationCanceledException) { aborted = true; /* service shutting down */ }
+        catch (TimeoutException)
+        {
+            aborted = true;
+            Console.WriteLine($"Self-play game {gameState.GameId} aborted: a move took over " +
+                              $"{_moveTimeout.TotalSeconds:n0}s (likely a frozen engine).");
+        }
         catch (Exception ex)
         {
+            aborted = true;
             Console.WriteLine($"Self-play game {gameState.GameId} stopped: {ex.Message}");
         }
 
         ApplyLearning(gameState);
 
-        // Leave the finished game in place briefly so spectators can see the result.
+        // A clean finish lingers briefly so spectators see the result; an aborted/hung game is
+        // torn down immediately so its engines (and any frozen Stockfish process) are released.
         if (gameStore.Contains(gameState.GameId))
-            gameStore.ScheduleRemove(gameState.GameId, _selfPlayResultTimeout);
+            gameStore.ScheduleRemove(gameState.GameId, aborted ? TimeSpan.Zero : _selfPlayResultTimeout);
+    }
+
+    /// <summary>
+    /// Plays one engine move, abandoning it if it exceeds <see cref="_moveTimeout"/> (throwing
+    /// <see cref="TimeoutException"/>). The abandoned move's eventual fault — it errors once the
+    /// game's engines are disposed — is observed so it isn't an unobserved task exception.
+    /// </summary>
+    private async Task PlayMoveWithTimeoutAsync(GameState gameState, CancellationToken cancellationToken)
+    {
+        var play = orchestrator.PlayAsync(gameState);
+        try
+        {
+            await play.WaitAsync(_moveTimeout, cancellationToken);
+        }
+        catch (TimeoutException)
+        {
+            _ = play.ContinueWith(static t => { _ = t.Exception; }, TaskScheduler.Default);
+            throw;
+        }
     }
 
     private bool IsLive(GameState gameState, CancellationToken cancellationToken) =>

JoshHeaps.Net/wwwroot/js/ChessScripts/Spectate.js

@@ -23,9 +23,40 @@ const Spectate = {
         }
 
         await this.refreshGames();
+        await this.loadAutoTrainCount();
         setInterval(() => this.refreshGames(), 5000);
     },
 
+    // Auto-training runs server-side; show its target count and let it be changed here.
+    async loadAutoTrainCount() {
+        const input = document.getElementById("autoTrainCount");
+        if (!input) return;
+
+        try {
+            const response = await fetch("/api/chess/autotrain");
+            const data = await response.json();
+            input.max = data.max;
+            // Don't clobber the value while the user is editing it.
+            if (document.activeElement !== input)
+                input.value = data.count;
+        } catch {
+            // Leave the control as-is if auto-training status can't be read.
+        }
+    },
+
+    async setAutoTrainCount() {
+        const input = document.getElementById("autoTrainCount");
+        const count = Math.max(0, parseInt(input.value, 10) || 0);
+
+        try {
+            const response = await fetch(`/api/chess/autotrain?count=${count}`, { method: "POST" });
+            const data = await response.json();
+            input.value = data.count;
+        } catch (err) {
+            console.error("❌ Could not set the auto-training game count.", err);
+        }
+    },
+
     async startCpuGame() {
         const params = new URLSearchParams({
             whiteEngine: document.getElementById("whiteEngine").value,