Merge pull request 'Learn fractional win-rate advantage instead of unbounded plus/minus' (#9) from feature/fractional-learned-weights into master

native/chess_engine/include/chess_engine.h

@@ -103,9 +103,13 @@ CHESS_API void          CHESS_CALL trainer_destroy(TrainerHandle trainer);
 /* Record one played position (post-move FEN) into the accumulator. */
 CHESS_API void CHESS_CALL trainer_record(TrainerHandle trainer, const char* fen);
 
-/* Apply a finished game's outcome to the global weights and save: rewards the winner's
+/* Apply a finished game's outcome to the global weights and save. Each touched cell (a piece
- * occupied squares / features and punishes the loser's, scaled by `weight` (e.g. 0.5 for a
+ * on a square, or a feature) tracks a presence-weighted running average of the game's
- * material-imbalance draw). winner: 0 = white, 1 = black. */
+ * "advantage" — the game's outcome (scaled by `weight`, e.g. 0.5 for a material-imbalance
+ * draw) minus how much of that outcome the winner's average material edge already explains —
+ * so mg/eg/featW end up as a fractional win-rate (percent of games won with that piece there,
+ * minus percent lost), not an unbounded plus/minus tally, and a piece doesn't get credit
+ * merely for existing on the winning side of a material blowout. winner: 0 = white, 1 = black. */
 CHESS_API void CHESS_CALL trainer_apply(TrainerHandle trainer, int winner, double weight);
 
 #ifdef __cplusplus

native/chess_engine/src/chess_engine.cpp

@@ -26,6 +26,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <fstream>
+#include <iomanip>
 #include <mutex>
 #include <new>
 #include <string>
@@ -115,11 +116,25 @@ struct ChessEngine {
 /* The process-global learned weights: the single source of truth, loaded from disk once and
  * updated in place by training. Engine handles snapshot it at creation; the visualization
  * snapshots it on demand. Guarded by g_weightsMutex for updates/saves (eval reads its own
- * per-handle copy, so it never touches this concurrently). */
+ * per-handle copy, so it never touches this concurrently).
+ *
+ * mg/eg/featW are DERIVED, not accumulated: each is always exactly OUT_SCALE * (score/visits)
+ * for its cell (see trainer_apply), recomputed whenever that cell's stats change. visits is
+ * the presence-weighted count of games that touched the cell (how much to trust the average);
+ * score is the presence-weighted sum of each game's material-adjusted advantage (see
+ * trainer_apply) — so score/visits is literally "fraction of (presence-weighted) games won
+ * with this piece here, minus fraction lost," bounded to [-1,1] by construction. */
 struct LearnedWeights {
     int mg[chess::PIECE_TYPE_NB][64] = {};
     int eg[chess::PIECE_TYPE_NB][64] = {};
     int featW[FEATURE_NB] = {};
+
+    double mgVisits[chess::PIECE_TYPE_NB][64] = {};
+    double mgScore [chess::PIECE_TYPE_NB][64] = {};
+    double egVisits[chess::PIECE_TYPE_NB][64] = {};
+    double egScore [chess::PIECE_TYPE_NB][64] = {};
+    double featVisits[FEATURE_NB] = {};
+    double featScore [FEATURE_NB] = {};
 };
 
 static LearnedWeights g_weights;
@@ -127,13 +142,15 @@ static std::mutex     g_weightsMutex;
 static std::string    g_weightsPath;
 
 /* Per-game training accumulator (one per learned CPU-vs-CPU game). Records, per ply, where
- * each side's pieces sat (split into midgame/endgame by phase) and each side's feature
+ * each side's pieces sat (split into midgame/endgame by phase), each side's feature
- * activations; trainer_apply turns the totals into weight nudges. Squares are white-relative
+ * activations, and the white-minus-black material balance; trainer_apply turns the totals
+ * into a presence-weighted update to the running per-cell averages. Squares are white-relative
  * (black indexes sq ^ 56), so a side's tally lines up with the shared white-relative table. */
 struct Trainer {
     double mgOcc[chess::COLOR_NB][chess::PIECE_TYPE_NB][64] = {};
     double egOcc[chess::COLOR_NB][chess::PIECE_TYPE_NB][64] = {};
     double featAcc[chess::COLOR_NB][FEATURE_NB] = {};
+    double matDiffAcc = 0.0;      /* sum over plies of (white material - black material), cp */
     int    plies = 0;
 };
 
@@ -172,9 +189,12 @@ static int parse_variant(const char* options) {
 }
 
 /* On-disk format: 6*64 mg ints (PAWN..KING, squares 0..63), then 6*64 eg ints, then
- * FEATURE_NB feature ints, whitespace-separated. A missing file or short read leaves the
+ * FEATURE_NB feature ints, then the backing visits/score doubles for each (mgVisits, mgScore,
- * rest neutral (0), so an absent weights file just means "train from a blank slate".
+ * egVisits, egScore, featVisits, featScore) in the same shapes, all whitespace-separated. A
- * Caller holds g_weightsMutex. */
+ * missing file or short read leaves the rest neutral (0), so an absent weights file just means
+ * "train from a blank slate" — and a file written by the pre-fractional-learning format (which
+ * ends after the feature ints) loads its mg/eg/featW fine and simply starts every cell's
+ * visits/score history fresh from the next game that touches it. Caller holds g_weightsMutex. */
 static void load_global_weights(const char* path) {
     g_weights = LearnedWeights{};                    /* reset to neutral before loading */
 
@@ -190,9 +210,29 @@ static void load_global_weights(const char* path) {
             if (!(f >> g_weights.eg[pt][sq])) return;
     for (int i = 0; i < FEATURE_NB; ++i)
         if (!(f >> g_weights.featW[i])) return;
+
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq)
+            if (!(f >> g_weights.mgVisits[pt][sq])) return;
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq)
+            if (!(f >> g_weights.mgScore[pt][sq])) return;
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq)
+            if (!(f >> g_weights.egVisits[pt][sq])) return;
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq)
+            if (!(f >> g_weights.egScore[pt][sq])) return;
+    for (int i = 0; i < FEATURE_NB; ++i)
+        if (!(f >> g_weights.featVisits[i])) return;
+    for (int i = 0; i < FEATURE_NB; ++i)
+        if (!(f >> g_weights.featScore[i])) return;
 }
 
-/* Persist g_weights to g_weightsPath in the format load_global_weights reads. Caller holds the lock. */
+/* Persist g_weights to g_weightsPath in the format load_global_weights reads. Full round-trip
+ * precision on the doubles (visits/score are running sums that keep accumulating across many
+ * games, so truncating them would drift the derived mg/eg/featW over time). Caller holds the
+ * lock. */
 static void save_global_weights() {
     if (g_weightsPath.empty()) return;
     std::ofstream f(g_weightsPath);
@@ -203,6 +243,18 @@ static void save_global_weights() {
     for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
         for (int sq = 0; sq < 64; ++sq) f << g_weights.eg[pt][sq] << (sq == 63 ? '\n' : ' ');
     for (int i = 0; i < FEATURE_NB; ++i) f << g_weights.featW[i] << (i == FEATURE_NB - 1 ? '\n' : ' ');
+
+    f << std::setprecision(17);
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq) f << g_weights.mgVisits[pt][sq] << (sq == 63 ? '\n' : ' ');
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq) f << g_weights.mgScore[pt][sq] << (sq == 63 ? '\n' : ' ');
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq) f << g_weights.egVisits[pt][sq] << (sq == 63 ? '\n' : ' ');
+    for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
+        for (int sq = 0; sq < 64; ++sq) f << g_weights.egScore[pt][sq] << (sq == 63 ? '\n' : ' ');
+    for (int i = 0; i < FEATURE_NB; ++i) f << g_weights.featVisits[i] << (i == FEATURE_NB - 1 ? '\n' : ' ');
+    for (int i = 0; i < FEATURE_NB; ++i) f << g_weights.featScore[i] << (i == FEATURE_NB - 1 ? '\n' : ' ');
 }
 
 /* Maps the 1..20 difficulty to a search depth. Kept modest: the search has no
@@ -382,13 +434,18 @@ static int evaluate(const chess::Position& pos) {
  * compute_features() is the single source of feature activations, used by BOTH the eval here
  * and the trainer, so the two can never disagree. Constants below are the only tunables. */
 
-/* Per-game-outcome learning rates and clamps. Squares accumulate occupancy (plies on a
+/* Per-game-outcome learning constants. Squares and features both learn a presence/activation-
- * square, summed); features accumulate normalized per-ply activation (averaged, divided by a
+ * weighted AVERAGE advantage in [-1,1] (see trainer_apply) — OUT_SCALE just converts that
- * nominal scale so high-magnitude mobility doesn't dwarf the small pawn-structure terms). */
+ * average into a centipawn-ish int for the eval/table to add directly. FEAT_SCALE normalizes
-static constexpr double SQUARE_LR  = 0.5;
+ * each feature's raw per-ply activation to a comparable magnitude before it's used as a weight
-static constexpr int    SQ_CLAMP   = 250;
+ * (so high-count mobility doesn't dwarf the small pawn-structure terms), same as before.
-static constexpr double FEAT_LR    = 2.0;
+ * MATERIAL_ELO_SCALE is the classic centipawn-to-win-probability curve
-static constexpr int    FEAT_CLAMP = 500;
+ * (expected = 1/(1+10^(-cp/400))) used to compute how much of a result material already
+ * explains, so training credits placement/features only for the part of the result that
+ * material didn't already predict. */
+static constexpr int    SQ_OUT_SCALE      = 250;
+static constexpr int    FEAT_OUT_SCALE    = 500;
+static constexpr double MATERIAL_ELO_SCALE = 400.0;
 static constexpr double FEAT_SCALE[FEATURE_NB] = { 4, 6, 8, 14, 2, 1, 1, 2 };
 
 /* Game phase in [0,1] from remaining non-pawn material (PeSTO weights N=B=1, R=2, Q=4; max
@@ -823,7 +880,9 @@ CHESS_API void CHESS_CALL trainer_record(TrainerHandle t, const char* fen) {
     chess::Position pos = chess::Position::from_fen(fen);
     double phase = game_phase(pos);
 
-    /* Per-square occupancy, split into midgame/endgame by phase, white-relative. */
+    /* Per-square occupancy, split into midgame/endgame by phase, white-relative; and the
+     * white-minus-black material balance at this ply, used at trainer_apply time to strip
+     * "material already explains this" out of the training signal. */
     chess::Bitboard occ = pos.pieces();
     while (occ) {
         chess::Square s  = chess::pop_lsb(occ);
@@ -833,6 +892,7 @@ CHESS_API void CHESS_CALL trainer_record(TrainerHandle t, const char* fen) {
         int relSq = (c == chess::WHITE) ? int(s) : (int(s) ^ 56);
         t->mgOcc[c][pt][relSq] += (1.0 - phase);
         t->egOcc[c][pt][relSq] += phase;
+        t->matDiffAcc += (c == chess::WHITE ? 1 : -1) * piece_value(pt);
     }
 
     /* Per-side feature activations. */
@@ -847,34 +907,66 @@ CHESS_API void CHESS_CALL trainer_record(TrainerHandle t, const char* fen) {
     t->plies++;
 }
 
+/* Applies a finished game's outcome as a presence-weighted running average per cell, instead
+ * of an unbounded incremental nudge:
+ *
+ *   score[cell] / visits[cell]  ==  "fraction of (presence-weighted) games won with this piece
+ *                                    on this square, minus fraction lost"
+ *
+ * `weight` is the caller's confidence in the result (1.0 for a checkmate, 0.5 for a
+ * material-imbalance draw, as before). On top of that, the update is scaled by how much of the
+ * result ISN'T already explained by material: `expected` is the win probability the winner's
+ * average material edge over the game implies (the same centipawn->win-probability curve
+ * engines use for Elo estimates); `1 - expected` is the "surprise" left over once material is
+ * accounted for. A win where the winner was already up a rook the whole game is unsurprising
+ * (expected ~= 1, ~no update) — the eval already knows material matters via piece_value(), so
+ * square/feature tables shouldn't get credit for "a piece existed"; a win pulled off (or a draw
+ * held) while materially behind is highly informative and gets a correspondingly large update.
+ * This same magnitude applies to BOTH the winner's reward and the loser's punishment: by the
+ * symmetry of win probabilities (P(loser wins) = 1 - P(winner wins)), the loser's own advantage
+ * — actual(0) minus their expected win probability — has exactly the same magnitude, `1 -
+ * expected`, just negative. */
 CHESS_API void CHESS_CALL trainer_apply(TrainerHandle t, int winner, double weight) {
-    if (!t) return;
+    if (!t || t->plies <= 0) return;
+
+    double avgMaterialDiff = t->matDiffAcc / t->plies;                        /* white - black, cp */
+    double winnerMaterialDiff = (winner == chess::WHITE) ? avgMaterialDiff : -avgMaterialDiff;
+    double expected = 1.0 / (1.0 + std::pow(10.0, -winnerMaterialDiff / MATERIAL_ELO_SCALE));
+    double advantageMag = weight * (1.0 - expected);
 
     std::lock_guard<std::mutex> lock(g_weightsMutex);
 
-    /* pass 0 = winner (reward, +1); pass 1 = loser (punish, -1). */
+    /* pass 0 = winner (positive advantage); pass 1 = loser (negative advantage). */
     for (int pass = 0; pass < 2; ++pass) {
         chess::Color side = chess::Color((pass == 0 ? winner : (winner ^ 1)) & 1);
-        int sign = pass == 0 ? 1 : -1;
+        double advantage = pass == 0 ? advantageMag : -advantageMag;
 
         for (int pt = chess::PAWN; pt <= chess::KING; ++pt)
             for (int sq = 0; sq < 64; ++sq) {
-                if (t->mgOcc[side][pt][sq] != 0.0) {
+                double mgFrac = t->mgOcc[side][pt][sq] / t->plies;   /* fraction of the game spent here, [0,1] */
-                    int d = sign * int(std::lround(SQUARE_LR * t->mgOcc[side][pt][sq] * weight));
+                if (mgFrac != 0.0) {
-                    g_weights.mg[pt][sq] = std::clamp(g_weights.mg[pt][sq] + d, -SQ_CLAMP, SQ_CLAMP);
+                    g_weights.mgVisits[pt][sq] += mgFrac;
+                    g_weights.mgScore [pt][sq] += mgFrac * advantage;
+                    g_weights.mg[pt][sq] = int(std::lround(
+                        SQ_OUT_SCALE * g_weights.mgScore[pt][sq] / g_weights.mgVisits[pt][sq]));
                 }
-                if (t->egOcc[side][pt][sq] != 0.0) {
+                double egFrac = t->egOcc[side][pt][sq] / t->plies;
-                    int d = sign * int(std::lround(SQUARE_LR * t->egOcc[side][pt][sq] * weight));
+                if (egFrac != 0.0) {
-                    g_weights.eg[pt][sq] = std::clamp(g_weights.eg[pt][sq] + d, -SQ_CLAMP, SQ_CLAMP);
+                    g_weights.egVisits[pt][sq] += egFrac;
+                    g_weights.egScore [pt][sq] += egFrac * advantage;
+                    g_weights.eg[pt][sq] = int(std::lround(
+                        SQ_OUT_SCALE * g_weights.egScore[pt][sq] / g_weights.egVisits[pt][sq]));
                 }
             }
 
-        if (t->plies > 0)
+        for (int i = 0; i < FEATURE_NB; ++i) {
-            for (int i = 0; i < FEATURE_NB; ++i) {
+            double activation = (t->featAcc[side][i] / t->plies) / FEAT_SCALE[i];  /* per-ply average, normalized */
-                double avg = t->featAcc[side][i] / t->plies;        /* per-ply average, normalized */
+            if (activation == 0.0) continue;
-                int d = sign * int(std::lround(FEAT_LR * (avg / FEAT_SCALE[i]) * weight));
+            g_weights.featVisits[i] += std::abs(activation);
-                g_weights.featW[i] = std::clamp(g_weights.featW[i] + d, -FEAT_CLAMP, FEAT_CLAMP);
+            g_weights.featScore [i] += activation * advantage;
-            }
+            g_weights.featW[i] = int(std::lround(
+                FEAT_OUT_SCALE * g_weights.featScore[i] / g_weights.featVisits[i]));
+        }
     }
 
     save_global_weights();