Improve AI difficulty scaling.

internal/ai/ai.go

@@ -149,6 +149,17 @@ func (b *Bot) pick(cands []candidate) candidate {
 	if len(cands) == 1 {
 		return cands[0]
 	}
+	// Below competent play, the bot sometimes ignores its evaluation entirely
+	// and plays a random legal move. Softmax temperature alone can't make a bot
+	// a genuine pushover: it still samples among sensibly-generated candidates
+	// and, on this game's shop/dice variance, that wins ~1 game in 5 even
+	// against expert play. Real, occasional mistakes — buying the wrong pet,
+	// passing with coins in hand, a scrambled battle order — are what let a
+	// competent human beat "easy" almost every time. The rate is zero at and
+	// above competent level, so medium and hard never blunder.
+	if p := blunderProb(b.level); p > 0 && rand.Float64() < p {
+		return cands[rand.IntN(len(cands))]
+	}
 	temp := 0.02 + 0.30*(1-b.level)
 	best := math.Inf(-1)
 	for _, c := range cands {
@@ -170,6 +181,24 @@ func (b *Bot) pick(cands []candidate) candidate {
 	return cands[len(cands)-1]
 }
 
+// competentLevel is the skill (on the [0,1] level scale) at which the bot is
+// meant to be an even match for a competent human: at or above it the bot
+// never throws a game on purpose. Difficulties are calibrated around it —
+// easy well below, medium at it, hard above.
+const competentLevel = 0.6
+
+// blunderProb is the chance pick discards its evaluation and plays a uniformly
+// random legal move. It is zero at competent level and above, and ramps up
+// steeply below, so only the easy tier makes real mistakes. blunderMax is the
+// rate at level 0 (the weakest possible bot).
+func blunderProb(level float64) float64 {
+	const blunderMax = 0.75
+	if level >= competentLevel {
+		return 0
+	}
+	return blunderMax * (competentLevel - level) / competentLevel
+}
+
 // budget returns the rollout counts for this difficulty: how many opponent
 // deck/order guesses to test against, and how many dice-randomized battle
 // simulations to run per guess. Fewer samples means noisier estimates, which

internal/ai/ai_test.go

@@ -254,6 +254,26 @@ func TestSimulateBattleIsPure(t *testing.T) {
 	}
 }
 
+// TestBlunderCurve pins the difficulty calibration's shape: only sub-competent
+// bots ever throw a game on purpose, and they do so more the weaker they are.
+// Medium (== competentLevel) and hard must never blunder, or they would not be
+// the even-match / favourite the difficulty design promises.
+func TestBlunderCurve(t *testing.T) {
+	if p := blunderProb(competentLevel); p != 0 {
+		t.Errorf("competent bot blunders with p=%.3f, want 0", p)
+	}
+	if p := blunderProb(1.0); p != 0 {
+		t.Errorf("hard bot blunders with p=%.3f, want 0", p)
+	}
+	weakest, easy := blunderProb(0), blunderProb(0.25)
+	if !(weakest > easy && easy > 0) {
+		t.Errorf("blunder rate must rise as level falls: level0=%.3f easy=%.3f", weakest, easy)
+	}
+	if weakest > 1 {
+		t.Errorf("blunder probability %.3f exceeds 1", weakest)
+	}
+}
+
 // TestDecideShopNeverSellsLastPet guards the invariant that the bot never
 // voluntarily turns its whole deck into food. In a hopeless late-game spot
 // (final round, a strong modeled opponent, so every rollout is a loss) the

internal/ai/winrate_test.go

@@ -0,0 +1,134 @@
+package ai
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/greyson/super-auto-pets-board-game/internal/game"
+)
+
+// playHeadToHead runs one full bot-vs-bot game and returns the winning seat
+// (0 or 1), or -1 for a tie on trophies. level0/level1 set each seat's skill.
+func playHeadToHead(t *testing.T, pack string, level0, level1 float64) int {
+	t.Helper()
+	g := game.New()
+	pa, _ := g.AddBot("Bot A", level0)
+	pb, _ := g.AddBot("Bot B", level1)
+	if err := g.SetPack(pack); err != nil {
+		t.Fatalf("SetPack: %v", err)
+	}
+	if err := g.StartGame(); err != nil {
+		t.Fatalf("StartGame: %v", err)
+	}
+	bots := map[string]*Bot{pa.ID: New(level0), pb.ID: New(level1)}
+	mems := map[string]*Memory{pa.ID: {}, pb.ID: {}}
+	observe := func() {
+		for _, p := range g.Players {
+			v := g.ViewFor(p.ID)
+			Observe(&v, mems[p.ID])
+		}
+	}
+	observe()
+	for steps := 0; g.Phase != game.PhaseGameOver; steps++ {
+		if steps > 6000 {
+			t.Fatalf("stuck phase %s round %d", g.Phase, g.Round)
+		}
+		acted := false
+		for _, p := range g.Players {
+			v := g.ViewFor(p.ID)
+			if !Pending(&v) {
+				continue
+			}
+			act := bots[p.ID].Act(&v, mems[p.ID])
+			if act == nil {
+				t.Fatalf("bot %s owes action, got none, phase %s", p.Name, g.Phase)
+			}
+			if err := applyAction(g, p.ID, act); err != nil {
+				t.Fatalf("illegal %q: %v", act.Type, err)
+			}
+			observe()
+			acted = true
+			break
+		}
+		if !acted {
+			t.Fatalf("no bot owes action, phase %s", g.Phase)
+		}
+	}
+	switch t0, t1 := g.Players[0].Trophies, g.Players[1].Trophies; {
+	case t0 > t1:
+		return 0
+	case t1 > t0:
+		return 1
+	default:
+		return -1
+	}
+}
+
+// matchup plays gamesEach games with A on seat 0 and gamesEach with A on seat 1
+// (cancelling any first-move advantage) and reports A's win rate as a fraction.
+func matchup(t *testing.T, pack string, levelA, levelB float64, gamesEach int) float64 {
+	wins, total := 0, 0
+	for range gamesEach {
+		if playHeadToHead(t, pack, levelA, levelB) == 0 {
+			wins++
+		}
+		total++
+	}
+	for range gamesEach {
+		if playHeadToHead(t, pack, levelB, levelA) == 1 {
+			wins++
+		}
+		total++
+	}
+	return float64(wins) / float64(total)
+}
+
+// TestDiagWinRateCurve documents and guards the difficulty calibration: it
+// plays each difficulty against a competent player (a bot at competentLevel)
+// and checks the win rates match the design goals — easy is a near-certain
+// loss for the difficulty, medium is a coin flip, and hard is a strong
+// favourite. It is slow (hundreds of full games) and only runs when
+// AI_CALIBRATION is set in the environment:
+//
+//	AI_CALIBRATION=1 go test ./internal/ai/ -run TestDiagWinRateCurve -v
+//
+// Reference numbers from the calibration run (120 games per matchup):
+//
+//	competent vs easy   (0.60 vs 0.25): competent wins 94%
+//	competent vs medium (0.60 vs 0.60): 50% by construction
+//	hard vs competent   (1.00 vs 0.60): hard wins 78%  (competent wins ~17%)
+func TestDiagWinRateCurve(t *testing.T) {
+	if os.Getenv("AI_CALIBRATION") == "" {
+		t.Skip("set AI_CALIBRATION=1 to run the (slow) difficulty calibration")
+	}
+	const (
+		gamesEach = 60 // 120 games per matchup
+		easy      = 0.25
+		medium    = 0.60
+		hard      = 1.00
+	)
+
+	easyVsCompetent := matchup(t, game.DefaultPack, competentLevel, easy, gamesEach)
+	hardVsCompetent := matchup(t, game.DefaultPack, hard, competentLevel, gamesEach)
+	fmt.Printf("competent beats easy: %.0f%%\n", 100*easyVsCompetent)
+	fmt.Printf("hard beats competent: %.0f%% (competent wins %.0f%%)\n",
+		100*hardVsCompetent, 100*(1-hardVsCompetent))
+
+	// Easy must be a near-certain loss for a competent player to face — the
+	// design goal is "always wins", i.e. only the game's shop/dice variance
+	// should ever hand easy a game. Loose bound to absorb sampling noise.
+	if easyVsCompetent < 0.85 {
+		t.Errorf("competent beats easy only %.0f%%, want ~95%% (easy not weak enough)", 100*easyVsCompetent)
+	}
+	// Medium is competentLevel itself, so it is an even match by construction;
+	// assert the identity holds rather than re-measuring a trivial 50%.
+	if competentLevel != medium {
+		t.Errorf("medium level %.2f != competentLevel %.2f; medium must be the even-match anchor", medium, competentLevel)
+	}
+	// Hard must be a clear favourite over a competent player — the design goal
+	// is that the player wins only 10-25%, i.e. hard wins 75-90%.
+	if hardVsCompetent < 0.70 {
+		t.Errorf("hard beats competent only %.0f%%, want 75-90%% (hard not strong enough)", 100*hardVsCompetent)
+	}
+}