super-auto-pets-board-game/internal/ai/winrate_test.go

package ai

import (
	"fmt"
	"os"
	"testing"

	"github.com/greyson/super-auto-pets-board-game/internal/game"
)

// playHeadToHead runs one full bot-vs-bot game and returns the winning seat
// (0 or 1), or -1 for a tie on trophies. level0/level1 set each seat's skill.
func playHeadToHead(t *testing.T, pack string, level0, level1 float64) int {
	t.Helper()
	g := game.New()
	pa, _ := g.AddBot("Bot A", level0)
	pb, _ := g.AddBot("Bot B", level1)
	if err := g.SetPacks([]string{pack}); err != nil {
		t.Fatalf("SetPacks: %v", err)
	}
	if err := g.StartGame(); err != nil {
		t.Fatalf("StartGame: %v", err)
	}
	bots := map[string]*Bot{pa.ID: New(level0), pb.ID: New(level1)}
	mems := map[string]*Memory{pa.ID: {}, pb.ID: {}}
	observe := func() {
		for _, p := range g.Players {
			v := g.ViewFor(p.ID)
			Observe(&v, mems[p.ID])
		}
	}
	observe()
	for steps := 0; g.Phase != game.PhaseGameOver; steps++ {
		if steps > 6000 {
			t.Fatalf("stuck phase %s round %d", g.Phase, g.Round)
		}
		acted := false
		for _, p := range g.Players {
			v := g.ViewFor(p.ID)
			if !Pending(&v) {
				continue
			}
			act := bots[p.ID].Act(&v, mems[p.ID])
			if act == nil {
				t.Fatalf("bot %s owes action, got none, phase %s", p.Name, g.Phase)
			}
			if err := applyAction(g, p.ID, act); err != nil {
				t.Fatalf("illegal %q: %v", act.Type, err)
			}
			observe()
			acted = true
			break
		}
		if !acted {
			t.Fatalf("no bot owes action, phase %s", g.Phase)
		}
	}
	switch t0, t1 := g.Players[0].Trophies, g.Players[1].Trophies; {
	case t0 > t1:
		return 0
	case t1 > t0:
		return 1
	default:
		return -1
	}
}

// matchup plays gamesEach games with A on seat 0 and gamesEach with A on seat 1
// (cancelling any first-move advantage) and reports A's win rate as a fraction.
func matchup(t *testing.T, pack string, levelA, levelB float64, gamesEach int) float64 {
	wins, total := 0, 0
	for range gamesEach {
		if playHeadToHead(t, pack, levelA, levelB) == 0 {
			wins++
		}
		total++
	}
	for range gamesEach {
		if playHeadToHead(t, pack, levelB, levelA) == 1 {
			wins++
		}
		total++
	}
	return float64(wins) / float64(total)
}

// TestDiagWinRateCurve documents and guards the difficulty calibration: it
// plays each difficulty against a competent player (a bot at competentLevel)
// and checks the win rates match the design goals — easy is a near-certain
// loss for the difficulty, medium is a coin flip, and hard is a strong
// favourite. It is slow (hundreds of full games) and only runs when
// AI_CALIBRATION is set in the environment:
//
//	AI_CALIBRATION=1 go test ./internal/ai/ -run TestDiagWinRateCurve -v
//
// Reference numbers from the calibration run (120 games per matchup):
//
//	competent vs easy   (0.60 vs 0.25): competent wins 94%
//	competent vs medium (0.60 vs 0.60): 50% by construction
//	hard vs competent   (1.00 vs 0.60): hard wins 78%  (competent wins ~17%)
func TestDiagWinRateCurve(t *testing.T) {
	if os.Getenv("AI_CALIBRATION") == "" {
		t.Skip("set AI_CALIBRATION=1 to run the (slow) difficulty calibration")
	}
	const (
		gamesEach = 60 // 120 games per matchup
		easy      = 0.25
		medium    = 0.60
		hard      = 1.00
	)

	easyVsCompetent := matchup(t, game.DefaultPack, competentLevel, easy, gamesEach)
	hardVsCompetent := matchup(t, game.DefaultPack, hard, competentLevel, gamesEach)
	fmt.Printf("competent beats easy: %.0f%%\n", 100*easyVsCompetent)
	fmt.Printf("hard beats competent: %.0f%% (competent wins %.0f%%)\n",
		100*hardVsCompetent, 100*(1-hardVsCompetent))

	// Easy must be a near-certain loss for a competent player to face — the
	// design goal is "always wins", i.e. only the game's shop/dice variance
	// should ever hand easy a game. Loose bound to absorb sampling noise.
	if easyVsCompetent < 0.85 {
		t.Errorf("competent beats easy only %.0f%%, want ~95%% (easy not weak enough)", 100*easyVsCompetent)
	}
	// Medium is competentLevel itself, so it is an even match by construction;
	// assert the identity holds rather than re-measuring a trivial 50%.
	if competentLevel != medium {
		t.Errorf("medium level %.2f != competentLevel %.2f; medium must be the even-match anchor", medium, competentLevel)
	}
	// Hard must be a clear favourite over a competent player — the design goal
	// is that the player wins only 10-25%, i.e. hard wins 75-90%.
	if hardVsCompetent < 0.70 {
		t.Errorf("hard beats competent only %.0f%%, want 75-90%% (hard not strong enough)", 100*hardVsCompetent)
	}
}