package ai import ( "fmt" "os" "testing" "github.com/greyson/super-auto-pets-board-game/internal/game" ) // playHeadToHead runs one full bot-vs-bot game and returns the winning seat // (0 or 1), or -1 for a tie on trophies. level0/level1 set each seat's skill. func playHeadToHead(t *testing.T, pack string, level0, level1 float64) int { t.Helper() g := game.New() pa, _ := g.AddBot("Bot A", level0) pb, _ := g.AddBot("Bot B", level1) if err := g.SetPack(pack); err != nil { t.Fatalf("SetPack: %v", err) } if err := g.StartGame(); err != nil { t.Fatalf("StartGame: %v", err) } bots := map[string]*Bot{pa.ID: New(level0), pb.ID: New(level1)} mems := map[string]*Memory{pa.ID: {}, pb.ID: {}} observe := func() { for _, p := range g.Players { v := g.ViewFor(p.ID) Observe(&v, mems[p.ID]) } } observe() for steps := 0; g.Phase != game.PhaseGameOver; steps++ { if steps > 6000 { t.Fatalf("stuck phase %s round %d", g.Phase, g.Round) } acted := false for _, p := range g.Players { v := g.ViewFor(p.ID) if !Pending(&v) { continue } act := bots[p.ID].Act(&v, mems[p.ID]) if act == nil { t.Fatalf("bot %s owes action, got none, phase %s", p.Name, g.Phase) } if err := applyAction(g, p.ID, act); err != nil { t.Fatalf("illegal %q: %v", act.Type, err) } observe() acted = true break } if !acted { t.Fatalf("no bot owes action, phase %s", g.Phase) } } switch t0, t1 := g.Players[0].Trophies, g.Players[1].Trophies; { case t0 > t1: return 0 case t1 > t0: return 1 default: return -1 } } // matchup plays gamesEach games with A on seat 0 and gamesEach with A on seat 1 // (cancelling any first-move advantage) and reports A's win rate as a fraction. func matchup(t *testing.T, pack string, levelA, levelB float64, gamesEach int) float64 { wins, total := 0, 0 for range gamesEach { if playHeadToHead(t, pack, levelA, levelB) == 0 { wins++ } total++ } for range gamesEach { if playHeadToHead(t, pack, levelB, levelA) == 1 { wins++ } total++ } return float64(wins) / float64(total) } // TestDiagWinRateCurve documents and guards the difficulty calibration: it // plays each difficulty against a competent player (a bot at competentLevel) // and checks the win rates match the design goals — easy is a near-certain // loss for the difficulty, medium is a coin flip, and hard is a strong // favourite. It is slow (hundreds of full games) and only runs when // AI_CALIBRATION is set in the environment: // // AI_CALIBRATION=1 go test ./internal/ai/ -run TestDiagWinRateCurve -v // // Reference numbers from the calibration run (120 games per matchup): // // competent vs easy (0.60 vs 0.25): competent wins 94% // competent vs medium (0.60 vs 0.60): 50% by construction // hard vs competent (1.00 vs 0.60): hard wins 78% (competent wins ~17%) func TestDiagWinRateCurve(t *testing.T) { if os.Getenv("AI_CALIBRATION") == "" { t.Skip("set AI_CALIBRATION=1 to run the (slow) difficulty calibration") } const ( gamesEach = 60 // 120 games per matchup easy = 0.25 medium = 0.60 hard = 1.00 ) easyVsCompetent := matchup(t, game.DefaultPack, competentLevel, easy, gamesEach) hardVsCompetent := matchup(t, game.DefaultPack, hard, competentLevel, gamesEach) fmt.Printf("competent beats easy: %.0f%%\n", 100*easyVsCompetent) fmt.Printf("hard beats competent: %.0f%% (competent wins %.0f%%)\n", 100*hardVsCompetent, 100*(1-hardVsCompetent)) // Easy must be a near-certain loss for a competent player to face — the // design goal is "always wins", i.e. only the game's shop/dice variance // should ever hand easy a game. Loose bound to absorb sampling noise. if easyVsCompetent < 0.85 { t.Errorf("competent beats easy only %.0f%%, want ~95%% (easy not weak enough)", 100*easyVsCompetent) } // Medium is competentLevel itself, so it is an even match by construction; // assert the identity holds rather than re-measuring a trivial 50%. if competentLevel != medium { t.Errorf("medium level %.2f != competentLevel %.2f; medium must be the even-match anchor", medium, competentLevel) } // Hard must be a clear favourite over a competent player — the design goal // is that the player wins only 10-25%, i.e. hard wins 75-90%. if hardVsCompetent < 0.70 { t.Errorf("hard beats competent only %.0f%%, want 75-90%% (hard not strong enough)", 100*hardVsCompetent) } }