240 lines
8.8 KiB
Python
240 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Parse a Forge sim log into a readable report + JSON artifact.
|
|
|
|
python3 tools/forge_report.py <log> <json_out> <txt_out> [games] [seed] [deck_path] [ours_name]
|
|
|
|
Usually invoked by tools/run_forge.sh / batch_run.py / variant.py rather than
|
|
directly. `ours_name` is the Forge [metadata] Name= value for our seat
|
|
(DeckContext.forge_metadata_name(), from config.yaml) -- never a hardcoded
|
|
deck name. If omitted, it's derived from deck_path's deck directory.
|
|
"""
|
|
import sys, os, re, json, statistics as st
|
|
from datetime import datetime, timezone
|
|
from collections import Counter
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
OURS = "our deck"
|
|
|
|
|
|
def is_ours(name):
|
|
"""Match our seat even when an AI profile is in play.
|
|
|
|
Passing -a <profile> makes Forge rename the seat to "<name> [Combo]".
|
|
Exact matching would silently report a 0% win rate and an empty card
|
|
table, so match the bare name or name + [profile]. Opponent precons
|
|
legitimately contain brackets ("[C19] [2019]"), so this deliberately
|
|
anchors on OURS rather than stripping all bracket suffixes.
|
|
"""
|
|
return name == OURS or name.startswith(OURS + " [")
|
|
|
|
|
|
def deck_name_for(deck_path):
|
|
"""Best-effort: walk up from a decklist path to find the deck's config.yaml
|
|
and return its Forge metadata name. Falls back to None (caller keeps OURS)."""
|
|
d = os.path.dirname(os.path.abspath(deck_path))
|
|
for _ in range(6):
|
|
cfg = os.path.join(d, "config.yaml")
|
|
if os.path.exists(cfg):
|
|
from deckctx import DeckContext
|
|
return DeckContext(d).forge_metadata_name()
|
|
parent = os.path.dirname(d)
|
|
if parent == d:
|
|
break
|
|
d = parent
|
|
return None
|
|
|
|
|
|
def parse_match_result(log_path):
|
|
"""The 'Match Result:' line is authoritative for wins per player.
|
|
|
|
Individual 'Game Result' lines can be missing if output was truncated, so
|
|
prefer this tally when present.
|
|
"""
|
|
tally, total = {}, 0
|
|
for line in open(log_path, errors="ignore"):
|
|
if not line.startswith("Match Result:"):
|
|
continue
|
|
body = line[len("Match Result:"):].strip()
|
|
for m in re.finditer(r"(Ai\(\d+\)-.+?):\s*(\d+)(?=\s+Ai\(|\s*$)", body):
|
|
name = re.sub(r"^Ai\(\d+\)-", "", m.group(1)).strip()
|
|
tally[name] = int(m.group(2))
|
|
total = sum(tally.values())
|
|
return tally, total
|
|
|
|
|
|
def count_timeouts(log_path):
|
|
"""Games Forge abandoned as a draw for exceeding the -c clock.
|
|
|
|
A timeout is scored a Draw -- nobody wins -- so these games are unwinnable
|
|
by construction. They cluster in long grindy games, which is exactly what
|
|
an aristocrats deck is built to win, so they bias our win rate DOWN.
|
|
"""
|
|
n = 0
|
|
for line in open(log_path, errors="ignore"):
|
|
if "Stopping slow match as draw" in line:
|
|
n += 1
|
|
return n
|
|
|
|
|
|
def parse(log_path):
|
|
games, cur = [], {}
|
|
for line in open(log_path, errors="ignore"):
|
|
line = line.strip()
|
|
m = re.match(r"Game Outcome: Turn (\d+)", line)
|
|
if m:
|
|
cur = {"turns": int(m.group(1)), "winner": None, "losers": []}
|
|
continue
|
|
m = re.match(r"Game Outcome: (.+?) has (won|lost) because (.+)", line)
|
|
if m:
|
|
who, verdict, why = m.group(1), m.group(2), m.group(3)
|
|
name = re.sub(r"^Ai\(\d+\)-", "", who)
|
|
if verdict == "won":
|
|
cur["winner"] = name
|
|
cur["win_reason"] = why
|
|
else:
|
|
cur["losers"].append({"player": name, "reason": why})
|
|
continue
|
|
m = re.match(r"Game Result: Game (\d+) ended in (\d+) ms\. (.+?) has won", line)
|
|
if m:
|
|
cur["game"] = int(m.group(1))
|
|
cur["ms"] = int(m.group(2))
|
|
games.append(cur)
|
|
cur = {}
|
|
return games
|
|
|
|
|
|
def report(games, n_req, seed, log, jsonp, txtp, deck_path="decklist.mtg",
|
|
tally=None, tally_total=0, timeouts=0):
|
|
L, add = [], lambda s: L.append(s)
|
|
# prefer the authoritative Match Result tally over per-game lines
|
|
if tally and tally_total:
|
|
wins, n = Counter(tally), tally_total
|
|
else:
|
|
wins = Counter(g["winner"] for g in games if g.get("winner"))
|
|
n = len(games)
|
|
ours = sum(v for k, v in wins.items() if is_ours(k))
|
|
turns = [g["turns"] for g in games if g.get("turns")]
|
|
secs = [g["ms"] / 1000 for g in games if g.get("ms")]
|
|
|
|
add("=" * 68)
|
|
add("FORGE SIMULATION REPORT — 4-player Commander pod")
|
|
add("=" * 68)
|
|
add(f"games completed : {n} (requested {n_req}) seed: {seed}")
|
|
add(f"decklist : {deck_path} (full 100 cards)")
|
|
add(f"engine : Forge 2.0.14, full rules, AI vs AI")
|
|
add("")
|
|
|
|
add("WIN RATE")
|
|
add("-" * 68)
|
|
if n:
|
|
add(f" our deck : {ours}/{n} ({100*ours/n:.1f}%) "
|
|
f"[random baseline in a 4-pod = 25%]")
|
|
add("")
|
|
add(" all players:")
|
|
for name, w in wins.most_common():
|
|
tag = " <-- ours" if is_ours(name) else ""
|
|
bar = "#" * int(30 * w / n)
|
|
add(f" {name[:38]:<40} {w:>3} ({100*w/n:4.1f}%) {bar}{tag}")
|
|
add("")
|
|
|
|
add("TIMEOUT DRAWS")
|
|
add("-" * 68)
|
|
if timeouts:
|
|
add(f" {timeouts} game(s) abandoned as a draw for exceeding the clock "
|
|
f"({100*timeouts/max(n,1):.0f}% of games)")
|
|
add(" A draw means NOBODY wins -- these games are unwinnable by")
|
|
add(" construction, and they cluster in long grindy games, which is")
|
|
add(" exactly what this deck is built to win. Raise -c to fix.")
|
|
else:
|
|
add(" none -- no games hit the clock limit")
|
|
add("")
|
|
|
|
add("GAME LENGTH")
|
|
add("-" * 68)
|
|
if turns:
|
|
add(f" turns median {int(st.median(turns))} min {min(turns)} max {max(turns)}")
|
|
if secs:
|
|
add(f" seconds median {st.median(secs):.0f} total {sum(secs)/60:.1f} min")
|
|
add("")
|
|
|
|
add("HOW OUR DECK LOST")
|
|
add("-" * 68)
|
|
reasons = Counter()
|
|
for g in games:
|
|
for l in g.get("losers", []):
|
|
if is_ours(l["player"]):
|
|
reasons[l["reason"][:52]] += 1
|
|
if reasons:
|
|
for r, c in reasons.most_common():
|
|
add(f" {c:>3}x {r}")
|
|
else:
|
|
add(" (never lost)")
|
|
add("")
|
|
|
|
add("WHO BEAT US (winner, in games our deck lost)")
|
|
add("-" * 68)
|
|
beat = Counter()
|
|
for g in games:
|
|
w = g.get("winner")
|
|
if w and not is_ours(w) and any(is_ours(l["player"]) for l in g.get("losers", [])):
|
|
beat[w] += 1
|
|
if beat:
|
|
for name, c in beat.most_common():
|
|
add(f" {c:>3}x {name}")
|
|
else:
|
|
add(" (n/a)")
|
|
add("")
|
|
|
|
add("CAVEAT — READ THIS BEFORE TRUSTING THE WIN RATE")
|
|
add("-" * 68)
|
|
add(" Forge's AI is heuristic and, by its own documentation, WEAK AT COMBO")
|
|
add(" decks. Ours is a combo/aristocrats deck, so the AI systematically")
|
|
add(" under-plays it: it will miss sac-loop lines a human would see.")
|
|
add(" Read this as a FLOOR on the deck's power, and as proof the deck")
|
|
add(" functions and its cards interact — not as a power rating.")
|
|
add(" Opponents are stock precons, which are roughly bracket 2.")
|
|
add("")
|
|
add("=" * 68)
|
|
add(f"decklist : {deck_path}")
|
|
add(f"full log : {log} (every game, all Forge output)")
|
|
add(f"raw data : {jsonp}")
|
|
add(f"this report: {txtp}")
|
|
add("=" * 68)
|
|
return "\n".join(L)
|
|
|
|
|
|
def main():
|
|
global OURS
|
|
log, jsonp, txtp = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
n_req = sys.argv[4] if len(sys.argv) > 4 else "?"
|
|
seed = sys.argv[5] if len(sys.argv) > 5 else "?"
|
|
deck_path = sys.argv[6] if len(sys.argv) > 6 else "decklist.mtg"
|
|
ours_name = sys.argv[7] if len(sys.argv) > 7 else deck_name_for(deck_path)
|
|
if ours_name:
|
|
OURS = ours_name
|
|
games = parse(log)
|
|
tally, total = parse_match_result(log)
|
|
timeouts = count_timeouts(log)
|
|
txt = report(games, n_req, seed, log, jsonp, txtp, deck_path=deck_path,
|
|
tally=tally, tally_total=total, timeouts=timeouts)
|
|
wins = Counter(tally) if tally else Counter(g["winner"] for g in games if g.get("winner"))
|
|
json.dump({
|
|
"meta": {"generated": datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ"),
|
|
"engine": "Forge 2.0.14", "format": "Commander, 4 players",
|
|
"games_requested": n_req, "games_completed": len(games), "seed": seed,
|
|
"decklist_file": deck_path, "log_file": log,
|
|
"caveat": "Forge AI is weak at combo decks; treat win rate as a floor"},
|
|
"win_counts": dict(wins),
|
|
"timeout_draws": timeouts,
|
|
"our_win_rate": ((sum(v for k, v in wins.items() if is_ours(k)) / n_games)
|
|
if (n_games := (total or len(games))) else None),
|
|
"games": games,
|
|
}, open(jsonp, "w"), indent=1)
|
|
open(txtp, "w").write(txt + "\n")
|
|
print(txt)
|
|
print(f"\nsaved: {txtp}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|