"""score.py <runs.csv> [--capped-as-zero]: score the OSWorld hard-20 run from its published per-run metrics.

runs.csv has one row per run: task, tool, trial, reward (OSWorld's evaluator score; empty when the
evaluator never scored the run), subtype (Claude Code's end state; error_max_turns = hit the 100-turn
cap), wall_s, cost_usd, input_tokens, model_calls (distinct model responses in the run's transcript).

Default, as RULE.txt was written before the runs: the score is the evaluator's, so an unscored run is
left out and listed. --capped-as-zero: the later choice in DEVIATIONS.txt, a turn-capped run scores 0.
"""
import csv, statistics, sys
from collections import defaultdict

LABEL = {"computer-mcp-0112": "Ours: computer mcp 0.1.12", "cua-mcp": "Cua MCP"}
capped_as_zero = "--capped-as-zero" in sys.argv
runs, unscored = [], []
for r in csv.DictReader(open(sys.argv[1])):
    for field in ("wall_s", "cost_usd", "input_tokens", "model_calls"):
        if r[field] == "":
            sys.exit(f"missing {field} for {r['task']} {r['tool']} {r['trial']}")
    capped = r["subtype"] == "error_max_turns"
    if r["reward"] == "":
        if not (capped and capped_as_zero):
            unscored.append(r)
            continue
        r["reward"] = "0"
    runs.append(dict(task=r["task"], arm=r["tool"], reward=float(r["reward"]), cost=float(r["cost_usd"]),
                     wall=float(r["wall_s"]), tin=int(r["input_tokens"]), calls=int(r["model_calls"]), capped=capped))

by = defaultdict(list)
for r in runs:
    by[r["arm"]].append(r)
print(f"{'tool':28} {'pass':>9} {'rate':>6} {'$/run':>7} {'calls':>6} {'wall s':>7} {'tok in':>8} {'capped':>7}")
for arm, v in by.items():
    n = len(v)
    passed = sum(r["reward"] >= 1 for r in v)
    print(f"{LABEL.get(arm, arm):28} {passed:>4}/{n:<4} {passed / n:>6.0%} "
          f"{statistics.mean(r['cost'] for r in v):>7.3f} {statistics.mean(r['calls'] for r in v):>6.1f} "
          f"{statistics.mean(r['wall'] for r in v):>7.1f} {statistics.mean(r['tin'] for r in v):>8.0f} {sum(r['capped'] for r in v):>7}")
print("\nper task (passes out of runs scored, mean calls):")
for t in sorted({r["task"] for r in runs}):
    cells = []
    for arm in by:
        v = [r for r in by[arm] if r["task"] == t]
        cells.append(f"{LABEL.get(arm, arm).split(':')[0]:>4} {sum(r['reward'] >= 1 for r in v)}/{len(v)} {statistics.mean(r['calls'] for r in v):5.1f}c" if v else "-")
    print(f"  {t:34} " + "   ".join(cells))
if unscored:
    print("\nunscored:")
    for r in unscored:
        why = "hit the 100-turn cap" if r["subtype"] == "error_max_turns" else f"not evaluated ({r['subtype'] or 'no end state'})"
        print(f"  {r['task']} {r['tool']} {r['trial']}: {why}, {r['wall_s']} s, {r['model_calls']} calls, ${float(r['cost_usd']):.3f}")