#!/usr/bin/env python3
"""Replay the original ABAP-Bench rubric scores from the frozen public archive."""

import argparse
from collections import defaultdict
import hashlib
import json
from pathlib import Path
import subprocess
import sys


RELEASE = "2037ebb0ba325ddd9d68a000d7485f1375d857a8"
TRACKED_INPUTS = ("src", "data", "results/v4.0")


def main() -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("repository", type=Path, help="ABAP-Bench checkout at v1.0.0")
    args = parser.parse_args()
    root = args.repository.resolve()
    revision = subprocess.check_output(
        ["git", "-C", str(root), "rev-parse", "HEAD"], text=True,
    ).strip()
    if revision != RELEASE:
        parser.error(f"checkout {RELEASE} (v1.0.0); found {revision}")
    result = subprocess.run(
        ["git", "-C", str(root), "status", "--porcelain", "--", *TRACKED_INPUTS],
        check=True, capture_output=True, text=True,
    )
    if result.stdout.strip():
        parser.error("the scorer, task set and saved answers must match the release")

    sys.dont_write_bytecode = True
    sys.path.insert(0, str(root))
    from src.evaluate_v2 import ABAPBenchScorerV2

    scorer = ABAPBenchScorerV2()
    task_file = root / "data/tasks.jsonl"
    tasks = [json.loads(line) for line in task_file.read_text(encoding="utf-8").splitlines() if line.strip()]
    expected = {task["task_id"]: (task["dimension"], task["max_score"]) for task in tasks}
    denominator = sum(maximum for _, maximum in expected.values())
    archives = sorted((root / "results/v4.0").glob("*.json"))
    if len(tasks) != len(expected) or len(expected) != 60 or denominator != 1200 or len(archives) != 9:
        parser.error("unexpected task or archive count in the release")

    print(f"Commit {revision} · Python {sys.version.split()[0]}")
    print(f"Tasks {len(expected)} · Maximum {denominator} · SHA-256 {hashlib.sha256(task_file.read_bytes()).hexdigest()}")
    print("Model | Scored rows | API errors | Saved raw | Replayed raw | Different rows")
    total_rows = total_differences = 0
    notes = []
    for path in archives:
        data = json.loads(path.read_text(encoding="utf-8"))
        rows = data["results"]
        actual = {row["task_id"]: (row["dimension"], row["max_score"]) for row in rows}
        if actual != expected or len(rows) != len(expected):
            parser.error(f"{path.name}: incomplete, duplicate or inconsistent task rows")

        by_dimension = defaultdict(int)
        observed = []
        for row in rows:
            by_dimension[row["dimension"]] += row["score"]
            replay = 0 if row["error"] else scorer.score(row["task_id"], row["response"])[0]
            observed.append(replay)
        saved = sum(row["score"] for row in rows)
        differences = sum(row["score"] != replay for row, replay in zip(rows, observed))
        total_rows += len(rows)
        total_differences += differences
        model = data["model"]["id"]
        errors = sum(bool(row["error"]) for row in rows)
        print(f"{model} | {len(rows)} | {errors} | {saved} | {sum(observed)} | {differences}")

        stored = data["summary"]
        for dim, score in by_dimension.items():
            summary_score = stored["dimensions"][dim]["raw_score"]
            if score != summary_score:
                notes.append(f"{model} {dim}: rows {score}, summary {summary_score}")
        tokens = sum(row["tokens_used"] for row in rows)
        seconds = round(sum(row["latency_seconds"] for row in rows), 2)
        if tokens != stored["total_tokens"]:
            notes.append(f"{model} tokens: rows {tokens:,}, summary {stored['total_tokens']:,}")
        if abs(seconds - stored["total_latency"]) > 0.5:
            notes.append(f"{model} summed request latency (s): rows {seconds:,.2f}, summary {stored['total_latency']:,.2f}")

    print(f"\n{total_rows} saved task scores replayed; {total_differences} differences.")
    if notes:
        print("\nDifferences between archived task rows and the file's summary fields:")
        print("\n".join(notes))
    return int(total_differences != 0)


if __name__ == "__main__":
    raise SystemExit(main())
