"""Smarty Pants eval kit for Python 3.8+. Standard library only, nothing to install.

Pairs one pinned model build with a fresh sandbox VM and returns a receipt of the exact versions
used, so the same eval can be run again later and checked against the first run. Uses smartypants.py
(keep both files in the same folder). Receipts are the same format as eval-kit.mjs, so either kit can
check the other's. Docs: https://smartypants.network/docs/agents

    from smartypants import SmartyPants
    from eval_kit import EvalKit, compare_receipts
    sp = SmartyPants()
    cases = [
        {"id": "sum", "prompt": "What is 17 + 25? Reply with the number only.", "expect": "42"},
        {"id": "capital", "prompt": "Capital of France? One word.", "expect": "Paris"},
    ]
    run = EvalKit(sp, model="smartypants/qwen3.8-27b").evaluate(cases)
    print(run["passed"], "of", run["total"])
    json.dump(run["receipt"], open("receipt.json", "w"), indent=2)

    # Later, or on another computer: same model build, same settings, fresh VM.
    again = EvalKit.from_receipt(sp, run["receipt"]).evaluate(cases)
    print(compare_receipts(run["receipt"], again["receipt"]))

Grade in the sandbox instead: evaluate(cases, grader={"language": "python", "source": ...}). The kit
writes answers.jsonl ({"id","prompt","answer","expect"} per line) into a fresh VM, runs the grader there
and reads the LAST line of its output as JSON: {"passed": n, "total": n}. The VM has no network and is
destroyed afterwards. Workspaces are open to our own account first, for now.

"Pinned" means the receipt holds the model's build_id (exact weights, quantization and runtime build).
Every answer says which build made it; if the build changes, the kit stops with BUILD_CHANGED.
Temperature 0 and thinking off by default.
"""

import datetime
import hashlib
import json
import re
import time

from smartypants import SmartyPantsError

EVAL_KIT_VERSION = "1"
RECEIPT_KIND = "smartypants.eval_receipt"
PART_BYTES = 6 * 1024  # answers are uploaded in parts below the 8 KiB file limit

# Read inside the VM: OS, kernel, language versions and the hash of the session handler. One JSON line.
SANDBOX_PROBE = r'''h() { [ -r "$1" ] && sha256sum "$1" 2>/dev/null | cut -d' ' -f1; }
v() { command -v "$1" >/dev/null 2>&1 && "$@" 2>&1 | head -n 1 | tr -d '"\\' ; }
printf '{"os":"%s","kernel":"%s","machine":"%s","python":"%s","node":"%s","busybox":"%s","init_sha256":"%s","code_handler_sha256":"%s","session_handler_sha256":"%s"}\n' \
  "$( (. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME") | tr -d '"\\')" "$(uname -r)" "$(uname -m)" \
  "$(v python3 --version)" "$(v node --version)" "$(v busybox | sed 's/ multi-call.*//')" "$(h /init)" "$(h /fixture.sh)" "$(h /session.sh)"'''


def canonical_json(value):
    """JSON with keys sorted, no spaces: the same bytes as eval-kit.mjs (receipts hold no fractions)."""
    return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)


def sha256(text):
    return hashlib.sha256(text.encode("utf-8")).hexdigest()


def verify_receipt(receipt):
    """True when the receipt's digest matches its contents (it was not edited after the run)."""
    if not isinstance(receipt, dict) or receipt.get("kind") != RECEIPT_KIND or not isinstance(receipt.get("digest"), str):
        return False
    rest = {k: v for k, v in receipt.items() if k != "digest"}
    return receipt["digest"] == "sha256:" + sha256(canonical_json(rest))


def compare_receipts(a, b):
    """same_versions: same model build, settings, sandbox and grader. same_answers: every case gave
    the same answer text. differences: a plain-language list of what changed."""
    differences = []
    pick = lambda r: {k: (r.get("model") or {}).get(k) for k in ("id", "build_id", "upstream_model_id", "quantization")}
    pa, pb = pick(a), pick(b)
    for key in pa:
        if pa[key] != pb[key]:
            differences.append("model %s: %s -> %s" % (key, pa[key], pb[key]))
    sa, sb = a.get("settings") or {}, b.get("settings") or {}
    for key in sorted(set(sa) | set(sb)):
        if canonical_json(sa.get(key)) != canonical_json(sb.get(key)):
            differences.append("setting %s: %s -> %s" % (key, canonical_json(sa.get(key)), canonical_json(sb.get(key))))
    va, vb = (a.get("sandbox") or {}).get("versions"), (b.get("sandbox") or {}).get("versions")
    if va and vb:
        for key in sorted(set(va) | set(vb)):
            if va.get(key) != vb.get(key):
                differences.append("sandbox %s: %s -> %s" % (key, va.get(key), vb.get(key)))
    if bool(va) != bool(vb):
        differences.append("sandbox: %s -> %s" % ("used" if va else "not used", "used" if vb else "not used"))
    if (a.get("grader") or {}).get("source_sha256") != (b.get("grader") or {}).get("source_sha256"):
        differences.append("grader source changed")
    aa = {c["case_id"]: c["answer_sha256"] for c in a.get("calls") or []}
    ab = {c["case_id"]: c["answer_sha256"] for c in b.get("calls") or []}
    changed = [i for i in aa if i in ab and aa[i] != ab[i]]
    missing = [i for i in aa if i not in ab] + [i for i in ab if i not in aa]
    if changed:
        differences.append("answers changed: " + ", ".join(changed))
    if missing:
        differences.append("cases only in one run: " + ", ".join(missing))
    same_versions = not any(re.match(r"^(model|setting|sandbox|grader)", d) for d in differences)
    return {"same_versions": same_versions, "same_answers": not changed and not missing,
            "same_score": canonical_json(a.get("result")) == canonical_json(b.get("result")), "differences": differences}


def _now():
    t = datetime.datetime.now(datetime.timezone.utc)
    return t.strftime("%Y-%m-%dT%H:%M:%S.") + "%03dZ" % (t.microsecond // 1000)


def _normal(text):
    return re.sub(r"[\s.]+$", "", str(text if text is not None else "").strip().lower())


def score_case(case, answer):
    """Local scoring: case["expect"] (exact, ignoring case and a trailing full stop) or case["match"] (a regex)."""
    if case.get("match") is not None:
        pattern = case["match"]
        return bool(pattern.search(answer or "")) if hasattr(pattern, "search") else bool(re.search(pattern, answer or "", re.I))
    if case.get("expect") is not None:
        return _normal(answer) == _normal(case["expect"])
    return None


class _RecordedWorkspace:
    """The workspace, with every run() written into the receipt."""

    def __init__(self, ws, runs):
        self._ws, self._runs = ws, runs

    def __getattr__(self, name):
        return getattr(self._ws, name)

    def run(self, language, source, **options):
        out = self._ws.run(language, source, **options)
        self._runs.append({"language": language, "source_sha256": sha256(source), "exit_code": out.get("exitCode"),
                           "timed_out": out.get("timedOut"), "stdout_sha256": sha256(out["stdout"]), "stderr_sha256": sha256(out["stderr"])})
        return out


class EvalKit:
    def __init__(self, sp, model, build_id=None, temperature=0, max_tokens=512, thinking=False, system=None, retry_delays=(2, 5, 10)):
        """sp: a SmartyPants client. model: exact model id. build_id: the build to insist on (from an
        earlier receipt). Temperature 0, 512 tokens, thinking off unless you say otherwise."""
        if not model:
            raise SmartyPantsError("EvalKit needs an exact model id, for example smartypants/qwen3.8-27b (see sp.models()).", code="MODEL_REQUIRED")
        self.sp = sp
        self.settings = {"model": model, "temperature": temperature, "max_tokens": max_tokens, "thinking": thinking, "system": system}
        self.wanted_build_id = build_id
        # When every Mac with this build is busy (429) or none is online (503), wait and try again.
        self.retry_delays = tuple(retry_delays)
        self.reset()

    @classmethod
    def from_receipt(cls, sp, receipt, **options):
        """A kit that insists on the same model build and settings as an earlier receipt."""
        s = receipt.get("settings") or {}
        return cls(sp, receipt["model"]["id"], build_id=receipt["model"]["build_id"], temperature=s.get("temperature", 0),
                   max_tokens=s.get("max_tokens", 512), thinking=s.get("thinking", False), system=s.get("system"), **options)

    def reset(self):
        self.pinned = None
        self.calls = []
        self.sandbox_info = None
        self.grader_info = None
        self.result = None
        self.started_at = None
        self.finished_at = None

    def pin(self):
        """Look up the model's exact build in the live list. Fails with BUILD_CHANGED if it is not the build asked for."""
        if self.pinned:
            return self.pinned
        wanted = self.settings["model"]
        m = next((x for x in self.sp.models() if x["id"] == wanted or wanted in (x.get("aliases") or [])), None)
        if not m:
            raise SmartyPantsError("Model %s is not offered right now. See %s/v1/models." % (wanted, self.sp.base_url), code="MODEL_NOT_FOUND")
        if not m.get("build_id"):
            raise SmartyPantsError("Model %s does not report a build_id, so it cannot be pinned." % wanted, code="BUILD_UNKNOWN")
        if self.wanted_build_id and m["build_id"] != self.wanted_build_id:
            raise SmartyPantsError("Model %s now serves build %s, not %s. Results would not be comparable." % (wanted, m["build_id"], self.wanted_build_id),
                                   code="BUILD_CHANGED", body={"wanted": self.wanted_build_id, "live": m["build_id"]})
        self.pinned = {"id": m["id"], "build_id": m["build_id"], "upstream_model_id": m.get("upstream_model_id"),
                       "canonical_model_id": m.get("canonical_model_id"), "quantization": m.get("quantization"),
                       "tokenizer": (m.get("architecture") or {}).get("tokenizer"), "license": m.get("license"),
                       "context_length": m.get("context_length"), "pricing": m.get("pricing")}
        self.started_at = self.started_at or _now()
        return self.pinned

    def ask(self, prompt, case_id=None, system="__default__", **options):
        """One prompt to the pinned build. Records the call (hashes, build, Mac, tokens) in the receipt."""
        pin = self.pin()
        case_id = case_id or "call-%d" % (len(self.calls) + 1)
        system = self.settings["system"] if system == "__default__" else system
        messages = ([{"role": "system", "content": system}] if system else []) + [{"role": "user", "content": prompt}]
        started = time.time()
        body = {"model": pin["id"], "temperature": self.settings["temperature"], "max_tokens": self.settings["max_tokens"],
                "reasoning": {"enabled": self.settings["thinking"]}}
        body.update(options)
        body["messages"] = messages
        attempt = 0
        while True:
            try:
                reply = self.sp.request("POST", "/v1/chat/completions", body) or {}
                break
            except SmartyPantsError as error:
                if error.status not in (429, 503) or attempt >= len(self.retry_delays):
                    raise
                time.sleep(self.retry_delays[attempt])
                attempt += 1
        choice = (reply.get("choices") or [{}])[0]
        answer = (choice.get("message") or {}).get("content") or ""
        meta = reply.get("smartypants") or {}
        if meta.get("build_id") != pin["build_id"]:
            raise SmartyPantsError("Answer for %s came from build %s, not the pinned %s." % (case_id, meta.get("build_id") or "unknown", pin["build_id"]),
                                   code="BUILD_CHANGED", body={"wanted": pin["build_id"], "live": meta.get("build_id")})
        usage = reply.get("usage") or {}
        self.calls.append({"case_id": case_id, "request_id": reply.get("id"), "build_id": meta["build_id"], "node_id": meta.get("node_id"),
                           "prompt_sha256": sha256(canonical_json(messages)), "answer_sha256": sha256(answer),
                           "finish_reason": choice.get("finish_reason"), "prompt_tokens": usage.get("prompt_tokens"),
                           "completion_tokens": usage.get("completion_tokens"), "ms": int((time.time() - started) * 1000)})
        return answer

    def sandbox(self, fn, minutes=10, poll_seconds=1.0):
        """Open a fresh sandbox VM, record its versions, and call fn(ws). The VM is always closed after.
        Every ws.run() inside is recorded (source hash, exit code, output hash)."""
        self.started_at = self.started_at or _now()
        with self.sp.workspace(minutes=minutes, poll_seconds=poll_seconds) as ws:
            try:
                probe = ws.run("sh", SANDBOX_PROBE, timeout_seconds=30)
                try:
                    versions = json.loads(probe["stdout"].strip().split("\n")[-1])
                except ValueError:
                    versions = {"probe_error": (probe["stderr"] or probe["stdout"])[:200]}
                self.sandbox_info = {"session_id": ws.id, "opened_at": _now(), "versions": versions, "runs": []}
                return fn(_RecordedWorkspace(ws, self.sandbox_info["runs"]))
            finally:
                if self.sandbox_info:
                    self.sandbox_info["closed_at"] = _now()

    def evaluate(self, cases, grader=None, minutes=10, poll_seconds=1.0):
        """Run every case through the pinned build, then score it: locally (expect / match), or in a
        fresh VM with grader={"language", "source"}. Returns passed, total, results, answers, receipt."""
        if not cases:
            raise SmartyPantsError("evaluate() needs at least one case.", code="NO_CASES")
        ids = [str(c.get("id") or "case-%d" % (i + 1)) for i, c in enumerate(cases)]
        if len(set(ids)) != len(ids):
            raise SmartyPantsError("Each case needs a different id.", code="DUPLICATE_CASE_ID")
        self.reset()
        self.pin()
        rows = []
        for case_id, c in zip(ids, cases):
            extra = {"system": c["system"]} if c.get("system") else {}
            rows.append({"id": case_id, "prompt": c["prompt"], "answer": self.ask(c["prompt"], case_id=case_id, **extra), "expect": c.get("expect")})
        extra = {}
        if grader:
            self.grader_info = {"language": grader["language"], "source_sha256": sha256(grader["source"])}

            def grade(ws):
                parts = []
                for row in rows:
                    line = json.dumps(row, separators=(",", ":"), ensure_ascii=False) + "\n"
                    if len(line.encode("utf-8")) > PART_BYTES:
                        raise SmartyPantsError("Case %s is too big to upload (over %d bytes)." % (row["id"], PART_BYTES), code="CASE_TOO_LARGE")
                    if not parts or len((parts[-1] + line).encode("utf-8")) > PART_BYTES:
                        parts.append("")
                    parts[-1] += line
                for n, part in enumerate(parts):
                    ws.put_file("answers/part-%04d.jsonl" % n, part)
                joined = ws.run("sh", "cat answers/part-*.jsonl > answers.jsonl && wc -l < answers.jsonl", timeout_seconds=30)
                if joined["exitCode"] != 0:
                    raise SmartyPantsError("Could not join the answers in the VM: " + joined["stderr"], code="SANDBOX_FAILED")
                out = ws.run(grader["language"], grader["source"], timeout_seconds=grader.get("timeout_seconds", 120))
                if out["exitCode"] != 0:
                    raise SmartyPantsError("Grader exited %s: %s" % (out["exitCode"], out["stderr"][-500:]), code="GRADER_FAILED", body=out)
                try:
                    return json.loads(out["stdout"].strip().split("\n")[-1])
                except ValueError:
                    raise SmartyPantsError('The grader\'s last line of output must be JSON like {"passed": 3, "total": 4}, and all its output must stay under 4 KB.', code="GRADER_OUTPUT")

            graded = self.sandbox(grade, minutes=minutes, poll_seconds=poll_seconds)
            passed, total = graded.get("passed"), graded.get("total")
            results = graded.get("results") if isinstance(graded.get("results"), list) else None
            extra = {k: v for k, v in graded.items() if k not in ("passed", "total")}
        else:
            results = [{"id": r["id"], "pass": score_case(c, r["answer"])} for r, c in zip(rows, cases)]
            scored = [r for r in results if r["pass"] is not None]
            passed, total = sum(1 for r in scored if r["pass"]), len(scored)
        is_int = lambda n: isinstance(n, int) and not isinstance(n, bool)
        self.result = {"passed": passed if is_int(passed) else None, "total": total if is_int(total) else None,
                       "scored_in": "sandbox" if grader else "local",
                       "per_case": [{"id": str(r.get("id")), "pass": r.get("pass")} for r in (results or [])]}
        self.finished_at = _now()
        return {"passed": self.result["passed"], "total": self.result["total"], "results": results or [{"id": r["id"]} for r in rows],
                "answers": rows, "extra": extra, "receipt": self.receipt()}

    def receipt(self):
        """The receipt: exact model build, settings, sandbox versions, one line per call, the score, and a digest."""
        body = {"kind": RECEIPT_KIND, "kit_version": EVAL_KIT_VERSION, "base_url": self.sp.base_url,
                "started_at": self.started_at, "finished_at": self.finished_at or _now(),
                "settings": {"temperature": self.settings["temperature"], "max_tokens": self.settings["max_tokens"],
                             "thinking": self.settings["thinking"], "system": self.settings["system"]},
                "model": self.pinned, "sandbox": self.sandbox_info, "grader": self.grader_info, "calls": self.calls,
                "result": self.result, "nodes": sorted({c["node_id"] for c in self.calls if c.get("node_id")})}
        body["digest"] = "sha256:" + sha256(canonical_json(body))
        return body
