"""Small, dependency-free evaluation evidence runner; optional local HF generation.

This is a development integration layer. It does not certify releases or replace
Inspect's execution engine. All generated tool plans are interpreted by toy code.
"""

from __future__ import annotations

import copy
import hashlib
import importlib.metadata
import json
import math
import platform
import re
import time
from datetime import datetime, timezone
from decimal import Decimal, InvalidOperation
from pathlib import Path
from typing import Any, Callable

SCORER_VERSION = "1"
HARNESS_VERSION = "1"
Generator = Callable[[dict, int, dict], str | dict]


def sha256_file(path: str | Path) -> str:
    digest = hashlib.sha256()
    with Path(path).open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _json(path: Path) -> Any:
    return json.loads(path.read_text(encoding="utf-8"))


def _jsonl_snapshot(raw: bytes, filename: str) -> list[dict]:
    rows = []
    for number, line in enumerate(raw.decode("utf-8").splitlines(), 1):
        if line.strip():
            value = json.loads(line)
            if not isinstance(value, dict):
                raise ValueError(f"{filename}:{number}: expected a JSON object")
            rows.append(value)
    return rows


def _required_text(value: Any, name: str) -> str:
    if not isinstance(value, str) or not value.strip():
        raise ValueError(f"{name} must be a nonempty string")
    return value


def _positive_int(value: Any, name: str) -> int:
    if type(value) is not int or value < 1:
        raise ValueError(f"{name} must be a positive integer")
    return value


def _frozen_snapshot(base: Path, config: dict, name: str) -> tuple[Path, bytes]:
    path = (base / _required_text(config.get("path"), f"{name}.path")).resolve()
    expected = config.get("sha256")
    if not isinstance(expected, str) or not re.fullmatch(r"[a-f0-9]{64}", expected):
        raise ValueError(f"{name}.sha256 must be an exact lowercase SHA256")
    raw = path.read_bytes()
    if hashlib.sha256(raw).hexdigest() != expected:
        raise ValueError(f"{name} SHA256 mismatch: {path}")
    return path, raw


def load_spec(spec_path: str | Path) -> tuple[dict, list[dict], Path]:
    """Validate the manifest and verified dataset; all paths are spec-relative."""
    spec, samples, path, _ = _load_spec_snapshot(spec_path)
    return spec, samples, path


def _load_spec_snapshot(spec_path: str | Path) -> tuple[dict, list[dict], Path, dict]:
    path = Path(spec_path).resolve()
    spec_raw = path.read_bytes()
    spec = json.loads(spec_raw.decode("utf-8"))
    if not isinstance(spec, dict) or spec.get("schema_version") != 1:
        raise ValueError("evaluation spec schema_version must be 1")
    if (
        spec.get("scorer_version") != SCORER_VERSION
        or spec.get("harness_version") != HARNESS_VERSION
    ):
        raise ValueError("unsupported scorer_version or harness_version")
    for name in ("model", "sampling", "benchmark", "adapter"):
        if not isinstance(spec.get(name), dict):
            raise ValueError(f"{name} must be an object")
    model, sampling, benchmark = spec["model"], spec["sampling"], spec["benchmark"]
    _validate_adapter(spec["adapter"])
    template_kwargs = model.get("chat_template_kwargs", {})
    if not isinstance(template_kwargs, dict) or set(template_kwargs) - {"enable_thinking"}:
        raise ValueError("model.chat_template_kwargs supports only enable_thinking")
    if (
        "enable_thinking" in template_kwargs
        and type(template_kwargs["enable_thinking"]) is not bool
    ):
        raise ValueError("model.chat_template_kwargs.enable_thinking must be boolean")
    for key in ("id", "revision", "tokenizer_id", "tokenizer_revision", "chat_template"):
        _required_text(model.get(key), f"model.{key}")
    for key in ("id", "version", "status"):
        _required_text(benchmark.get(key), f"benchmark.{key}")
    _positive_int(spec.get("trials"), "trials")
    _positive_int(benchmark.get("count"), "benchmark.count")
    if spec["adapter"].get("type") != "fixture":
        for key in ("revision", "tokenizer_revision"):
            if not re.fullmatch(r"(?:[a-f0-9]{40}|[a-f0-9]{64})", model[key]):
                raise ValueError(f"model.{key} must be an immutable SHA1 or SHA256")
    _positive_int(sampling.get("max_new_tokens"), "sampling.max_new_tokens")
    temperature = sampling.get("temperature")
    if type(temperature) not in (int, float) or not math.isfinite(temperature) or temperature < 0:
        raise ValueError("sampling.temperature must be finite and nonnegative")
    if type(sampling.get("seed")) is not int or sampling["seed"] < 0:
        raise ValueError("sampling.seed must be a nonnegative integer")
    if set(sampling) - {"max_new_tokens", "temperature", "seed", "top_p", "top_k", "min_p"}:
        raise ValueError("unsupported sampling fields; do not silently ignore generation budgets")
    for key, lower, inclusive in (("top_p", 0, False), ("min_p", 0, True)):
        if key in sampling:
            value = sampling[key]
            if (
                type(value) not in (int, float)
                or not math.isfinite(value)
                or value > 1
                or (value < lower if inclusive else value <= lower)
            ):
                raise ValueError(
                    f"sampling.{key} must be finite and {'in [0, 1]' if inclusive else 'in (0, 1]'}"
                )
    if "top_k" in sampling and (type(sampling["top_k"]) is not int or sampling["top_k"] < 0):
        raise ValueError("sampling.top_k must be a nonnegative integer")
    dataset_path, dataset_raw = _frozen_snapshot(path.parent, benchmark, "benchmark")
    samples = _jsonl_snapshot(dataset_raw, dataset_path.name)
    if not samples or len(samples) != benchmark["count"]:
        raise ValueError("benchmark count mismatch or empty dataset")
    ids = set()
    for row in samples:
        for key in ("id", "prompt", "category", "scorer"):
            _required_text(row.get(key), f"sample.{key}")
        if row["id"] in ids:
            raise ValueError(f"duplicate sample id: {row['id']}")
        ids.add(row["id"])
        if row["scorer"] not in ("exact", "numeric", "final_numeric", "workflow"):
            raise ValueError(f"unknown scorer: {row['scorer']}")
        if row["scorer"] in ("exact", "numeric", "final_numeric") and "answer" not in row:
            raise ValueError(f"missing answer for {row['id']}")
        if row["scorer"] in ("numeric", "final_numeric"):
            try:
                if not Decimal(str(row["answer"])).is_finite():
                    raise ValueError("numeric target must be finite")
            except InvalidOperation as error:
                raise ValueError("invalid numeric target") from error
        if row["scorer"] == "workflow":
            workflow = row.get("workflow", {})
            if not isinstance(workflow, dict):
                raise ValueError("sample.workflow must be an object")
            _validate_state(workflow.get("initial_state"))
            _validate_state(workflow.get("target_state"))
            _positive_int(workflow.get("max_steps"), "workflow.max_steps")
    return spec, samples, path, {"spec": spec_raw, "benchmark": dataset_raw}


def _validate_adapter(config: dict) -> None:
    allowed = {
        "fixture": {"type", "path", "sha256"},
        "callable": {"type", "name", "revision"},
        "transformers": {
            "type",
            "path",
            "sha256",
            "tokenizer_path",
            "tokenizer_sha256",
            "device",
            "dtype",
        },
    }
    kind = config.get("type")
    if kind not in allowed:
        raise ValueError("adapter.type must be fixture, callable, or transformers")
    if set(config) - allowed[kind]:
        raise ValueError(
            f"unsupported {kind} adapter fields: {sorted(set(config) - allowed[kind])}"
        )
    if kind == "transformers":
        if config.get("device", "cpu") not in ("cpu", "cuda"):
            raise ValueError("adapter.device must be cpu or cuda")
        if config.get("dtype", "float32") not in ("float32", "bfloat16", "float16"):
            raise ValueError("adapter.dtype must be float32, bfloat16, or float16")


def _execution_settings(spec: dict) -> dict:
    config = spec["adapter"]
    if config["type"] == "transformers":
        return {"device": config.get("device", "cpu"), "dtype": config.get("dtype", "float32")}
    return {}


def _torch_preflight(config: dict, torch: Any) -> Any:
    device, dtype = config.get("device", "cpu"), config.get("dtype", "float32")
    if device == "cuda":
        if not torch.cuda.is_available():
            raise ValueError("adapter.device=cuda requires an available CUDA device")
        if dtype == "bfloat16" and not torch.cuda.is_bf16_supported():
            raise ValueError("adapter.dtype=bfloat16 requires CUDA BF16 support")
    return getattr(torch, dtype)


def _final_numeric(output: str) -> str | None:
    """A whole canonical decimal or one terminal answer block after reasoning."""
    text = output.strip()
    if "<answer" in text or "</answer" in text:
        if text.count("<answer") != 1 or text.count("</answer") != 1:
            return None
        match = re.search(r"<answer>([^<>]+)</answer>\Z", text)
        if match is None:
            return None
        prefix = text[: match.start()]
        if prefix.rfind("<think>") > prefix.rfind("</think>"):
            return None
        text = match.group(1).strip()
    if not re.fullmatch(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?", text):
        return None
    return text


def _validate_state(state: Any) -> None:
    if not isinstance(state, dict) or set(state) != {"inventory", "reserved"}:
        raise ValueError("workflow state must have inventory and reserved maps")
    for quantities in state.values():
        if not isinstance(quantities, dict) or any(
            not isinstance(key, str) or type(value) is not int or value < 0
            for key, value in quantities.items()
        ):
            raise ValueError("workflow quantities must be nonnegative integers")
    if set(state["inventory"]) != set(state["reserved"]):
        raise ValueError("workflow item sets must match")


def replay_workflow(sample: dict, output: str) -> tuple[bool, dict]:
    """Interpret a bounded JSON plan using reserve/release/read; no code execution."""
    config = sample["workflow"]
    state = copy.deepcopy(config["initial_state"])
    trace = []
    try:
        plan = json.loads(output)
        if (
            not isinstance(plan, dict)
            or set(plan) != {"actions"}
            or not isinstance(plan["actions"], list)
        ):
            raise ValueError("expected exactly {actions: [...]}")
        if len(plan["actions"]) > config["max_steps"]:
            raise ValueError("tool step budget exceeded")
        for action in plan["actions"]:
            if not isinstance(action, dict):
                raise ValueError("each action must be an object")
            name, item = action.get("tool"), action.get("item")
            if name not in ("reserve", "release", "read"):
                raise ValueError("unknown tool")
            required = {"tool", "item"} if name == "read" else {"tool", "item", "quantity"}
            if set(action) != required or item not in state["inventory"]:
                raise ValueError("invalid tool arguments")
            if name == "read":
                trace.append(
                    {
                        "tool": name,
                        "item": item,
                        "inventory": state["inventory"][item],
                        "reserved": state["reserved"][item],
                    }
                )
                continue
            quantity = action["quantity"]
            if type(quantity) is not int or quantity <= 0:
                raise ValueError("quantity must be a positive integer")
            source, target = (
                ("inventory", "reserved") if name == "reserve" else ("reserved", "inventory")
            )
            if quantity > state[source][item]:
                raise ValueError("insufficient quantity")
            state[source][item] -= quantity
            state[target][item] += quantity
            trace.append({"tool": name, "item": item, "quantity": quantity, "status": "ok"})
        return state == config["target_state"], {"final_state": state, "trace": trace}
    except (ValueError, TypeError, KeyError) as error:
        return False, {"final_state": state, "trace": trace, "invalid_plan": str(error)}


def score_sample(sample: dict, output: str) -> tuple[bool, dict]:
    """Deterministic scorers: final answer only, never infer correctness from prose."""
    if sample["scorer"] == "workflow":
        return replay_workflow(sample, output)
    if sample["scorer"] == "exact":
        return output.strip() == str(sample["answer"]).strip(), {}
    if sample["scorer"] == "final_numeric":
        answer = _final_numeric(output)
        if answer is None:
            return False, {
                "invalid_answer": "expected one terminal <answer>canonical-decimal</answer> block or a whole canonical decimal"
            }
        output = answer
    try:
        actual, target = Decimal(output.strip()), Decimal(str(sample["answer"]))
        # Exact decimal semantics are intentional: no permissive regex extraction.
        return actual.is_finite() and actual == target, {}
    except InvalidOperation:
        return False, {"invalid_answer": "expected a single finite decimal number"}


def wilson_interval(successes: int, total: int) -> list[float] | None:
    if total == 0:
        return None
    z = 1.959963984540054
    p = successes / total
    denominator = 1 + z * z / total
    center = (p + z * z / (2 * total)) / denominator
    radius = z * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) / denominator
    return [
        0.0 if successes == 0 else max(0.0, center - radius),
        1.0 if successes == total else min(1.0, center + radius),
    ]


def _fixture_generator(base: Path, config: dict, samples: list[dict], trials: int) -> Generator:
    path, raw = _frozen_snapshot(base, config, "adapter")
    entries = {}
    allowed = {(sample["id"], trial) for sample in samples for trial in range(trials)}
    for row in _jsonl_snapshot(raw, path.name):
        key = (row.get("id"), row.get("trial"))
        if type(row.get("trial")) is not int or key not in allowed or key in entries:
            raise ValueError("fixture contains duplicate or unexpected sample/trial")
        entries[key] = row

    def generate(sample: dict, trial: int, spec: dict) -> dict:
        key = sample["id"], trial
        if key not in entries:
            raise RuntimeError("missing fixture prediction")
        row = entries[key]
        if "error" in row:
            raise RuntimeError("recorded fixture infrastructure failure")
        return {"output": row["output"], "usage": row.get("usage")}

    return generate


def checkpoint_sha256(path: str | Path) -> str:
    """Hash local checkpoint file names and content, reject mutable external links."""
    path = Path(path).resolve()
    digest = hashlib.sha256()
    if not path.is_dir():
        raise ValueError("checkpoint must be a directory")
    entries = list(path.rglob("*"))
    if any(entry.is_symlink() and not entry.resolve().is_relative_to(path) for entry in entries):
        raise ValueError(
            "checkpoint symlinks outside its directory are unsupported; materialize the snapshot"
        )
    files = sorted(p for p in entries if p.is_file())
    if not files:
        raise ValueError("local checkpoint directory is empty")
    for file in files:
        if file.is_symlink() and not file.resolve().is_relative_to(path.resolve()):
            raise ValueError(
                "checkpoint symlinks outside its directory are unsupported; materialize the snapshot"
            )
        digest.update(file.relative_to(path).as_posix().encode() + b"\0")
        digest.update(bytes.fromhex(sha256_file(file)))
    return digest.hexdigest()


def _transformers_generator(base: Path, spec: dict) -> tuple[Generator, dict]:
    config, model_spec = spec["adapter"], spec["model"]
    checkpoint = (base / _required_text(config.get("path"), "adapter.path")).resolve()
    if not checkpoint.is_dir():
        raise ValueError("transformers adapter requires an existing local checkpoint directory")
    if (checkpoint / "adapter_config.json").exists():
        raise ValueError(
            "standalone PEFT adapters are unsupported; evaluate a full or merged local checkpoint"
        )
    if config.get("sha256") != checkpoint_sha256(checkpoint):
        raise ValueError("local checkpoint directory SHA256 mismatch")
    tokenizer_path = (base / config.get("tokenizer_path", config["path"])).resolve()
    tokenizer_digest = checkpoint_sha256(tokenizer_path)
    if config.get("tokenizer_sha256") != tokenizer_digest:
        raise ValueError("local tokenizer directory SHA256 mismatch")
    # Tokenizer identity is independent of weights, so adapted models can compare.
    if (
        model_spec["revision"] != config["sha256"]
        or model_spec["tokenizer_revision"] != tokenizer_digest
    ):
        raise ValueError("model/tokenizer revisions must equal their respective directory SHA256")
    try:
        import torch
        from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig, set_seed
    except ImportError as error:
        raise ValueError("install the training extra for local Transformers evaluation") from error
    _validate_adapter(config)
    dtype = _torch_preflight(config, torch)
    device = config.get("device", "cpu")
    tokenizer = AutoTokenizer.from_pretrained(
        str(tokenizer_path), local_files_only=True, trust_remote_code=False
    )
    tokenizer.chat_template = model_spec["chat_template"]
    model = AutoModelForCausalLM.from_pretrained(
        str(checkpoint), local_files_only=True, trust_remote_code=False, dtype=dtype
    )
    model.to(device)
    model.eval()
    if config["sha256"] != checkpoint_sha256(checkpoint) or tokenizer_digest != checkpoint_sha256(
        tokenizer_path
    ):
        raise ValueError("local model/tokenizer files changed while loading")

    def generate(sample: dict, trial: int, run_spec: dict) -> dict:
        params = run_spec["sampling"]
        set_seed(params["seed"] + trial)
        text = tokenizer.apply_chat_template(
            [{"role": "user", "content": sample["prompt"]}],
            tokenize=False,
            add_generation_prompt=True,
            **model_spec.get("chat_template_kwargs", {}),
        )
        inputs = tokenizer(text, return_tensors="pt", add_special_tokens=False).to(device)
        # Do not inherit hidden top-p/top-k/beam/penalty defaults from checkpoints.
        settings = GenerationConfig(
            max_new_tokens=params["max_new_tokens"],
            do_sample=params["temperature"] > 0,
            temperature=params["temperature"] if params["temperature"] > 0 else 1.0,
            top_k=params.get("top_k", 0 if params["temperature"] > 0 else 50),
            top_p=params.get("top_p", 1.0),
            min_p=params.get("min_p", 0.0) if params["temperature"] > 0 else None,
            repetition_penalty=1.0,
            num_beams=1,
            eos_token_id=tokenizer.eos_token_id,
            pad_token_id=tokenizer.pad_token_id
            if tokenizer.pad_token_id is not None
            else tokenizer.eos_token_id,
            bos_token_id=tokenizer.bos_token_id,
        )
        with torch.inference_mode():
            tokens = model.generate(**inputs, generation_config=settings)
        generated = tokens[0, inputs["input_ids"].shape[1] :]
        return {
            "output": tokenizer.decode(generated, skip_special_tokens=True),
            "usage": {
                "prompt_tokens": inputs["input_ids"].shape[1],
                "completion_tokens": len(generated),
            },
        }

    runtime = {
        **_execution_settings(spec),
        "cuda_version": torch.version.cuda,
        "device_name": torch.cuda.get_device_name() if device == "cuda" else "CPU",
    }
    return generate, runtime


def _summary(results: list[dict]) -> dict:
    scored = [row for row in results if row["outcome"] != "infrastructure_error"]
    correct = sum(row["outcome"] == "correct" for row in scored)
    complete = len(scored) == len(results)
    return {
        "expected": len(results),
        "scored": len(scored),
        "correct": correct,
        "incorrect": len(scored) - correct,
        "infrastructure_errors": len(results) - len(scored),
        "accuracy": correct / len(results) if complete and results else None,
        "scored_accuracy": correct / len(scored) if scored else None,
        "wilson_95": wilson_interval(correct, len(results)) if complete else None,
        "wall_seconds": sum(row["wall_seconds"] for row in results),
    }


def run(
    spec_path: str | Path, output_dir: str | Path, *, generator: Generator | None = None
) -> dict:
    """Run every item/trial and write JSON + Markdown evidence, failing closed."""
    destination = Path(output_dir)
    destination.mkdir(parents=True, exist_ok=True)
    if any((destination / name).exists() for name in ("report.json", "report.md")):
        raise FileExistsError(
            "evaluation output already contains report.json or report.md; choose a fresh output directory"
        )
    spec, samples, spec_file, snapshots = _load_spec_snapshot(spec_path)
    config = spec["adapter"]
    adapter_type = config.get("type")
    runtime = _execution_settings(spec)
    if generator is not None:
        if adapter_type != "callable":
            raise ValueError("injected generator requires adapter.type='callable'")
        _required_text(config.get("name"), "adapter.name")
        revision = _required_text(config.get("revision"), "adapter.revision")
        if not re.fullmatch(r"(?:[a-f0-9]{40}|[a-f0-9]{64})", revision):
            raise ValueError("adapter.revision must be an immutable SHA1 or SHA256")
    elif adapter_type == "fixture":
        generator = _fixture_generator(spec_file.parent, config, samples, spec["trials"])
    elif adapter_type == "transformers":
        generator, runtime = _transformers_generator(spec_file.parent, spec)
    else:
        raise ValueError(
            "adapter must be fixture, transformers, or an explicitly injected callable"
        )
    started = time.perf_counter()
    results = []
    for sample in samples:
        for trial in range(spec["trials"]):
            began = time.perf_counter()
            record = {"id": sample["id"], "category": sample["category"], "trial": trial}
            try:
                public_sample = {key: sample[key] for key in ("id", "prompt", "category")}
                generated = generator(public_sample, trial, copy.deepcopy(spec))
                if isinstance(generated, str):
                    generated = {"output": generated}
                if not isinstance(generated, dict) or not isinstance(generated.get("output"), str):
                    raise ValueError("adapter must return text or {output: text, usage?: object}")
                output = generated["output"]
                if len(output) > 1_000_000:
                    raise ValueError("adapter output exceeds 1,000,000-character safety cap")
                usage = generated.get("usage")
                if usage is not None:
                    if not isinstance(usage, dict) or any(
                        type(usage.get(key)) is not int or usage[key] < 0
                        for key in ("prompt_tokens", "completion_tokens")
                    ):
                        raise ValueError(
                            "usage requires nonnegative integer prompt_tokens and completion_tokens"
                        )
                    if usage["completion_tokens"] > spec["sampling"]["max_new_tokens"]:
                        raise ValueError("adapter exceeded declared completion token budget")
                correct, details = score_sample(sample, output)
                record.update(
                    outcome="correct" if correct else "incorrect",
                    output=output,
                    details=details,
                    usage=usage,
                )
            except Exception as error:
                # Continue so that missing/failed samples remain visible in coverage.
                record.update(
                    outcome="infrastructure_error",
                    error={"type": type(error).__name__, "message": str(error)},
                )
            record["wall_seconds"] = time.perf_counter() - began
            results.append(record)
    metrics = _summary(results)
    categories = {
        category: _summary([row for row in results if row["category"] == category])
        for category in sorted({row["category"] for row in results})
    }
    usage_rows = [row for row in results if row.get("usage") is not None]
    token_usage = {
        "reported_trials": len(usage_rows),
        "unreported_trials": len(results) - len(usage_rows),
        "prompt_tokens": sum(row["usage"]["prompt_tokens"] for row in usage_rows),
        "completion_tokens": sum(row["usage"]["completion_tokens"] for row in usage_rows),
    }
    report = {
        "schema_version": 1,
        "status": "complete" if metrics["infrastructure_errors"] == 0 else "failed",
        "release_eligible": False,
        "release_note": "Development evidence only. A passing regression gate does not certify a release; independent benchmark validation and release policy are required.",
        "created_at": datetime.now(timezone.utc).isoformat(),
        "provenance": {
            "spec_sha256": hashlib.sha256(snapshots["spec"]).hexdigest(),
            "spec_snapshot": snapshots["spec"].decode("utf-8"),
            "benchmark_snapshot": snapshots["benchmark"].decode("utf-8"),
            "spec": spec,
            "scorer_sha256": sha256_file(Path(__file__)),
            "python": platform.python_version(),
            "packages": {
                name: importlib.metadata.version(name) for name in ("torch", "transformers")
            }
            if adapter_type == "transformers"
            else {},
            "sample_ids": [sample["id"] for sample in samples],
            "execution_mode": adapter_type,
            "runtime": runtime,
            "trial_seed_policy": "sampling.seed + trial (Transformers); callable must honor the same policy",
        },
        "metrics": metrics,
        "categories": categories,
        "token_usage": token_usage,
        "total_wall_seconds": time.perf_counter() - started,
        "uncertainty_note": "Wilson 95% intervals are descriptive binomial intervals over item/trial outcomes. Repeated trials on the same item and related items are correlated; these intervals do not establish generalization or independent-sample significance.",
        "results": results,
    }
    with (destination / "report.json").open("x", encoding="utf-8") as stream:
        stream.write(json.dumps(report, indent=2, allow_nan=False) + "\n")
    with (destination / "report.md").open("x", encoding="utf-8") as stream:
        stream.write(_markdown(report))
    return report


def _markdown(report: dict) -> str:
    metrics = report["metrics"]
    accuracy = (
        "unavailable (incomplete run)"
        if metrics["accuracy"] is None
        else f"{metrics['accuracy']:.1%}"
    )
    lines = [
        "# Development evaluation report",
        "",
        f"Status: **{report['status']}**. Accuracy: **{accuracy}**.",
        "",
        report["release_note"],
        "",
        f"Coverage: {metrics['scored']}/{metrics['expected']} trials scored; {metrics['infrastructure_errors']} infrastructure failures.",
        "",
        "| Category | Correct | Scored / expected | Accuracy |",
        "| --- | ---: | ---: | ---: |",
    ]
    for category, value in report["categories"].items():
        rate = "unavailable" if value["accuracy"] is None else f"{value['accuracy']:.1%}"
        lines.append(
            f"| {category} | {value['correct']} | {value['scored']} / {value['expected']} | {rate} |"
        )
    lines.extend(
        [
            "",
            f"Wilson 95% interval: {metrics['wilson_95']}",
            "",
            report["uncertainty_note"],
            "",
            f"Run wall time: {report['total_wall_seconds']:.3f} s. Token usage: {json.dumps(report['token_usage'], sort_keys=True)}.",
            "",
            f"Spec SHA256: `{report['provenance']['spec_sha256']}`. All immutable inputs and individual outcomes are in `report.json`.",
            "",
        ]
    )
    return "\n".join(lines)


def _verify_report(report: dict) -> None:
    """Recompute coverage and scores; never trust editable aggregate metrics alone."""
    provenance = report["provenance"]
    spec = provenance["spec"]
    _validate_adapter(spec["adapter"])
    if provenance["execution_mode"] != spec["adapter"]["type"]:
        raise ValueError("execution mode disagrees with frozen adapter")
    for key, value in _execution_settings(spec).items():
        if provenance["runtime"].get(key) != value:
            raise ValueError("execution runtime disagrees with frozen device/dtype")
    spec_snapshot = provenance["spec_snapshot"].encode("utf-8")
    if (
        hashlib.sha256(spec_snapshot).hexdigest() != provenance["spec_sha256"]
        or json.loads(spec_snapshot) != spec
    ):
        raise ValueError("spec snapshot/hash/object mismatch")
    dataset_snapshot = provenance["benchmark_snapshot"].encode("utf-8")
    if hashlib.sha256(dataset_snapshot).hexdigest() != spec["benchmark"]["sha256"]:
        raise ValueError("benchmark snapshot SHA256 mismatch")
    samples = _jsonl_snapshot(dataset_snapshot, "benchmark_snapshot")
    sample_map = {sample["id"]: sample for sample in samples}
    if len(sample_map) != spec["benchmark"]["count"] or len(samples) != len(sample_map):
        raise ValueError("benchmark snapshot count/identity mismatch")
    if provenance["scorer_sha256"] != sha256_file(Path(__file__)):
        raise ValueError("report requires a different scorer implementation")
    if report.get("schema_version") != 1 or spec.get("schema_version") != 1:
        raise ValueError("unsupported report or spec schema")
    if (
        spec.get("scorer_version") != SCORER_VERSION
        or spec.get("harness_version") != HARNESS_VERSION
    ):
        raise ValueError("unsupported scorer or harness version")
    expected = spec["benchmark"]["count"] * spec["trials"]
    if report["metrics"]["expected"] != expected:
        raise ValueError("expected trial count disagrees with frozen benchmark count")
    results = report["results"]
    if not results or len(results) != expected:
        raise ValueError("missing per-trial evidence")
    keys = {(row["id"], row["trial"]) for row in results}
    if len(keys) != expected:
        raise ValueError("duplicate per-trial evidence")
    ids = set(provenance["sample_ids"])
    if ids != set(sample_map) or len(ids) != spec["benchmark"]["count"]:
        raise ValueError("sample identity count mismatch")
    if keys != {(sample_id, trial) for sample_id in ids for trial in range(spec["trials"])}:
        raise ValueError("trial coverage mismatch")
    if any(
        row["outcome"] not in ("correct", "incorrect", "infrastructure_error") for row in results
    ):
        raise ValueError("unknown outcome")
    for row in results:
        sample = sample_map[row["id"]]
        if row["category"] != sample["category"]:
            raise ValueError("trial category disagrees with frozen sample")
        if row["outcome"] != "infrastructure_error":
            if not isinstance(row.get("output"), str):
                raise ValueError("trial output missing")
            correct, _ = score_sample(sample, row["output"])
            if row["outcome"] != ("correct" if correct else "incorrect"):
                raise ValueError("trial outcome disagrees with re-scored output")
    recomputed = _summary(results)
    for key in (
        "expected",
        "scored",
        "correct",
        "incorrect",
        "infrastructure_errors",
        "accuracy",
        "scored_accuracy",
    ):
        if report["metrics"].get(key) != recomputed[key]:
            raise ValueError("aggregate metrics disagree with per-trial evidence")
    if report.get("status") != "complete" or recomputed["infrastructure_errors"]:
        raise ValueError("run is incomplete or has infrastructure errors")


def compare(
    baseline_report_path: str | Path, candidate_report_path: str | Path, max_drop: float = 0.0
) -> dict:
    """A descriptive regression gate; no significance or release certification claim."""
    if type(max_drop) not in (float, int) or not math.isfinite(max_drop) or not 0 <= max_drop <= 1:
        raise ValueError("max_drop must be a finite fraction between 0 and 1")
    baseline, candidate = _json(Path(baseline_report_path)), _json(Path(candidate_report_path))
    reasons = []
    category_deltas = {}
    for label, report in (("baseline", baseline), ("candidate", candidate)):
        try:
            _verify_report(report)
        except (KeyError, TypeError, ValueError) as error:
            reasons.append(f"{label}: {error}")
    try:
        a, b = baseline["provenance"]["spec"], candidate["provenance"]["spec"]
        for field in ("benchmark", "sampling", "trials", "scorer_version", "harness_version"):
            # File locations can move; benchmark identity depends on id/version/hash/status.
            left, right = a[field], b[field]
            if field == "benchmark":
                left = {key: value for key, value in left.items() if key != "path"}
                right = {key: value for key, value in right.items() if key != "path"}
            if left != right:
                reasons.append(f"incompatible {field}")
        for field in ("tokenizer_id", "tokenizer_revision", "chat_template"):
            if a["model"][field] != b["model"][field]:
                reasons.append(f"incompatible model.{field}")
        if a["model"].get("chat_template_kwargs", {}) != b["model"].get("chat_template_kwargs", {}):
            reasons.append("incompatible model.chat_template_kwargs")
        if _execution_settings(a) != _execution_settings(b):
            reasons.append("incompatible execution device/dtype")
        for field in ("scorer_sha256", "execution_mode"):
            if baseline["provenance"][field] != candidate["provenance"][field]:
                reasons.append(f"incompatible {field}")
        if {(r["id"], r["trial"], r["category"]) for r in baseline["results"]} != {
            (r["id"], r["trial"], r["category"]) for r in candidate["results"]
        }:
            reasons.append("incompatible sample coverage")
        old, new = baseline["metrics"]["accuracy"], candidate["metrics"]["accuracy"]
        delta = new - old if old is not None and new is not None else None
        if delta is None:
            reasons.append("accuracy unavailable")
        elif delta < -max_drop - 1e-12:
            reasons.append("accuracy regression exceeds allowed drop")
        categories = {row["category"] for row in baseline["results"]}
        for category in sorted(categories):
            old_category = _summary(
                [row for row in baseline["results"] if row["category"] == category]
            )["accuracy"]
            new_category = _summary(
                [row for row in candidate["results"] if row["category"] == category]
            )["accuracy"]
            change = (
                new_category - old_category
                if old_category is not None and new_category is not None
                else None
            )
            category_deltas[category] = change
            if change is None:
                reasons.append(f"category {category}: accuracy unavailable")
            elif change < -max_drop - 1e-12:
                reasons.append(f"category {category}: regression exceeds allowed drop")
    except (KeyError, TypeError) as error:
        reasons.append(f"malformed comparison evidence: {error}")
        old, new, delta = None, None, None
    return {
        "passed": not reasons,
        "release_eligible": False,
        "reasons": reasons,
        "baseline_accuracy": old,
        "candidate_accuracy": new,
        "delta": delta,
        "max_drop": max_drop,
        "category_deltas": category_deltas,
        "note": "Descriptive regression gate only; inspect category-level outcomes and apply an independently approved release policy.",
    }