"""Private-evidence summary of one completed matched German reasoning pilot."""

from __future__ import annotations

import hashlib
import json
import os
from pathlib import Path
import random
from statistics import mean

from .evaluation import _final_numeric, _verify_report, checkpoint_sha256, compare, sha256_file


def _read(path: Path) -> tuple[dict, str]:
    raw = path.read_bytes()
    return json.loads(raw), hashlib.sha256(raw).hexdigest()


def _require(condition: bool, message: str) -> None:
    if not condition:
        raise ValueError(message)


def _counts(report: dict) -> dict:
    rows, spec = report["results"], report["provenance"]["spec"]
    usage = [row["usage"] for row in rows if row.get("usage") is not None]
    malformed = sum(_final_numeric(row["output"]) is None for row in rows)
    clipped = sum(row["completion_tokens"] >= spec["sampling"]["max_new_tokens"] for row in usage)
    return {
        "correct": report["metrics"]["correct"],
        "total": len(rows),
        "strict_final_answer_protocol_accuracy": report["metrics"]["accuracy"],
        "malformed_outputs": malformed,
        "wrong_accepted_answers": len(rows) - malformed - report["metrics"]["correct"],
        "format_valid_rate": 1 - malformed / len(rows),
        "hit_token_cap_trials": clipped,
        "token_cap_unknown_trials": len(rows) - len(usage),
        "token_cap_note": "Completion count at or above budget marks possible clipping; EOS termination at exactly the budget cannot be distinguished.",
        "usage": {
            "reported_trials": len(usage),
            "prompt_tokens": sum(row["prompt_tokens"] for row in usage),
            "completion_tokens": sum(row["completion_tokens"] for row in usage),
        },
        "generation_seconds": sum(row["wall_seconds"] for row in rows),
        "mean_trial_seconds": mean(row["wall_seconds"] for row in rows),
    }


def _paired(baseline: dict, candidate: dict) -> dict:
    a = {(row["id"], row["trial"]): row["outcome"] == "correct" for row in baseline["results"]}
    b = {(row["id"], row["trial"]): row["outcome"] == "correct" for row in candidate["results"]}
    _require(a.keys() == b.keys(), "paired sample/trial identities differ")
    by_item = {}
    for key in sorted(a):
        by_item.setdefault(key[0], []).append(int(b[key]) - int(a[key]))
    differences = [mean(values) for values in by_item.values()]
    rng, repeats = random.Random(20261015), 10000
    draws = sorted(mean(rng.choices(differences, k=len(differences))) for _ in range(repeats))
    return {
        "items": len(by_item),
        "trials": len(a),
        "candidate_wins": sum(b[key] and not a[key] for key in a),
        "candidate_losses": sum(a[key] and not b[key] for key in a),
        "both_correct": sum(a[key] and b[key] for key in a),
        "both_incorrect": sum(not a[key] and not b[key] for key in a),
        "accuracy_delta": mean(differences),
        "bootstrap": {
            "method": "paired item percentile; repeated trials grouped by item",
            "seed": 20261015,
            "resamples": repeats,
            "ci_95": [draws[int(0.025 * (repeats - 1))], draws[int(0.975 * (repeats - 1))]],
        },
    }


def summarize(run_dir: str | Path, output_dir: str | Path) -> dict:
    """Validate completed evidence, then write summary.json/.md into a fresh directory."""
    root, output = Path(run_dir).resolve(), Path(output_dir).resolve()
    _require(not output.exists(), "choose a fresh summary output directory")
    paths = {
        "baseline": root / "baseline/report.json",
        "candidate": root / "candidate/report.json",
        "plan": root / "plan.json",
        "run": root / "run-manifest.json",
        "training": root / "training/recipe-manifest.json",
        "merge": root / "merged-checkpoint.merge-manifest.json",
    }
    loaded = {key: _read(path) for key, path in paths.items()}
    baseline, candidate, plan, run, training, merge = (loaded[key][0] for key in paths)
    for report in (baseline, candidate):
        _verify_report(report)
        samples = [
            json.loads(line)
            for line in report["provenance"]["benchmark_snapshot"].splitlines()
            if line.strip()
        ]
        _require(
            all(row["scorer"] == "final_numeric" for row in samples),
            "summary requires final_numeric samples",
        )
    gate = compare(paths["baseline"], paths["candidate"])
    _require(
        all("regression exceeds allowed drop" in reason for reason in gate["reasons"]),
        f"incomparable evaluation evidence: {gate['reasons']}",
    )
    _require(
        run["status"] == "completed" and training["status"] == merge["status"] == "completed",
        "run, training, and merge must all be completed",
    )
    _require(
        set(run["completed_stages"]) >= {"baseline", "sft", "merge", "candidate"},
        "run stages incomplete",
    )
    _require(run["plan_sha256"] == loaded["plan"][1], "run plan hash mismatch")
    a, b = (report["provenance"]["spec"] for report in (baseline, candidate))
    _require(
        a["model"]["revision"] == plan["model_sha256"] == merge["base_model"]["sha256"],
        "base model identity mismatch",
    )
    _require(
        b["model"]["revision"] == merge["output"]["sha256"], "candidate model identity mismatch"
    )
    _require(
        training["dataset_sha256"] == plan["train_sha256"] == sha256_file(root / "train.jsonl"),
        "training data identity mismatch",
    )
    _require(
        training["config_sha256"] == plan["recipe_sha256"] == sha256_file(root / "sft.json"),
        "training recipe identity mismatch",
    )
    _require(
        a["benchmark"]["sha256"]
        == b["benchmark"]["sha256"]
        == plan["evaluation_sha256"]
        == sha256_file(root / "evaluation.jsonl"),
        "evaluation data identity mismatch",
    )
    _require(
        a["model"]["tokenizer_revision"]
        == b["model"]["tokenizer_revision"]
        == plan["tokenizer_sha256"],
        "tokenizer identity mismatch",
    )
    _require(
        Path(training["model"]["name"]).resolve() == Path(plan["model"]).resolve(),
        "training base path mismatch",
    )
    _require(
        Path(training["checkpoint"]).resolve() == Path(merge["adapter"]["path"]).resolve(),
        "trained adapter path mismatch",
    )
    _require(
        checkpoint_sha256(training["checkpoint"]) == merge["adapter"]["sha256"],
        "trained adapter hash mismatch",
    )
    for spec, identity in ((a, plan["model_sha256"]), (b, merge["output"]["sha256"])):
        _require(spec["adapter"].get("sha256") == identity, "evaluation adapter identity mismatch")
    mode = a["model"].get("chat_template_kwargs", {})
    _require(
        training.get("chat_template_kwargs", {}) == plan["chat_template_kwargs"] == mode,
        "training/evaluation thinking mode mismatch",
    )
    links = {key: os.path.relpath(path, output) for key, path in paths.items()}
    summary = {
        "schema_version": 1,
        "status": "ready",
        "release_eligible": False,
        "quality_gate": "passed" if gate["passed"] else "regression",
        "comparison": gate,
        "baseline": _counts(baseline),
        "candidate": _counts(candidate),
        "paired": _paired(baseline, candidate),
        "sampling": a["sampling"],
        "trials_per_item": a["trials"],
        "chat_template_kwargs": mode,
        "runtime": {
            key: report["provenance"]["runtime"]
            for key, report in (("baseline", baseline), ("candidate", candidate))
        },
        "training": {
            key: training.get(key)
            for key in ("method", "rows", "seed", "training", "lora", "metrics", "dependencies")
        },
        "identity": {
            "baseline_model_sha256": plan["model_sha256"],
            "candidate_model_sha256": b["model"]["revision"],
            "tokenizer_sha256": plan["tokenizer_sha256"],
            "evaluation_sha256": plan["evaluation_sha256"],
            "source_data": plan.get("source_data", {}),
            "train_sha256": plan["train_sha256"],
            "recipe_sha256": plan["recipe_sha256"],
            "scorer_sha256": baseline["provenance"]["scorer_sha256"],
        },
        "evidence": {key: {"path": links[key], "sha256": loaded[key][1]} for key in paths},
        "artifacts": {
            "base": a["adapter"]["path"],
            "candidate": b["adapter"]["path"],
            "adapter": training["checkpoint"],
        },
        "limitations": "One bounded German mathematical-reasoning pilot. Paired item bootstrap is descriptive and excludes training-seed and generation-seed variance; a small stochastic item subset with one trial does not establish broad German reasoning ability or release readiness. This is strict final-answer protocol accuracy: improvement may reflect learned answer tags or shorter traces and token-cap compliance rather than better reasoning. Generation timing excludes model loading; malformed and token-cap-hit outputs remain in accuracy.",
    }
    output.mkdir(parents=True)
    (output / "summary.json").write_text(
        json.dumps(summary, indent=2, ensure_ascii=False, allow_nan=False) + "\n"
    )
    lines = [
        "# German reasoning pilot",
        "",
        f"Quality gate: **{summary['quality_gate']}**; release eligible: **false**.",
        "",
        "| Model | Correct / total | Protocol accuracy | Malformed | Hit token cap | Generation seconds |",
        "| --- | ---: | ---: | ---: | ---: | ---: |",
    ]
    for label in ("baseline", "candidate"):
        value = summary[label]
        lines.append(
            f"| {label} | {value['correct']} / {value['total']} | {value['strict_final_answer_protocol_accuracy']:.1%} | {value['malformed_outputs']} | {value['hit_token_cap_trials']} | {value['generation_seconds']:.2f} |"
        )
    paired = summary["paired"]
    lines += [
        "",
        f"Paired wins/losses: {paired['candidate_wins']}/{paired['candidate_losses']}. Accuracy delta: {paired['accuracy_delta']:+.1%}. Descriptive paired-item bootstrap 95% interval: {paired['bootstrap']['ci_95']}.",
        "",
        summary["limitations"],
        "",
        "Evidence: " + ", ".join(f"[{key}]({link})" for key, link in links.items()) + ".",
        "",
    ]
    (output / "summary.md").write_text("\n".join(lines))
    return summary