import copy
import hashlib
import json
from pathlib import Path

import pytest

from hessian_recipes.evaluation import compare, load_spec, run, score_sample, wilson_interval

ROOT = Path(__file__).resolve().parents[1]
REASONING = ROOT / "benchmarks/reasoning_reliability"
WORKFLOW = ROOT / "benchmarks/tool_workflow_reliability"


def copy_track(tmp_path, source=REASONING):
    for file in source.iterdir():
        if file.is_file():
            (tmp_path / file.name).write_bytes(file.read_bytes())
    return tmp_path / "spec.json"


def edit_spec(path, mutate):
    spec = json.loads(path.read_text())
    mutate(spec)
    path.write_text(json.dumps(spec))


def save_report(path, report):
    path.write_text(json.dumps(report))


def callable_spec(tmp_path):
    path = copy_track(tmp_path)

    def edit(spec):
        spec["adapter"] = {"type": "callable", "name": "test-generator", "revision": "a" * 40}
        spec["model"]["revision"] = "b" * 40
        spec["model"]["tokenizer_revision"] = "c" * 40

    edit_spec(path, edit)
    return path


@pytest.mark.parametrize("track,expected", [(REASONING, 0.5), (WORKFLOW, 1 / 3)])
def test_fixture_replay_and_candidate_gate(tmp_path, track, expected):
    baseline = run(track / "spec.json", tmp_path / "baseline")
    candidate = run(track / "spec-candidate.json", tmp_path / "candidate")
    assert baseline["metrics"]["accuracy"] == expected
    assert candidate["metrics"]["accuracy"] == 1
    assert baseline["status"] == "complete"
    assert baseline["token_usage"]["reported_trials"] == 0
    gate = compare(tmp_path / "baseline/report.json", tmp_path / "candidate/report.json")
    assert gate["passed"]
    assert gate["release_eligible"] is False
    assert candidate["release_eligible"] is False
    assert (tmp_path / "baseline/report.md").is_file()


def test_dataset_hash_rejects_changed_question(tmp_path):
    spec_path = copy_track(tmp_path)
    with (tmp_path / "data.jsonl").open("a") as stream:
        stream.write("\n")
    with pytest.raises(ValueError, match="SHA256 mismatch"):
        run(spec_path, tmp_path / "out")


def test_prediction_hash_rejects_changed_outputs(tmp_path):
    spec_path = copy_track(tmp_path)
    (tmp_path / "predictions.jsonl").write_text("{}\n")
    with pytest.raises(ValueError, match="SHA256 mismatch"):
        run(spec_path, tmp_path / "out")


def test_missing_fixture_prediction_is_infrastructure_failure(tmp_path):
    spec_path = copy_track(tmp_path)
    predictions = tmp_path / "predictions.jsonl"
    lines = predictions.read_text().splitlines()
    predictions.write_text("\n".join(lines[:-1]) + "\n")
    edit_spec(
        spec_path,
        lambda spec: spec["adapter"].update(
            sha256=hashlib.sha256(predictions.read_bytes()).hexdigest()
        ),
    )
    report = run(spec_path, tmp_path / "out")
    assert report["status"] == "failed"
    assert report["metrics"]["infrastructure_errors"] == 1
    assert report["metrics"]["accuracy"] is None
    assert report["metrics"]["wilson_95"] is None
    assert report["metrics"]["incorrect"] == 4
    assert not compare(tmp_path / "out/report.json", tmp_path / "out/report.json")["passed"]


def test_callable_labels_hidden_and_trial_budget_failures_visible(tmp_path):
    spec_path = callable_spec(tmp_path)
    seen = []

    def generator(sample, trial, spec):
        assert set(sample) == {"id", "prompt", "category"}
        seen.append((sample["id"], trial))
        if sample["id"] == "logic-01":
            raise TimeoutError("offline generator failure")
        return "wrong"

    report = run(spec_path, tmp_path / "out", generator=generator)
    assert len(seen) == 8
    assert report["metrics"]["scored"] == 6
    assert report["metrics"]["infrastructure_errors"] == 2
    assert report["metrics"]["incorrect"] == 6
    assert report["metrics"]["scored_accuracy"] == 0
    assert report["metrics"]["accuracy"] is None


def test_usage_budget_excess_is_infrastructure_contract_failure(tmp_path):
    spec_path = callable_spec(tmp_path)

    def generator(sample, trial, spec):
        return {"output": "30", "usage": {"prompt_tokens": 10, "completion_tokens": 257}}

    report = run(spec_path, tmp_path / "out", generator=generator)
    assert report["status"] == "failed"
    assert report["metrics"]["infrastructure_errors"] == 8


def test_injected_callable_must_be_declared(tmp_path):
    with pytest.raises(ValueError, match="requires adapter.type"):
        run(REASONING / "spec.json", tmp_path, generator=lambda *args: "0")


def test_unpinned_callable_revision_rejected(tmp_path):
    spec_path = callable_spec(tmp_path)
    edit_spec(spec_path, lambda spec: spec["model"].update(revision="main"))
    with pytest.raises(ValueError, match="immutable"):
        run(spec_path, tmp_path / "out", generator=lambda *args: "0")


@pytest.mark.parametrize("field", ["sampling", "benchmark", "trials", "chat_template"])
def test_comparison_rejects_unmatched_conditions(tmp_path, field):
    baseline = run(REASONING / "spec.json", tmp_path / "baseline")
    candidate = copy.deepcopy(baseline)
    if field == "sampling":
        candidate["provenance"]["spec"][field]["max_new_tokens"] = 512
    elif field == "benchmark":
        candidate["provenance"]["spec"][field]["sha256"] = "f" * 64
    elif field == "trials":
        candidate["provenance"]["spec"][field] = 3
    else:
        candidate["provenance"]["spec"]["model"][field] = "different"
    save_report(tmp_path / "candidate.json", candidate)
    result = compare(tmp_path / "baseline/report.json", tmp_path / "candidate.json")
    assert not result["passed"]
    assert any("incompatible" in reason for reason in result["reasons"])


def test_gate_recomputes_metrics_and_rejects_deleted_trials(tmp_path):
    baseline = run(REASONING / "spec.json", tmp_path / "baseline")
    altered = copy.deepcopy(baseline)
    altered["metrics"]["accuracy"] = 1
    save_report(tmp_path / "altered.json", altered)
    assert not compare(tmp_path / "baseline/report.json", tmp_path / "altered.json")["passed"]
    altered = copy.deepcopy(baseline)
    altered["results"].pop()
    save_report(tmp_path / "altered.json", altered)
    assert not compare(tmp_path / "baseline/report.json", tmp_path / "altered.json")["passed"]


def test_gate_checks_regression_with_explicit_fraction_tolerance(tmp_path):
    run(REASONING / "spec-candidate.json", tmp_path / "baseline")
    run(REASONING / "spec.json", tmp_path / "candidate")
    assert not compare(tmp_path / "baseline/report.json", tmp_path / "candidate/report.json")[
        "passed"
    ]
    assert compare(
        tmp_path / "baseline/report.json", tmp_path / "candidate/report.json", max_drop=1.0
    )["passed"]
    with pytest.raises(ValueError):
        compare(
            tmp_path / "baseline/report.json",
            tmp_path / "candidate/report.json",
            max_drop=float("nan"),
        )


@pytest.mark.parametrize("output", ["NaN", "Infinity", "The answer is 30", "3e1 and 31"])
def test_numeric_format_errors_are_incorrect_not_infrastructure(output):
    assert score_sample({"scorer": "numeric", "answer": "30"}, output)[0] is False


def test_numeric_decimal_exactness():
    assert score_sample({"scorer": "numeric", "answer": "30"}, "3e1")[0] is True
    assert score_sample({"scorer": "numeric", "answer": "30"}, "30.0000000001")[0] is False


@pytest.mark.parametrize(
    "output",
    [
        '{"actions":[{"tool":"shell","item":"sensor","quantity":1}]}',
        '{"actions":[{"tool":"reserve","item":"sensor","quantity":-1}]}',
        '{"actions":[{"tool":"reserve","item":"sensor","quantity":true}]}',
        '{"actions":[{"tool":"reserve","item":"sensor","quantity":4}]}',
        '{"actions":[{"tool":"reserve","item":"sensor","quantity":2,"command":"touch anything"}]}',
        '__import__("os").system("touch anything")',
    ],
)
def test_workflow_rejects_invalid_or_executable_plans(output):
    _, samples, _ = load_spec(WORKFLOW / "spec.json")
    correct, details = score_sample(samples[0], output)
    assert correct is False
    assert "invalid_plan" in details


def test_workflow_cannot_exceed_step_limit():
    _, samples, _ = load_spec(WORKFLOW / "spec.json")
    output = json.dumps({"actions": [{"tool": "read", "item": "sensor"}] * 5})
    assert score_sample(samples[0], output)[0] is False


def test_wilson_interval_edges():
    assert wilson_interval(0, 0) is None
    assert wilson_interval(0, 10)[0] == 0
    assert wilson_interval(10, 10)[1] == 1
    low, high = wilson_interval(5, 10)
    assert 0.2 < low < 0.3 and 0.7 < high < 0.8


def test_checkpoint_digest_tracks_content_and_paths(tmp_path):
    from hessian_recipes.evaluation import checkpoint_sha256

    checkpoint = tmp_path / "checkpoint"
    checkpoint.mkdir()
    weights = checkpoint / "weights.bin"
    weights.write_bytes(b"local fixture, not actual model weights")
    original = checkpoint_sha256(checkpoint)
    assert original == checkpoint_sha256(checkpoint)
    weights.rename(checkpoint / "renamed.bin")
    assert checkpoint_sha256(checkpoint) != original
    (checkpoint / "external").symlink_to(__file__)
    with pytest.raises(ValueError, match="symlinks"):
        checkpoint_sha256(checkpoint)


def test_local_checkpoint_hash_rejected_before_loading_dependencies(tmp_path):
    spec_path = callable_spec(tmp_path)
    checkpoint = tmp_path / "checkpoint"
    checkpoint.mkdir()
    (checkpoint / "config.json").write_text("{}")
    edit_spec(
        spec_path,
        lambda spec: spec.update(
            adapter={"type": "transformers", "path": "checkpoint", "sha256": "a" * 64}
        ),
    )
    with pytest.raises(ValueError, match="checkpoint directory SHA256 mismatch"):
        run(spec_path, tmp_path / "out")


def test_existing_output_rejected_before_calling_generator(tmp_path):
    spec_path = callable_spec(tmp_path)
    output = tmp_path / "out"
    output.mkdir()
    report_path = output / "report.json"
    report_path.write_text("existing evidence")
    called = []
    with pytest.raises(FileExistsError, match="fresh output"):
        run(spec_path, output, generator=lambda *args: called.append(True))
    assert called == []
    assert report_path.read_text() == "existing evidence"


def test_category_regression_rejected_despite_overall_improvement(tmp_path):
    from hessian_recipes.evaluation import _summary

    baseline = run(REASONING / "spec.json", tmp_path / "baseline")
    candidate = copy.deepcopy(baseline)
    _, samples, _ = load_spec(REASONING / "spec.json")
    targets = {sample["id"]: str(sample["answer"]) for sample in samples}
    for row in candidate["results"]:
        row["outcome"] = "incorrect" if row["category"] == "units" else "correct"
        row["output"] = "wrong" if row["category"] == "units" else targets[row["id"]]
    candidate["metrics"] = _summary(candidate["results"])
    candidate["categories"] = {
        category: _summary([row for row in candidate["results"] if row["category"] == category])
        for category in candidate["categories"]
    }
    assert candidate["metrics"]["accuracy"] > baseline["metrics"]["accuracy"]
    save_report(tmp_path / "candidate.json", candidate)
    result = compare(tmp_path / "baseline/report.json", tmp_path / "candidate.json")
    assert not result["passed"]
    assert result["category_deltas"]["units"] == -1
    assert any("category units" in reason for reason in result["reasons"])


def test_comparison_rescores_output_and_rejects_modified_category(tmp_path):
    baseline = run(REASONING / "spec.json", tmp_path / "baseline")
    candidate = copy.deepcopy(baseline)
    candidate["results"][0]["output"] = "117"  # Stored incorrect outcome now disagrees.
    save_report(tmp_path / "candidate.json", candidate)
    gate = compare(tmp_path / "baseline/report.json", tmp_path / "candidate.json")
    assert not gate["passed"]
    assert any("re-scored output" in reason for reason in gate["reasons"])
    candidate = copy.deepcopy(baseline)
    candidate["results"][0]["category"] = "mislabelled"
    save_report(tmp_path / "candidate.json", candidate)
    assert not compare(tmp_path / "baseline/report.json", tmp_path / "candidate.json")["passed"]


def test_report_preserves_original_spec_and_dataset_snapshots(tmp_path):
    spec_path = callable_spec(tmp_path)
    original_spec = spec_path.read_bytes()
    original_dataset = (tmp_path / "data.jsonl").read_bytes()
    changed = []

    def generator(sample, trial, spec):
        if not changed:
            spec_path.write_text("not the originally parsed spec")
            (tmp_path / "data.jsonl").write_text("not the originally parsed data")
            changed.append(True)
        return "wrong"

    report = run(spec_path, tmp_path / "out", generator=generator)
    assert report["provenance"]["spec_sha256"] == hashlib.sha256(original_spec).hexdigest()
    assert report["provenance"]["spec_snapshot"].encode() == original_spec
    assert report["provenance"]["benchmark_snapshot"].encode() == original_dataset
    # Comparison depends on its verified stored snapshot, not mutable source files.
    assert compare(tmp_path / "out/report.json", tmp_path / "out/report.json")["passed"]


def test_standalone_peft_adapters_rejected_before_resolving_unpinned_base(tmp_path):
    spec_path = callable_spec(tmp_path)
    checkpoint = tmp_path / "checkpoint"
    checkpoint.mkdir()
    (checkpoint / "adapter_config.json").write_text(
        json.dumps({"base_model_name_or_path": "external/unpinned-base"})
    )
    edit_spec(
        spec_path,
        lambda spec: spec.update(
            adapter={"type": "transformers", "path": "checkpoint", "sha256": "a" * 64}
        ),
    )
    with pytest.raises(ValueError, match="standalone PEFT adapters"):
        run(spec_path, tmp_path / "out")


@pytest.mark.parametrize(
    "output",
    [
        "<think>12, 43, 999 are intermediate values.</think>\n<answer>42</answer>",
        "Intermediate value 999.\n</think>\n<answer>4.2e1</answer>",
        "42",
    ],
)
def test_final_numeric_uses_only_unambiguous_final_answer(output):
    assert score_sample({"scorer": "final_numeric", "answer": "42"}, output)[0] is True


@pytest.mark.parametrize(
    "output",
    [
        "<answer>41</answer><answer>42</answer>",
        "<think>42<answer>42</answer>",
        "42 is the answer",
        "Intermediate result 42. Final result 41.",
        "<answer>42</answer> Done.",
        "<answer>4,2</answer>",
        "<answer>NaN</answer>",
        "<answer>+42</answer>",
        "<answer>042</answer>",
    ],
)
def test_final_numeric_rejects_ambiguous_or_noncanonical_answers(output):
    assert score_sample({"scorer": "final_numeric", "answer": "42"}, output)[0] is False


def test_final_numeric_exact_decimal_semantics():
    assert score_sample({"scorer": "final_numeric", "answer": "0.5"}, "<answer>0.50</answer>")[0]
    assert not score_sample(
        {"scorer": "final_numeric", "answer": "0.5"}, "<answer>0.5001</answer>"
    )[0]


@pytest.mark.parametrize(
    "kwargs", [{"enable_thinking": "true"}, {"enable_thinking": 1}, {"think": True}]
)
def test_template_kwargs_reject_silent_mode_typos(tmp_path, kwargs):
    spec_path = copy_track(tmp_path)
    edit_spec(spec_path, lambda spec: spec["model"].update(chat_template_kwargs=kwargs))
    with pytest.raises(ValueError, match="chat_template_kwargs"):
        load_spec(spec_path)


def test_comparison_rejects_thinking_mode_change(tmp_path):
    spec_path = copy_track(tmp_path)
    edit_spec(
        spec_path, lambda spec: spec["model"].update(chat_template_kwargs={"enable_thinking": True})
    )
    run(spec_path, tmp_path / "thinking")
    edit_spec(
        spec_path,
        lambda spec: spec["model"].update(chat_template_kwargs={"enable_thinking": False}),
    )
    run(spec_path, tmp_path / "nonthinking")
    gate = compare(tmp_path / "thinking/report.json", tmp_path / "nonthinking/report.json")
    assert not gate["passed"]
    assert "incompatible model.chat_template_kwargs" in gate["reasons"]


def test_unknown_adapter_fields_rejected_before_execution(tmp_path):
    spec_path = copy_track(tmp_path)
    edit_spec(spec_path, lambda spec: spec["adapter"].update(devcie="cuda"))
    with pytest.raises(ValueError, match="unsupported fixture adapter fields"):
        load_spec(spec_path)


def test_cuda_and_bfloat16_preflight_without_launching_model():
    from types import SimpleNamespace
    from hessian_recipes.evaluation import _torch_preflight

    torch = SimpleNamespace(
        float32=object(),
        bfloat16=object(),
        float16=object(),
        cuda=SimpleNamespace(is_available=lambda: False, is_bf16_supported=lambda: False),
    )
    assert _torch_preflight({}, torch) is torch.float32
    with pytest.raises(ValueError, match="available CUDA"):
        _torch_preflight({"device": "cuda", "dtype": "bfloat16"}, torch)
    torch.cuda.is_available = lambda: True
    with pytest.raises(ValueError, match="BF16 support"):
        _torch_preflight({"device": "cuda", "dtype": "bfloat16"}, torch)
    torch.cuda.is_bf16_supported = lambda: True
    assert _torch_preflight({"device": "cuda", "dtype": "bfloat16"}, torch) is torch.bfloat16


def test_execution_settings_normalize_defaults_and_keep_device_dtype():
    from hessian_recipes.evaluation import _execution_settings, _validate_adapter

    assert _execution_settings({"adapter": {"type": "transformers"}}) == {
        "device": "cpu",
        "dtype": "float32",
    }
    assert _execution_settings(
        {"adapter": {"type": "transformers", "device": "cuda", "dtype": "bfloat16"}}
    ) == {"device": "cuda", "dtype": "bfloat16"}
    with pytest.raises(ValueError, match="adapter.device"):
        _validate_adapter({"type": "transformers", "device": "gpu"})
    with pytest.raises(ValueError, match="adapter.dtype"):
        _validate_adapter({"type": "transformers", "dtype": "bf16"})


def test_qwen_sampling_knobs_accepted_and_compared_exactly(tmp_path):
    spec_path = copy_track(tmp_path)
    edit_spec(
        spec_path,
        lambda spec: spec["sampling"].update(temperature=0.6, top_p=0.95, top_k=20, min_p=0.0),
    )
    spec, _, _ = load_spec(spec_path)
    assert spec["sampling"]["top_k"] == 20
    run(spec_path, tmp_path / "baseline")
    edit_spec(spec_path, lambda spec: spec["sampling"].update(top_p=0.9))
    run(spec_path, tmp_path / "candidate")
    gate = compare(tmp_path / "baseline/report.json", tmp_path / "candidate/report.json")
    assert not gate["passed"]
    assert "incompatible sampling" in gate["reasons"]


@pytest.mark.parametrize(
    "knob,value",
    [
        ("top_p", 0),
        ("top_p", 1.1),
        ("top_p", True),
        ("top_p", float("nan")),
        ("top_k", -1),
        ("top_k", 2.5),
        ("top_k", True),
        ("min_p", -0.1),
        ("min_p", 1.1),
        ("min_p", True),
        ("unsupported_sampler", 1),
    ],
)
def test_sampling_knobs_reject_invalid_or_unsupported_values(tmp_path, knob, value):
    spec_path = copy_track(tmp_path)
    edit_spec(spec_path, lambda spec: spec["sampling"].update({knob: value}))
    with pytest.raises(ValueError, match="sampling"):
        load_spec(spec_path)