"""Tests on Lab 14's recorded run (October 2026, gpt-5.4-nano answering): the 50 cases sampled from the
week's traces and each version's result (07_traces_to_evals.json), plus a small made-up week for sampling.
Run: pytest test_evals_practice.py"""
import random

from evals_practice import (GATE, build_evalset, chance_at_most, gate, is_complaint, wilson)

# The 50 cases in the eval set's order: 10 what, 10 how, 10 list, 10 the articles don't answer, 10 long chats.
ANSWERABLE = "11111111111111111111111111111100000000001101111101"
LONG = "0" * 40 + "1" * 10
CITES = {
    "v1": "11111111111111111111111111011000000000001101111101",
    "v2": "00000000000000000000000000000000000000001001100000",
    "v3": "11111111011111011111111011111100000000001101111001",
}
LONG_TOKENS = {  # input tokens on the 10 long chats
    "v1": [464, 486, 530, 495, 585, 476, 489, 465, 563, 471],
    "v2": [1028, 983, 777, 866, 898, 1215, 1150, 710, 854, 964],
    "v3": [478, 500, 544, 509, 599, 490, 503, 479, 577, 485],
}


def rows(v):
    toks = iter(LONG_TOKENS[v])
    return [{"right": True, "answerable": a == "1", "cites": c == "1", "long": l == "1",
             "input_tokens": next(toks) if l == "1" else 300}
            for a, c, l in zip(ANSWERABLE, CITES[v], LONG)]


def test_complaints():
    assert is_complaint({"not_helpful"}) and is_complaint(["retry"]) and is_complaint({"edit", "helpful"})
    assert is_complaint({"human"})
    assert not is_complaint({"helpful"}) and not is_complaint(set())


def week():
    """A made-up week: 60 conversations of 9 turns, kinds in turn, some incidents and errors."""
    rng = random.Random(1)
    traces = []
    for c in range(60):
        for turn in range(1, 10):
            i = len(traces)
            traces.append({"trace_id": f"t{i}", "conversation_id": f"c{c}", "turn": turn,
                           "kind": ("what", "how", "list", "none")[(c + turn) % 4],
                           "question": f"q{i}", "answer": f"a{i}",
                           "incident": "I1" if rng.random() < 0.1 else None, "error": rng.random() < 0.05})
    return traces


def test_build_evalset_is_stratified_and_traceable():
    traces = week()
    by_id = {t["trace_id"]: t for t in traces}
    cases = build_evalset(traces)
    assert len(cases) == 50
    strata = [c["stratum"] for c in cases]
    for kind in ("what", "how", "list", "none"):
        assert strata.count(f"single turn, {kind}") == 10
    assert strata.count("long conversation (turn 8+)") == 10
    for c in cases:
        t = by_id[c["from_trace"]]  # every case leads back to its trace
        assert t["incident"] is None and not t["error"], "sample only clean traces"
        assert c["question"] == t["question"]
        if c["stratum"].startswith("single"):
            assert t["turn"] == 1 and c["history"] == [] and c["kind"] == t["kind"]
        else:
            earlier = [x for x in traces if x["conversation_id"] == t["conversation_id"]
                       and x["turn"] < t["turn"] and not x["error"]]
            assert t["turn"] >= 8
            assert c["history"] == [(x["question"], x["answer"]) for x in sorted(earlier, key=lambda x: x["turn"])]
    assert build_evalset(traces) == cases, "same seed, same set"


def test_wilson_ranges_match_the_lab():
    r = lambda k, n: tuple(round(100 * x) for x in wilson(k, n))  # noqa: E731
    assert r(36, 38) == (83, 99)   # v1
    assert r(3, 38) == (3, 21)     # v2
    assert r(34, 38) == (76, 96)   # v3, the fix
    assert r(50, 50) == (93, 100)  # right answers, every version
    assert r(0, 14) == (0, 22)     # a day's 10% judge sample with no bad answer
    assert wilson(0, 0) == (0.0, 1.0)


def test_gate_catches_v2_and_fails_the_fix_by_one_answer():
    g1, g2, g3 = gate(rows("v1")), gate(rows("v2")), gate(rows("v3"))
    assert g1["cites"] == (36, 38) and round(g1["long_tokens"]) == 502 and g1["pass"]
    assert g2["right"] == 1.0 and g2["checks"]["right"], "an answer-only test would pass v2"
    assert g2["cites"] == (3, 38) and round(g2["long_tokens"]) == 944
    assert g2["checks"] == {"right": True, "cites": False, "long_tokens": False} and not g2["pass"]
    assert g3["cites"] == (34, 38) and round(g3["long_tokens"]) == 516
    assert g3["checks"] == {"right": True, "cites": False, "long_tokens": True} and not g3["pass"]
    assert 35 / 38 >= GATE["cites"] > 34 / 38, "one more cited answer would pass"


def test_range_rule_fails_only_when_the_whole_range_is_below():
    assert gate(rows("v3"), rule="range")["pass"], "v3's range (76–96%) reaches the bar"
    g2 = gate(rows("v2"), rule="range")
    assert not g2["checks"]["cites"] and not g2["pass"], "v2's range tops out at 21%"
    assert gate(rows("v1"), rule="range")["pass"]


def test_free_throws():
    assert round(chance_at_most(34, 38, 0.9), 3) == 0.535  # a fix exactly at the bar fails about half the time
    assert round(chance_at_most(38, 38, 0.9), 9) == 1.0
    assert round(chance_at_most(34, 38, 0.95), 2) == 0.12
    assert round(chance_at_most(0, 38, 0.9), 12) == round(0.1 ** 38, 12)