"""Tests on the recorded runs from the video: Lab 05's pipeline and checks (gpt-5.4-nano answers, judged by
gpt-5.4-mini) on the made-up help center, 3 runs per prompt. Run: pytest test_tuning_practice.py"""
from tuning_practice import mean_gain, moved, passes, score, verdict

# Every question's checks, per set; FAILED lists, per prompt and run, the checks that failed (the rest passed).
DEV_CHECKS = {"plane": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "copy": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "quit": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "secure": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "locked": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "nocard": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "refund": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "team14": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "slack": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "okta": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "vat": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "csv": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "reminders": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "archive": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "reactivate": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "invite": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "student": ["refusal_accuracy"], "darkmode": ["refusal_accuracy"], "applepay": ["refusal_accuracy"], "phone": ["refusal_accuracy"], "linux": ["refusal_accuracy"]}
HOLDOUT_CHECKS = {"h-yearly": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-invoice-name": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-backup-codes": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-change-email": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-conflict": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-guests": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-shortcuts": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-servers": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "h-teams-app": ["refusal_accuracy"], "h-crypto": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"]}
TOPICS_CHECKS = {"t-bank": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "t-paypal": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "t-refund-late": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "t-login-step": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "t-download": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "t-team8": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "t-team25": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"], "t-sidebar": ["citation_validity", "faithfulness", "hit_at_1", "mrr_at_k", "recall_at_k"]}
FAILED = {
    "start/dev": [{"copy": ["hit_at_1", "mrr_at_k"], "quit": ["citation_validity"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "nocard": ["citation_validity"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["hit_at_1", "mrr_at_k"], "archive": ["citation_validity"]}, {"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "nocard": ["citation_validity"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["faithfulness", "hit_at_1", "mrr_at_k"], "archive": ["citation_validity"], "reactivate": ["faithfulness"]}, {"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "nocard": ["citation_validity"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["hit_at_1", "mrr_at_k"], "archive": ["citation_validity"]}],
    "start/holdout": [{"h-backup-codes": ["citation_validity"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}, {"h-backup-codes": ["citation_validity"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}, {"h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}],
    "start/topics": [{"t-bank": ["citation_validity"], "t-refund-late": ["faithfulness"], "t-sidebar": ["citation_validity"]}, {"t-bank": ["citation_validity"], "t-sidebar": ["citation_validity"]}, {"t-bank": ["citation_validity"], "t-refund-late": ["faithfulness"], "t-sidebar": ["citation_validity"]}],
    "honest/dev": [{"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "nocard": ["faithfulness"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["faithfulness", "hit_at_1", "mrr_at_k"]}, {"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["hit_at_1", "mrr_at_k"]}, {"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["hit_at_1", "mrr_at_k"]}],
    "honest/holdout": [{"h-invoice-name": ["faithfulness"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}, {"h-invoice-name": ["faithfulness"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}, {"h-invoice-name": ["faithfulness"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}],
    "honest/topics": [{}, {}, {}],
    "overfit/dev": [{"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["faithfulness", "hit_at_1", "mrr_at_k"]}, {"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["faithfulness", "hit_at_1", "mrr_at_k"]}, {"copy": ["hit_at_1", "mrr_at_k"], "secure": ["hit_at_1", "mrr_at_k", "recall_at_k"], "refund": ["hit_at_1", "mrr_at_k"], "team14": ["faithfulness", "hit_at_1", "mrr_at_k"]}],
    "overfit/holdout": [{"h-invoice-name": ["faithfulness"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}, {"h-invoice-name": ["faithfulness"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}, {"h-invoice-name": ["faithfulness"], "h-crypto": ["citation_validity", "hit_at_1", "mrr_at_k", "recall_at_k"]}],
    "overfit/topics": [{}, {}, {}],
}
CHECKS = {"dev": DEV_CHECKS, "holdout": HOLDOUT_CHECKS, "topics": TOPICS_CHECKS}


def runs(prompt, qset):
    """The 3 recorded runs of a prompt on a question set, as {question id: {check: 1.0 or 0.0}}."""
    if qset == "new":  # all 18 held-out questions: the first 10, plus the 8 on the targeted rules' topics
        return [{**a, **b} for a, b in zip(runs(prompt, "holdout"), runs(prompt, "topics"))]
    return [{q: {k: 0.0 if k in run.get(q, []) else 1.0 for k in ks} for q, ks in CHECKS[qset].items()}
            for run in FAILED[f"{prompt}/{qset}"]]


def test_a_question_passes_only_if_every_check_passes():
    assert passes({"hit_at_1": 1.0, "citation_validity": 1.0, "faithfulness": 1.0}) is True
    # "How do I hide old projects?" at the start: the right answer, archive them, but no source tag.
    assert passes(runs("start", "dev")[0]["archive"]) is False
    assert passes({"refusal_accuracy": 1.0}) is True


def test_scores():
    assert [score(r) for r in runs("start", "dev")] == [14, 14, 15]
    assert [score(r) for r in runs("honest", "dev")] == [16, 17, 17]
    assert [score(r) for r in runs("overfit", "dev")] == [17, 17, 17]
    assert [score(r) for r in runs("start", "new")] == [13, 14, 14]
    assert [score(r) for r in runs("honest", "new")] == [16, 16, 16]
    assert [score(r) for r in runs("overfit", "topics")] == [8, 8, 8]


def test_measure_each_round_on_its_own():
    # Round one, two general rules: about +2.3 on the exam AND on new questions.
    assert mean_gain(runs("start", "dev"), runs("honest", "dev")) == 2.33
    assert mean_gain(runs("start", "new"), runs("honest", "new")) == 2.33
    # Round two, six rules aimed at failing exam questions: +1 run of 1 question, nothing new.
    assert mean_gain(runs("honest", "dev"), runs("overfit", "dev")) == 0.33
    assert mean_gain(runs("honest", "new"), runs("overfit", "new")) == 0.0


def test_read_which_questions_moved():
    # Round one on the exam: three questions fixed every run, and "pay without a card" in 2 runs of 3.
    assert moved(runs("start", "dev"), runs("honest", "dev")) == {
        "quit": (2, 3), "nocard": (0, 2), "archive": (0, 3), "reactivate": (2, 3)}
    # Its reworded twin on the new questions was fixed too, and a new question started failing (the judge's mistake).
    new = moved(runs("start", "new"), runs("honest", "new"))
    assert new["t-sidebar"] == (0, 3)
    assert new["h-invoice-name"] == (3, 0)
    # Round two: one run of one question.
    assert moved(runs("honest", "dev"), runs("overfit", "dev")) == {"nocard": (2, 3)}
    # The four other targeted questions that still fail, fail on search: no prompt rule can fix that.
    last = runs("overfit", "dev")[0]
    for q in ("copy", "secure", "refund", "team14"):
        assert last[q]["hit_at_1"] == 0.0


def test_verdict():
    assert verdict(2.33, 2.33) == "carries over"
    assert verdict(0.33, 0.0) == "no effect"
    assert verdict(2.0, 0.0) == "fits the test"