"""Tests on Lab 11's recorded runs: Qwen3-0.6B's layer shapes and adapter sizes (data/adapters/*.json),
real answers to test email K0000 (01_prompt_first.json, 02_lora.json), and the paired counts behind
"99% vs. 96%". Run: pytest test_finetune_practice.py"""
from finetune_practice import invented_rate, lora_params, mcnemar_p, score

# One Qwen3-0.6B block: (d_in, d_out) of each linear layer LoRA wraps. The model has 28 blocks.
ATTENTION = [(1024, 2048), (1024, 1024), (1024, 1024), (2048, 1024)]   # q, k, v, o
MLP = [(1024, 3072), (1024, 3072), (3072, 1024)]                        # gate, up, down


def test_lora_size_matches_the_recorded_adapters():
    assert lora_params(ATTENTION + MLP, r=8, layers=28) == 5_046_272     # 0.8% of 596M, a 20 MB file
    assert lora_params(ATTENTION + MLP, r=1, layers=28) == 630_784
    assert lora_params(ATTENTION + MLP, r=64, layers=28) == 40_370_176
    assert lora_params(ATTENTION, r=8, layers=28) == 2_293_760           # attention layers only
    assert lora_params([(1024, 2048)], r=8, layers=1) == 24_576          # vs 2,097,152 in that layer


# K0000: Sam shouts URGENT about boots that haven't shipped. The right urgency is normal.
GOLD = {"intent": "order_status", "order_id": "A-9387", "sku": "BOOT-W", "urgency": "normal"}
PROMPTED_06B = {"intent": "missing_package", "order_id": "A-9387", "sku": "BOOT-W", "urgency": "high"}
FINE_TUNED = {"intent": "order_status", "order_id": "A-9387", "sku": "BOOT-W", "urgency": "normal"}


def test_grading_one_email():
    assert score(FINE_TUNED, GOLD)["all"] is True
    s = score(PROMPTED_06B, GOLD)
    assert (s["intent"], s["order_id"], s["sku"], s["urgency"], s["all"]) == (False, True, True, False, False)


def test_grading_normalizes_and_handles_missing():
    loose = {"intent": " Order_Status", "order_id": "a-9387 ", "sku": "boot-w", "urgency": "NORMAL"}
    assert score(loose, GOLD)["all"] is True
    assert score({"intent": "billing", "order_id": "None", "sku": None, "urgency": "low"},
                 {"intent": "billing", "order_id": None, "sku": None, "urgency": "low"})["all"] is True
    assert score(None, GOLD) == {"intent": False, "order_id": False, "sku": False, "urgency": False, "all": False}
    assert score({"intent": "order_status", "sku": "BOOT-W", "urgency": "normal"}, GOLD)["order_id"] is False


def test_made_up_order_numbers():
    golds = [{"order_id": None}] * 4 + [{"order_id": "A-1001"}]
    preds = [{"order_id": "A-1234"}, {"order_id": "null"}, None, {"order_id": " "}, {"order_id": "A-1001"}]
    assert invented_rate(preds, golds) == 0.25        # one made up, of the four emails with none
    assert invented_rate([{"order_id": "A-1"}], [{"order_id": "A-1"}]) == 0.0
    # The recorded totals: the prompted 0.6B model made one up on 48 of 124, the fine-tune on 0.
    prompted = [{"order_id": "A-0000"}] * 48 + [{"order_id": None}] * 76
    assert round(invented_rate(prompted, [{"order_id": None}] * 124), 2) == 0.39


def _pair(both, a_only, b_only, neither):
    a = [True] * both + [True] * a_only + [False] * b_only + [False] * neither
    b = [True] * both + [False] * a_only + [True] * b_only + [False] * neither
    return a, b


def test_is_99_really_better_than_96():
    # gpt-5.4-mini (288/300) vs the fine-tune (296/300): mini alone right on 4 emails, the fine-tune on 12.
    mini, ft = _pair(both=284, a_only=4, b_only=12, neither=0)
    assert sum(mini) == 288 and sum(ft) == 296
    assert round(mcnemar_p(mini, ft), 3) == 0.077     # not significant: call it a tie


def test_50_examples_vs_the_prompted_17b():
    # Qwen3-1.7B with rules + examples (160/300) vs LoRA on 50 emails (171/300): 57 vs 68 disagreements.
    big, small_ft = _pair(both=103, a_only=57, b_only=68, neither=72)
    assert sum(big) == 160 and sum(small_ft) == 171
    assert round(mcnemar_p(big, small_ft), 2) == 0.37  # level, not "past it"
    assert mcnemar_p([True, False], [True, False]) == 1.0