"""Tests on Lab 12's scripts 05 and 06, from the calls recorded for the video on 2026-11-02:
12 question pairs scored with text-embedding-3-small, and 24 labeled tickets answered by gpt-5.4-nano
and gpt-5.4-mini (run 1 of 3). Run: pytest test_shortcuts_practice.py"""
import math

from shortcuts_practice import cache_reuses, cascade, choose_cutoff, confidence, cosine

# (saved question, new question, is the saved answer right for it?, similarity: the mean of 3 runs)
PAIRS = [("How do I reset my password?", "I forgot my password, how can I change it?", True, 0.7665),
         ("How do I cancel my subscription?", "What are the steps to cancel my plan?", True, 0.7249),
         ("What are your support hours?", "When is your support team available?", True, 0.7809),
         ("How do I export my data to CSV?", "Can I download my data as a CSV file?", True, 0.7696),
         ("Do you offer a student discount?", "Is there a discount for students?", True, 0.8435),
         ("How do I add a teammate to my workspace?", "How can I invite a colleague to my workspace?", True, 0.7803),
         ("How do I cancel my subscription?", "How do I renew my subscription?", False, 0.7273),
         ("How do I delete my account?", "How do I delete my project?", False, 0.6805),
         ("Is my data encrypted at rest?", "Is my data encrypted in transit?", False, 0.8313),
         ("What does the Pro plan cost per month?", "What does the Team plan cost per month?", False, 0.7261),
         ("How do I add a teammate to my workspace?", "How do I remove a teammate from my workspace?", False, 0.7575),
         ("Can I get a refund within 30 days?", "Can I get a refund after 30 days?", False, 0.9576)]
OK = [p[2] for p in PAIRS]
SIMS = [p[3] for p in PAIRS]

# (id, split, right priority, ticket, nano's answer, nano's confidence, nano's cost,
#  mini's answer, mini's cost). Costs are in dollars per 1,000 such calls.
TICKETS = [(1, "dev", 1, "The dashboard returns a 500 error for everyone in our company since 9am.", 1, 0.9997, 0.038, 1, 0.14175),
           (2, "dev", 3, "I was charged twice for March, $49 each time. Please refund one.", 3, 0.9707, 0.0378, 3, 0.141),
           (3, "dev", 2, "We were billed $2,400 instead of $240 for our annual plan.", 2, 0.996, 0.0378, 2, 0.141),
           (4, "dev", 3, "Could you add dark mode? Our designers keep asking.", 3, 1.0, 0.0368, 3, 0.13725),
           (5, "dev", 3, "URGENT!!! We need SSO with Okta by Friday or we'll cancel our contract.", 3, 0.9984, 0.0382, 3, 0.1425),
           (6, "dev", 1, "I think someone logged into my account from another country. I see sessions I don't recognize.", 1, 0.9923, 0.0382, 1, 0.1425),
           (7, "dev", 1, "I accidentally pushed our API key to a public GitHub repo an hour ago.", 1, 0.9981, 0.0378, 1, 0.141),
           (8, "dev", 3, "Got an email claiming to be from your support team asking for my password. I didn't click anything, just forwarding it to you.", 3, 0.9989, 0.0398, 3, 0.1485),
           (9, "dev", 3, "The CSV export puts dates in the wrong format, but I can fix them in Excel afterwards.", 3, 1.0, 0.0384, 3, 0.14325),
           (10, "dev", 2, "The editor crashes every time I open any document, so I can't do any work today. My teammates are fine.", 2, 0.9796, 0.0392, 2, 0.14625),
           (11, "dev", 3, "How do I add a new user to our workspace?", 3, 1.0, 0.0368, 3, 0.13725),
           (12, "dev", 2, "I forgot my password and the reset email never arrives. I can't get in at all and have a client demo in an hour.", 2, 0.9645, 0.0398, 2, 0.1485),
           (13, "dev", 1, "After yesterday's sync, 3 months of project history disappeared from our workspace.", 1, 0.893, 0.0378, 1, 0.141),
           (14, "dev", 3, "Can I switch from monthly to annual billing in the middle of a cycle?", 3, 1.0, 0.0376, 3, 0.14025),
           (15, "dev", 3, "Search is slow, about 5 seconds per query, but it works.", 3, 1.0, 0.0376, 3, 0.14025),
           (16, "dev", 3, "Your mobile app would be better with an offline mode. Just a suggestion!", 3, 1.0, 0.0376, 3, 0.14025),
           (17, "holdout", 3, "Please remove my former employee's access. She left the company last week.", 2, 0.4524, 0.0376, 3, 0.14025),
           (18, "holdout", 2, "We were charged $650 for seats we removed last month.", 2, 0.9909, 0.037, 2, 0.138),
           (19, "holdout", 1, "One of our admins clicked a link in a fake login email and typed in their password.", 1, 0.987, 0.0382, 1, 0.1425),
           (20, "holdout", 1, "Nobody on our team can upload files. Every upload fails with 'network error' since your update.", 2, 0.8922, 0.0386, 1, 0.144),
           (21, "holdout", 3, "I can't change my profile picture, the button does nothing. Not a big deal.", 3, 0.9998, 0.038, 3, 0.14175),
           (22, "holdout", 3, "Refund request: I cancelled during the trial but was still charged $15.", 3, 0.9929, 0.0376, 3, 0.14025),
           (23, "holdout", 2, "My account is locked after too many login attempts and I need to submit payroll today. There's nothing else I can do until it's unlocked.", 2, 0.9955, 0.04, 2, 0.14925),
           (24, "holdout", 3, "We'd love an API endpoint for exporting audit logs. We're comparing you with a competitor and this is a must-have for us.", 3, 1.0, 0.0398, 3, 0.1485)]


def ticket(row):
    _, split, priority, text, sa, sc, scost, ba, bcost = row
    return {"split": split, "priority": priority, "text": text,
            "small": {"answer": sa, "confidence": sc, "cost": scost},
            "big": {"answer": ba, "cost": bcost}}


ALL = [ticket(r) for r in TICKETS]
DEV = [t for t in ALL if t["split"] == "dev"]
HOLDOUT = [t for t in ALL if t["split"] == "holdout"]
CANDIDATES = [0.0, 0.5, 0.7, 0.9, 0.99, 0.999]


def test_cosine_is_about_direction_not_length():
    assert cosine([1, 0], [1, 0]) == 1
    assert cosine([1, 0], [0, 1]) == 0
    assert math.isclose(cosine([3, 4], [4, 3]), 0.96)
    assert math.isclose(cosine([3, 4], [30, 40]), 1)


def test_the_two_groups_overlap():
    same = [s for s, ok in zip(SIMS, OK) if ok]
    alike = [s for s, ok in zip(SIMS, OK) if not ok]
    assert (round(min(same), 2), round(max(same), 2)) == (0.72, 0.84)
    assert (round(min(alike), 2), round(max(alike), 2)) == (0.68, 0.96)
    assert max(SIMS) == SIMS[-1]  # refund within vs after 30 days: a look-alike, and the highest score


def test_every_cutoff_that_saves_a_call_sends_a_wrong_answer():
    table = {c: cache_reuses(OK, SIMS, c) for c in [0.70, 0.75, 0.80, 0.85, 0.90, 0.95, 0.96]}
    assert table[0.70] == {"good": 6, "wrong": 5}
    assert table[0.75] == {"good": 5, "wrong": 3}
    assert table[0.80] == {"good": 1, "wrong": 2}
    assert table[0.85] == table[0.90] == table[0.95] == {"good": 0, "wrong": 1}
    assert table[0.96] == {"good": 0, "wrong": 0}


def test_confidence_from_a_logprob():
    assert confidence(0.0) == 1
    assert math.isclose(confidence(math.log(0.892)), 0.892)
    assert 0 < confidence(-5) < 0.01


def test_nano_alone_and_mini_alone():
    assert cascade(ALL, 0.0)["right"] == 22
    assert cascade(ALL, 0.0)["sent_up"] == 0
    assert cascade(ALL, 1.01)["right"] == 24  # a cutoff above 1 sends everything up
    # sure and right: 21 of 24 answers were over 0.95 sure, and every one was right
    sure = [t for t in ALL if t["small"]["confidence"] > 0.95]
    assert len(sure) == 21 and all(t["small"]["answer"] == t["priority"] for t in sure)


def test_mini_costs_about_3_7_times_as_much():
    nano = sum(t["small"]["cost"] for t in ALL) / len(ALL)
    mini = sum(t["big"]["cost"] for t in ALL) / len(ALL)
    assert (round(nano, 3), round(mini, 3), round(mini / nano, 1)) == (0.038, 0.142, 3.7)


def test_dev_has_no_mistakes_so_it_chooses_never_escalate():
    assert cascade(DEV, 0.0)["right"] == 16
    assert choose_cutoff(DEV, CANDIDATES) == 0.0


def test_the_holdout_misses_the_upload_outage():
    r = cascade(HOLDOUT, 0.0)
    assert (r["right"], r["sent_up"]) == (6, 0)
    outage = next(t for t in HOLDOUT if t["text"].startswith("Nobody on our team"))
    assert (outage["priority"], outage["small"]["answer"]) == (1, 2)


def test_cutoffs_found_by_peeking_at_the_holdout():
    mini = sum(t["big"]["cost"] for t in HOLDOUT) / 8
    r95, r99 = cascade(HOLDOUT, 0.95), cascade(HOLDOUT, 0.99)
    assert (r95["right"], r95["sent_up"], round(r95["cost"] / 8, 3)) == (8, 2, 0.074)
    assert (r99["right"], r99["sent_up"], round(r99["cost"] / 8, 3)) == (8, 3, 0.092)
    assert round(mini, 3) == 0.143
    assert (round(r95["cost"] / 8 / mini, 2), round(r99["cost"] / 8 / mini, 2)) == (0.52, 0.64)


def test_a_dev_set_with_a_mistake_in_it_chooses_a_cutoff():
    outage = next(t for t in HOLDOUT if t["text"].startswith("Nobody on our team"))
    assert choose_cutoff(DEV + [outage], CANDIDATES) == 0.9