"""Tests on Lab 13's recorded runs (October 2026, gpt-5.4-nano): Fatima's test email K0026 from script 03, and the
56 answerable questions with search returning wrong passages from script 04.
Run: pytest test_silent_practice.py"""
from silent_practice import (NOT_FOUND, REFUSAL, UNAVAILABLE, answer_defended, ask_until_valid, is_refusal,
                             refusal_alarm, refusal_rate, reask_note, validate_reply)

# Fatima's first reply and the reply after the note, as recorded.
FIRST = ("Hi Fatima—thanks for reaching out. I’m sorry about the color mix-up for order A-3235. Since the water "
         "filter received is the wrong color (FILT-1), we’ll help make it right. Please reply with a quick photo of "
         "the item and packaging label. Lantern Outfitters Support")
SECOND = ("Hi Fatima—thanks for reaching out. Sorry about the color mix-up for order A-3235. We’ll help fix it. "
          "Please reply with a quick photo of the water filter and the shipping label/packaging. "
          "Lantern Outfitters Support")

# Search returning wrong passages: 51 refusals, 3 right answers (from the model's own training) and 2 wrong.
WRONG_SEARCH = [REFUSAL] * 51 + [
    "Fine-tuning is a poor choice for a changing knowledge base when the needed knowledge changes frequently…",
    "A RAG system uses outside information at inference time by retrieving relevant content…",
    "A vector database is optimized to store vector embeddings. [provided_documents]",
    "Large language models still have problems with tool selection during response generation…",
    "Retrieval-augmented generation (RAG) combines retrieval with generation…",
]
WORKING_SEARCH = [REFUSAL] + ["An AI agent is a system that uses a language model… [ai_agents]"] * 55


def test_validate_reply_counts_in_code():
    assert validate_reply(FIRST, "A-3235") == ["reply has 44 words; the limit is 40"]
    assert validate_reply(SECOND, "A-3235") == []
    assert validate_reply("Thanks, we're on it. [Your Name]", "A-1111") == [
        "reply doesn't mention the order number A-1111", "reply contains a placeholder: [Your Name]"]
    assert validate_reply("Hello {name}, sorry!") == ["reply contains a placeholder: {name}"]
    assert validate_reply(" ".join(["word"] * 40)) == []  # exactly 40 is fine


def test_reask_note_says_what_was_wrong():
    assert reask_note(["reply has 44 words; the limit is 40"]) == (
        "That reply is not valid: reply has 44 words; the limit is 40. Reply again with only the corrected JSON.")
    assert reask_note(["a", "b"]) == "That reply is not valid: a; b. Reply again with only the corrected JSON."


def test_ask_until_valid_with_and_without_the_note():
    def model(note):  # like nano: too long again, unless told what was wrong
        calls.append(note)
        return SECOND if note and "44 words" in note else FIRST
    check = lambda text: validate_reply(text, "A-3235")  # noqa: E731

    calls = []
    r = ask_until_valid(model, check)
    assert r == {"text": SECOND, "valid": True, "attempts": 2}
    assert calls[0] is None and "44 words; the limit is 40" in calls[1]

    calls = []
    r = ask_until_valid(model, check, feedback=False)
    assert r == {"text": FIRST, "valid": False, "attempts": 3} and calls == [None, None, None]

    calls = []
    assert ask_until_valid(model, check, reasks=0)["attempts"] == 1 and len(calls) == 1
    assert ask_until_valid(lambda note: SECOND, check) == {"text": SECOND, "valid": True, "attempts": 1}


def test_is_refusal_knows_the_fixed_messages():
    assert is_refusal(REFUSAL) and is_refusal(NOT_FOUND) and is_refusal(UNAVAILABLE)
    assert is_refusal("I don’t know.") and is_refusal("  i DON'T KNOW based on the documents")
    assert not is_refusal(WRONG_SEARCH[-1]) and not is_refusal("")


def test_the_refusal_alarm():
    normal = refusal_rate(WORKING_SEARCH)
    assert round(normal, 3) == round(1 / 56, 3)
    assert round(refusal_rate(WRONG_SEARCH), 3) == round(51 / 56, 3)
    assert refusal_rate([]) == 0.0
    assert refusal_alarm(WRONG_SEARCH, normal)          # 1 → 51 of 56: search broke, and nothing errored
    assert not refusal_alarm(WORKING_SEARCH, normal)
    assert not refusal_alarm([REFUSAL] * 3 + ["ok"] * 53, normal)  # a few more refusals: noise, not a break


def test_answer_defended_never_shows_an_error_page():
    def timing_out(times):
        state = {"calls": 0}
        def search(q):
            state["calls"] += 1
            if state["calls"] <= times:
                raise TimeoutError("search timed out")
            return ["An AI agent is a system that uses a language model…"]
        return search, state
    model_calls = []
    answer = lambda q, ps: model_calls.append(ps) or "An AI agent is… [ai_agents]"  # noqa: E731

    search, state = timing_out(5)
    assert answer_defended("What is an AI agent?", search, answer) == UNAVAILABLE and state["calls"] == 2
    search, state = timing_out(1)
    assert answer_defended("What is an AI agent?", search, answer) == "An AI agent is… [ai_agents]"
    assert state["calls"] == 2 and len(model_calls) == 1
    assert answer_defended("What is an AI agent?", lambda q: [], answer) == NOT_FOUND and len(model_calls) == 1