"""Tests on Lab 17's script 03, from its recorded run (October 2026): what the checklist found in 16 systems, and which items would have caught 12 public failures, by our labels and by gpt-5.4-mini's. Run: pytest test_gate_practice.py""" from gate_practice import agreement, best_system, cases_with, ci_runs_on_change, evidence_counts, find_line ITEMS = ["risk", "model_card", "data", "guardrails", "escalation", "monitoring", "legal", "rollback", "fairness", "local_eval", "human", "change_test"] ROADMAP = ["risk", "model_card", "data", "guardrails", "escalation", "monitoring", "legal", "rollback"] ADDED = [k for k in ITEMS if k not in ROADMAP] # Which items the checker found evidence for in each system, in ITEMS order (1 = found). FOUND = {'Support message sorter': '000000000000', 'Question router': '000000000000', 'FAQ search': '000000010000', 'Help-article assistant': '000000011000', 'Release judge': '000100011001', 'Chat with memory': '000000000000', 'Employee tool assistant': '000000000000', 'Voice assistant': '010100000000', 'Refund agent': '011100000110', 'Refund agent team': '011100010110', 'Ticket triage model': '011100000100', 'Self-hosted models': '010100010100', 'Refund agent, crash-safe': '011100010100', 'Docs assistant, traced': '011101010100', 'Email assistant': '011100000100', 'HR documents assistant': '011100000110'} SYSTEMS = {name: {k: ("found" if bit == "1" else None) for k, bit in zip(ITEMS, bits)} for name, bits in FOUND.items()} # 12 public failures: which items would have caught each, by our labels and the model's. CASES = {'air-canada': {'ours': ['guardrails', 'legal', 'local_eval'], 'model': ['guardrails', 'local_eval', 'human']}, 'amazon-recruiting': {'ours': ['data', 'fairness', 'risk'], 'model': ['data', 'fairness', 'local_eval', 'change_test']}, 'uber-atg': {'ours': ['risk', 'guardrails', 'human', 'escalation'], 'model': ['guardrails', 'human', 'local_eval', 'change_test']}, 'zillow-offers': {'ours': ['monitoring', 'rollback', 'risk'], 'model': ['local_eval', 'monitoring', 'change_test']}, 'dutch-benefits': {'ours': ['fairness', 'data', 'human', 'legal'], 'model': ['fairness', 'local_eval', 'legal']}, 'robodebt': {'ours': ['legal', 'human', 'local_eval'], 'model': ['local_eval', 'human', 'legal']}, 'epic-sepsis': {'ours': ['local_eval', 'model_card', 'monitoring'], 'model': ['local_eval']}, 'care-algorithm': {'ours': ['fairness', 'data', 'model_card'], 'model': ['local_eval', 'fairness']}, 'nyc-mycity': {'ours': ['local_eval', 'legal', 'guardrails'], 'model': ['guardrails', 'local_eval', 'human']}, 'chevy-tahoe': {'ours': ['guardrails', 'monitoring'], 'model': ['guardrails', 'human']}, 'dpd': {'ours': ['change_test', 'guardrails', 'rollback'], 'model': ['guardrails', 'monitoring', 'rollback', 'change_test']}, 'mata-avianca': {'ours': ['human', 'risk'], 'model': ['guardrails', 'human', 'change_test']}} # A README line from the docs assistant (Lab 14) that the first "re-tested" check passed on. TRACE_README = """A **trace** is the spans of one request, as a tree: invoke_workflow support_assistant 1.72 s ├── retrieval docs 0.59 s """ # The start of the release judge's real CI file (trimmed). CI_FILE = """name: Lab 05 evals on: pull_request: paths: - "labs/05_evaluation_systems/**" push: branches: [main] paths: - "labs/05_evaluation_systems/**" """ def test_find_line_returns_the_evidence_not_a_tick(): files = {"README.md": "# Release judge\n\nOwner: nobody yet\n", "app.py": "OWNER = 'x'\n"} assert find_line(files, r"^\**owner\**\s*:") == "README.md: Owner: nobody yet" assert find_line(files, r"^RISK LEVEL:") is None def test_a_keyword_passes_on_a_word(): # The first version of the check matched "workflow" anywhere: a printed trace line passed. hit = find_line({"14_observability/README.md": TRACE_README}, r"workflow") assert hit == "14_observability/README.md: invoke_workflow support_assistant 1.72 s" def test_the_fixed_check_needs_the_ci_file(): wf = {".github/workflows/lab05-evals.yml": CI_FILE} assert ci_runs_on_change(wf, "05_evaluation_systems") == ".github/workflows/lab05-evals.yml: runs on every change" assert ci_runs_on_change(wf, "14_observability") is None manual = {".github/workflows/x.yml": CI_FILE.replace(" pull_request:", " workflow_dispatch:").replace(" push:", " schedule:")} assert ci_runs_on_change(manual, "05_evaluation_systems") is None def test_people_decisions_have_no_evidence_anywhere(): c = evidence_counts(SYSTEMS) assert (c["risk"], c["escalation"], c["legal"]) == (0, 0, 0) assert (c["guardrails"], c["model_card"], c["data"], c["rollback"], c["monitoring"]) == (10, 9, 7, 7, 1) assert (c["fairness"], c["local_eval"], c["human"], c["change_test"]) == (2, 8, 3, 1) def test_no_system_has_all_eight(): assert best_system(SYSTEMS, ROADMAP) == ("Docs assistant, traced", 5) assert sum(all(SYSTEMS[s][k] for k in ROADMAP) for s in SYSTEMS) == 0 def test_the_refund_agent(): c = evidence_counts({"Refund agent": SYSTEMS["Refund agent"]}) assert [k for k in ITEMS if c[k]] == ["model_card", "data", "guardrails", "local_eval", "human"] assert best_system({"Refund agent": SYSTEMS["Refund agent"]}, ROADMAP) == ("Refund agent", 3) def test_agreement_on_toy_raters(): assert agreement([True, False, True, False], [True, False, True, False])["kappa"] == 1 t = agreement([True, True, False, False], [True, False, True, False]) assert t["raw"] == 0.5 and t["chance"] == 0.5 and abs(t["kappa"]) < 1e-9 # Two raters who say no to almost everything agree a lot by habit, and kappa shows it. habit = agreement([False] * 18 + [True, False], [False] * 18 + [False, True]) assert habit["raw"] == 0.9 and habit["kappa"] < 0 def test_80_percent_agreement_is_only_moderate(): a = [k in c["ours"] for c in CASES.values() for k in ITEMS] b = [k in c["model"] for c in CASES.values() for k in ITEMS] assert len(a) == 144 g = agreement(a, b) assert round(g["raw"], 2) == 0.80 assert round(g["chance"], 2) == 0.63 assert round(g["kappa"], 2) == 0.46 def test_every_failure_has_a_roadmap_item_and_ten_need_an_added_one(): ours = [{"caught_by": c["ours"]} for c in CASES.values()] assert cases_with(ours, ROADMAP) == 12 assert cases_with(ours, ADDED) == 10 model = [{"caught_by": c["model"]} for c in CASES.values()] assert cases_with(model, ["risk", "model_card"]) == 0 assert (cases_with(model, ["local_eval"]), cases_with(ours, ["local_eval"])) == (9, 4)