"""Tests from Lab 07: the agent's work log, the structured summary and replies are real recorded gpt-5.4-nano runs; the skills and requests are the lab's. Run: pytest test_session_practice.py""" from session_practice import PLACEHOLDER, clear_tool_output, lab_grade, listing, missing_sections, pick_rate, strict_grade # The 16-message work log, each text cut to 70 characters LOG = [ { "role": "user", "text": "Payments webhook retries are hammering the partner API. Can you find o" }, { "role": "assistant", "text": "I'll start with the webhook worker logs. [calls read_logs(service='web" }, { "role": "tool", "text": "2026-03-09T14:10:05Z webhook-worker retry attempt=1 partner=acme-bank " }, { "role": "assistant", "text": "Retries fire within milliseconds of each other. Let me read the retry " }, { "role": "tool", "text": " # helper 1: normalizes partner response codes before retry schedul" }, { "role": "assistant", "text": "Found it: in retry.py line 88, BACKOFF_MULTIPLIER is 0, so every retry" }, { "role": "tool", "text": "OK: retry.py updated (1 line changed)" }, { "role": "assistant", "text": "Adding a regression test for the backoff. [calls edit_file('tests/test" }, { "role": "tool", "text": "OK: tests/test_retry.py updated (24 lines added)" }, { "role": "assistant", "text": "Running the test suite. [calls run_tests()]" }, { "role": "tool", "text": "tests/test_payments.py::test_case_1 PASSED\ntests/test_payments.py::tes" }, { "role": "assistant", "text": "Two retry tests fail because the dead-letter threshold is still read f" }, { "role": "user", "text": "Sounds good. Before that, what's the error rate in staging now?" }, { "role": "assistant", "text": "Checking staging metrics. [calls query_metrics(service='webhook-worker" }, { "role": "tool", "text": "{\"service\": \"webhook-worker\", \"env\": \"staging\", \"series\": [\n {\"minute" }, { "role": "assistant", "text": "Staging error rate is down to 0.8% since the backoff fix." } ] STRUCTURED_SUMMARY = '- Goal: what the user asked for \n - Find out why Payments webhook retries are hammering the partner API.\n\n- Rules from the user: anything they said to do or not do \n - Don’t touch anything in the migrations/ folder (another team owns it).\n\n- Findings: what was discovered, with exact locations and values \n - `retry.py` line 88: `BACKOFF_MULTIPLIER = 0 # TODO tune` \n - This causes `next_delay(attempt: int)` to return `BASE_DELAY * (0 ** attempt)`, resulting in effectively no backoff (retries fire immediately).\n - Webhook worker logs show rapid repeated retries with `status=503` and `delay_ms=0` / very small delays:\n - Example: `2026-03-09T14:10:05Z ... retry attempt=1 ... delay_ms=0 status=503`\n - Example: `2026-03-09T14:10:25Z ... retry attempt=2 ... delay_ms=0 status=503`\n - Example: `2026-03-09T14:12:05Z ... retry attempt=2 ... delay_ms=3 status=503`\n - Test failures indicate dead-letter/max-attempts behavior is still configured to 3:\n - `tests/test_retry.py::test_dead_letter_after_max_attempts` failed: expected dead-letter after 8 attempts, found after 3\n - `tests/test_retry.py::test_max_attempts_config` failed: expected `max_attempts == 8`, found `max_attempts == 3`\n\n- Changes made: every file changed and how \n - `retry.py` \n - Updated `BACKOFF_MULTIPLIER` at line 88 from `0` to `2` to enable non-zero backoff between retries.\n - `tests/test_retry.py`' SECTIONS = ["Goal", "Rules from the user", "Findings", "Changes made", "Current state", "Next step"] SKILLS = [ { "name": "api-design-review", "description": "Use when designing or reviewing a REST API endpoint, its URLs, request and response formats, or error handling." }, { "name": "customer-email", "description": "Use when drafting an email or message to a customer, for example about an outage, a refund, a delay, or a complaint." }, { "name": "incident-postmortem", "description": "Use when writing a post-mortem or incident review after an outage, bug, or production incident." }, { "name": "python-code-review", "description": "Use when reviewing Python code or a pull request with Python changes." }, { "name": "release-notes", "description": "Use when writing release notes or a changelog entry for a new version of a product or library." }, { "name": "sql-review", "description": "Use when reviewing, checking, or optimizing a SQL query someone wrote, for correctness or performance." } ] PICKS = [ { "expected": "incident-postmortem", "good": "incident-postmortem", "vague": "incident-postmortem", "coded": "release-notes" }, { "expected": "incident-postmortem", "good": "incident-postmortem", "vague": "incident-postmortem", "coded": "api-design-review" }, { "expected": "release-notes", "good": "release-notes", "vague": "release-notes", "coded": "release-notes" }, { "expected": "release-notes", "good": "release-notes", "vague": "release-notes", "coded": "release-notes" }, { "expected": "sql-review", "good": "sql-review", "vague": "sql-review", "coded": "python-code-review" }, { "expected": "sql-review", "good": "sql-review", "vague": "sql-review", "coded": "python-code-review" }, { "expected": "customer-email", "good": "customer-email", "vague": "customer-email", "coded": "customer-email" }, { "expected": "customer-email", "good": "customer-email", "vague": "customer-email", "coded": "customer-email" }, { "expected": "python-code-review", "good": "python-code-review", "vague": "python-code-review", "coded": "python-code-review" }, { "expected": "python-code-review", "good": "python-code-review", "vague": "python-code-review", "coded": "api-design-review" }, { "expected": "api-design-review", "good": "api-design-review", "vague": "api-design-review", "coded": "python-code-review" }, { "expected": "api-design-review", "good": "api-design-review", "vague": "api-design-review", "coded": "python-code-review" }, { "expected": "none", "good": "none", "vague": "none", "coded": "none" }, { "expected": "none", "good": "none", "vague": "none", "coded": "none" } ] FAILING = ["test_dead_letter_after_max_attempts", "test_max_attempts_config"] def test_clear_old_tool_output_keep_everything_else(): out = clear_tool_output(LOG, keep_recent=4) assert len(out) == 16 assert [m["role"] for m in out] == [m["role"] for m in LOG] cleared = [i for i, m in enumerate(out) if m["text"] == PLACEHOLDER] assert cleared == [2, 4, 6, 8, 10] # the five older tool outputs assert out[12:] == LOG[12:] # the last 4 messages, untouched assert out[5]["text"].startswith("Found it: in retry.py line 88") # the agent's own finding survives assert LOG[2]["text"] != PLACEHOLDER # the input wasn't changed def test_the_structured_summary_never_finished(): assert missing_sections(STRUCTURED_SUMMARY, SECTIONS) == ["Current state", "Next step"] assert STRUCTURED_SUMMARY.rstrip().endswith("`tests/test_retry.py`") # cut off mid-list at max_tokens=400 def test_a_false_pass_and_a_hedged_pass(): drop_older = 'I don’t know from the log you provided. The snippet only shows the staging error rate and mentions a “backoff fix,” but it doesn’t include the root-cause analysis for the retry storm.' assert lab_grade(drop_older, ["backoff"]) is True # the lab counted this ✓ assert strict_grade(drop_older, ["backoff"]) is False hedged = 'I don’t know which tests are still failing from the log you provided. The only test failures mentioned earlier were:\n\n- `tests/test_retry.py::test_dead_letter_after_max_attempts`\n- `tests/test_retry.py::test_max_attempts_config`' assert lab_grade(hedged, FAILING) and not strict_grade(hedged, FAILING) assert strict_grade('The still-failing tests are:\n\n- `tests/test_retry.py::test_dead_letter_after_max_attempts`\n- `tests/test_retry.py::test_max_attempts_config`', FAILING) is True def test_only_descriptions_stay_in_the_prompt(): text = listing(SKILLS) assert text.splitlines()[0] == "- api-design-review: " + SKILLS[0]["description"] assert len(text.splitlines()) == 6 def test_names_and_descriptions_decide_the_pick(): assert pick_rate(PICKS, "good") == (14, 14) assert pick_rate(PICKS, "vague") == (14, 14) # names like sql-review gave it away assert pick_rate(PICKS, "coded") == (7, 14) # vague descriptions + code names