"""
Practice: from traces to tests. Lab 14's scripts 06 (feedback linked to traces) and 07 (traces
-> tests): tell a complaint from other feedback, sample a stratified test set from traces, gate
a release on what production showed matters, and read the result against its likely range.

Fill in the TODOs, then run the tests:
    pytest test_evals_practice.py        (locally)
or press "Run tests" on the chapter page.
"""
import math
import random

# Feedback kinds the lab stores with each trace id. All but "helpful" count as a complaint.
COMPLAINTS = {"not_helpful", "retry", "edit", "human"}

KINDS = ("what", "how", "list", "none")  # "none": questions the articles don't answer
LONG_TURN = 8                           # a long conversation: turn 8 or later

# The gate (07_traces_to_evals.py): right >= 95%, cites a source >= 90% of the answerable
# questions it got right, long conversations average <= 700 input tokens.
GATE = {"right": 0.95, "cites": 0.90, "long_input_tokens": 700}


def is_complaint(kinds):
    """True if any of the feedback kinds on one request is a complaint (see COMPLAINTS)."""
    # TODO
    return False


def build_evalset(traces, per_stratum=10, seed=7):
    """Sample a test set from a week of traces, as 07_traces_to_evals.py does.

    Each trace is a dict with: trace_id, conversation_id, turn, kind, question, answer,
    incident (None, or the incident's name) and error (bool).

    - Use only clean traces: no incident and no error.
    - For each kind in KINDS, in that order: rng.sample(per_stratum) first questions (turn 1)
      of that kind. Each case: {"from_trace", "question", "kind", "history": [],
      "stratum": "single turn, <kind>"}.
    - Then rng.sample(per_stratum) clean traces at turn >= LONG_TURN. Each case's history is
      the (question, answer) pairs of the same conversation's earlier turns, in turn order,
      leaving out turns that errored; stratum "long conversation (turn 8+)".

    Use one rng = random.Random(seed) for every draw, in that order.
    """
    # TODO
    return []


def wilson(k, n, z=1.96):
    """95% Wilson score interval (lo, hi) for k successes out of n, clipped to [0, 1].
    centre = (p + z²/2n) / (1 + z²/n);  half = z·sqrt(p(1-p)/n + z²/4n²) / (1 + z²/n).
    n == 0 gives (0.0, 1.0)."""
    # TODO
    return 0.0, 1.0


def gate(rows, rule="bar"):
    """Gate one version's results. Each row: {"right", "answerable", "cites", "long", "input_tokens"}.

    Returns {"right": fraction right, "cites": (k, n), "long_tokens": mean input tokens over the
    long rows, "checks": {"right": bool, "cites": bool, "long_tokens": bool}, "pass": all three}.

    Citations are counted on rows that are answerable and right. Check "cites" passes when:
      rule == "bar":   k / n >= GATE["cites"]
      rule == "range": the top of wilson(k, n) >= GATE["cites"]  (fail only if the whole range is below)
    """
    # TODO
    return {}


def chance_at_most(k, n, p):
    """P(X <= k) for X ~ Binomial(n, p): how often a version that truly cites p of the time
    scores k or fewer of n."""
    # TODO
    return 0.0