diff --git a/dokimi/changelog.d/20261105_120000_napetrov_novelty-review-controls.md b/dokimi/changelog.d/20261105_120000_napetrov_novelty-review-controls.md new file mode 100644 index 000000000..08c5f0db3 --- /dev/null +++ b/dokimi/changelog.d/20261105_120000_napetrov_novelty-review-controls.md @@ -0,0 +1,12 @@ +### Added + +- **Novelty review controls.** These are the checks a reviewer asked of the novelty 2x2. + - `paired` now reports `added_over_placebo` when the `placebo` arm ran: right with the artifact, wrong without it, and wrong with the placebo. + - A new `no_artifact_repeat` arm asks the baseline again as a new cache draw. With it, `paired` reports `noise_floor`, `added_net_of_noise` and `degraded_within_noise`. + - `novelty analyze` and `novelty probe` take `--context-framing reference|neutral`, which controls for placebo abstention caused by the prompt. + +### Changed + +- **`ci95_added` resamples atoms rather than probes** (`ci_cluster: "atom"`). Paraphrases of one atom are correlated, so the old interval was too narrow. +- The `shuffle_text` docstring no longer cites the study's shuffled-arm win, because that arm injected more files than `full` did. +- `prompts_sha256` now also covers the two context preambles, so its value differs from earlier runs. diff --git a/dokimi/docs/reference/cli-interface.json b/dokimi/docs/reference/cli-interface.json index ce188e945..35c54a095 100644 --- a/dokimi/docs/reference/cli-interface.json +++ b/dokimi/docs/reference/cli-interface.json @@ -2602,7 +2602,7 @@ "nargs": null, "position": 0, "required": true, - "type": "pathlib._local.Path#661d9498960e" + "type": "pathlib.Path#683f66497b37" }, { "action": "_StoreAction", @@ -2650,7 +2650,7 @@ "nargs": null, "position": null, "required": true, - "type": "pathlib._local.Path#661d9498960e" + "type": "pathlib.Path#683f66497b37" }, { "action": "_StoreAction", @@ -9220,6 +9220,25 @@ "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" }, + { + "action": "_StoreAction", + "choices": [ + "'neutral'", + "'reference'" + ], + "const": null, + "default": "reference", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--context-framing", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreTrueAction", "choices": null, @@ -10221,6 +10240,25 @@ "required": false, "type": "dokimi.cli.build_parser..positive_int#faf57db5ff9f" }, + { + "action": "_StoreAction", + "choices": [ + "'neutral'", + "'reference'" + ], + "const": null, + "default": "reference", + "group_key": null, + "group_required": null, + "hidden": false, + "is_flag": false, + "key": "--context-framing", + "kind": "option", + "nargs": null, + "position": null, + "required": false, + "type": null + }, { "action": "_StoreAction", "choices": null, diff --git a/dokimi/docs/skill-novelty.md b/dokimi/docs/skill-novelty.md index 4c717c03b..ad6e2e2e3 100644 --- a/dokimi/docs/skill-novelty.md +++ b/dokimi/docs/skill-novelty.md @@ -818,6 +818,42 @@ of writing its own. Read the output as a statement about the *measurement*, not the atom: a non-discriminating atom can be true and simply universal. A **negative** `r` is the one to treat as a defect. +## Is `added` above placebo and above noise + +Three controls bound the headline `added` rate. Each one is added to the `paired` block +only when its arm ran, so a two-arm run reads exactly as before. + +- **Placebo-gated `added`** (`added_over_placebo`, needs the `placebo` arm). This counts a + probe only when it is right with the artifact, wrong without it, *and* wrong with the + placebo text. A gain the placebo also gets is a context effect, not knowledge. +- **Noise floor** (`noise_floor`, needs the `no_artifact_repeat` arm). That arm asks every + baseline question again as a new sample (a new cache `draw`). Its flips against the + + baseline are what `added` and `degraded` would be if the artifact did nothing. + - `added_net_of_noise` subtracts the floor's `added` from the measured one. + - `degraded_within_noise` is `true` when `degraded` is no larger than the floor's own + `degraded`. Do not report such a `degraded` as a defect. + - In the review rerun the floor was 5–10% in each direction, and every model's + `degraded` fell inside it. + - Both are point estimates; the floor carries its own `ci95_added` to read them against. + The floor is only as wide as the answerer's sampling: at temperature 0 it can read 0. + An injected transport is not given the `draw`, so there the repeat is no new sample. +- **Atom-clustered interval.** The paraphrases of one atom succeed or fail together, so + `ci95_added` (and the placebo-gated `ci95`) resamples atoms, not probes + (`ci_cluster: "atom"`). A probe-level bootstrap reads three correlated wordings as three + data points, so its interval comes out too narrow. A record without `ci_cluster` predates this + change and its interval is probe-level. + +`--context-framing neutral` changes the preamble from "Reference material:" to "Below is a +document. It may or may not be relevant to the question." Use it to check whether placebo +abstention is caused by the prompt. In the rerun the framing did not move the `full` arm on +any model (±0.02). With the placebo, one model's abstention fell from 0.845 to 0.703. The +framing is recorded in the run config and is covered by `prompts_sha256`. + +```bash +python cli.py novelty analyze --artifact skill:s --model M --arms no_artifact,full,placebo,no_artifact_repeat --placebo-text other-domain.md --context-framing neutral +``` + ## Comparing knowledge: which matcher is allowed to answer Two steps ask "is this the same knowledge?" — the retention oracle ("does the shortened text still diff --git a/dokimi/dokimi/commands/novelty.py b/dokimi/dokimi/commands/novelty.py index dd95f2b13..3c5b05ee2 100644 --- a/dokimi/dokimi/commands/novelty.py +++ b/dokimi/dokimi/commands/novelty.py @@ -84,7 +84,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: n_arms = len(_arms(args.arms)) estimate["expected_usd"] = _estimate_usd( args, n_tokens, estimate["estimate"]["by_role"], settings.tier, - context_share=(n_arms - 1) / n_arms, + context_share=(n_arms - 1 - ("no_artifact_repeat" in args.arms)) / n_arms, ) # Every arm but the baseline is priced as carrying the artifact; a placebo of another # length is not read. @@ -119,7 +119,7 @@ def cmd_novelty_analyze(args: argparse.Namespace) -> None: budgets=tuple(args.budgets), concurrency=args.concurrency, allow_self_judge=args.allow_self_judge, - placebo_text=placebo_text, + placebo_text=placebo_text, context_framing=args.context_framing, cache_dir=Path(args.cache_dir) if args.cache_dir else None, seed=args.seed, include_resources=args.include_resources, @@ -227,7 +227,7 @@ def cmd_novelty_probe(args: argparse.Namespace) -> None: atoms, probe_set.probes, verdict.text, answerer=answerer, judge=judge, arms=_arms(args.arms), concurrency=args.concurrency, allow_self_judge=args.allow_self_judge, seed=args.seed, - placebo_text=placebo_text, + placebo_text=placebo_text, framing=args.context_framing, ) except ValueError as exc: # a model failure under way raise SystemExit(str(exc)) from None @@ -814,7 +814,12 @@ def _add_model_args(parser: argparse.ArgumentParser, positive_int) -> None: help="Output-token ceiling. Raise for reasoning models, which can " "spend the provider default on reasoning and return empty text") parser.add_argument("--arms", default="no_artifact,full", - help="Arms to run: no_artifact,full[,placebo][,shuffled]") + help="Arms to run: no_artifact,full[,placebo][,shuffled]" + "[,no_artifact_repeat] (the last is the noise floor)") + parser.add_argument("--context-framing", default="reference", dest="context_framing", + choices=("reference", "neutral"), + help="How the context is introduced: as reference material, or as a " + "document that may be irrelevant (the placebo framing control)") parser.add_argument("--placebo-text", default=None, dest="placebo_text", help="File of domain-mismatched text for the placebo control: it " "separates 'this artifact carries knowledge' from 'this model " diff --git a/dokimi/dokimi/novelty/analyze.py b/dokimi/dokimi/novelty/analyze.py index dd521a403..ceaee9a6a 100644 --- a/dokimi/dokimi/novelty/analyze.py +++ b/dokimi/dokimi/novelty/analyze.py @@ -56,6 +56,7 @@ class AnalyzeConfig: # cited file, so a run that skips this must not present attribution as confirmed. confirm_carriers: bool = DEFAULT_SETTINGS["confirm_carriers"] placebo_text: str | None = None + context_framing: str = "reference" cache_dir: Path | None = None seed: int = 0 # Measured over the study corpus, 0.72 of a skill's words live outside `SKILL.md`, so @@ -81,7 +82,10 @@ def _prompts_sha256() -> str: def _run_config(config: "AnalyzeConfig") -> dict: """What decided this cell's numbers beyond the models: two cells compare only if these match.""" + framing = {} if config.context_framing == "reference" else { + "context_framing": config.context_framing} return { + **framing, "seed": config.seed, "max_atoms": config.max_atoms, "paraphrases": config.paraphrases, @@ -244,7 +248,7 @@ def analyze( atom_list, probe_list, text, answerer=answerer, judge=judge, arms=config.arms, concurrency=config.concurrency, allow_self_judge=config.allow_self_judge, - placebo_text=config.placebo_text, seed=config.seed, + placebo_text=config.placebo_text, seed=config.seed, framing=config.context_framing, ) if config.allow_self_judge and same_client(judge, answerer): warnings.append( diff --git a/dokimi/dokimi/novelty/controls.py b/dokimi/dokimi/novelty/controls.py new file mode 100644 index 000000000..208927f86 --- /dev/null +++ b/dokimi/dokimi/novelty/controls.py @@ -0,0 +1,127 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Controls that bound the novelty 2x2: placebo gating, a noise floor, atom-clustered CIs. + +Each answers a reviewer question the raw ``added`` rate cannot: is the gain just context +(placebo), is it above run-to-run noise (a repeat of the baseline), and how wide is the +interval when the paraphrases of one atom are not independent (resample atoms). +""" + +from __future__ import annotations + +import random + +# A second no-artifact answer of the same probe, sampled as a new draw. Its flips against +# the baseline are what ``added`` and ``degraded`` would be if the artifact did nothing. +REPEAT_ARM = "no_artifact_repeat" +PLACEBO_ARM = "placebo" + + +def _key(run) -> tuple[str, str, str]: + return (run.atom_id, run.probe_kind, run.question) + + +def _usable(runs, arm: str) -> dict: + return {_key(r): r for r in runs + if r.arm == arm and r.probe_kind != "reversal" and not r.errored} + + +def clustered_ci( + per_atom: dict[str, list[float]], *, iterations: int = 400, seed: int = 0 +) -> list[float] | None: + """Percentile bootstrap of a probe rate that resamples atoms, not probes. + + The paraphrases of one atom succeed or fail together, so a probe-level bootstrap + reads three correlated wordings as three data points and comes out too narrow. + """ + clusters = [values for values in per_atom.values() if values] + if len(clusters) < 5: + return None + rng = random.Random(f"novelty-ci-{seed}") + n = len(clusters) + means = [] + for _ in range(iterations): + picked = [clusters[rng.randrange(n)] for _ in range(n)] + total = sum(len(c) for c in picked) + means.append(sum(sum(c) for c in picked) / total) + means.sort() + lo = means[int(0.025 * iterations)] + hi = means[min(iterations - 1, int(0.975 * iterations))] + return [round(lo, 4), round(hi, 4)] + + +def placebo_gated(runs, *, baseline_arm: str, with_arm: str, scoring: str) -> dict | None: + """``added`` that also beats the placebo: right with the artifact, wrong without it, + and wrong with domain-mismatched text of the same length. + + ``None`` when no placebo arm ran. Only probes answered in all three arms count. + """ + placebo = _usable(runs, PLACEBO_ARM) + if not placebo or with_arm == PLACEBO_ARM: + return None + baseline, with_runs = _usable(runs, baseline_arm), _usable(runs, with_arm) + per_atom: dict[str, list[float]] = {} + for key, run in with_runs.items(): + before, ctrl = baseline.get(key), placebo.get(key) + if before is None or ctrl is None: + continue + gated = run.correct(scoring) and not before.correct(scoring) and not ctrl.correct(scoring) + per_atom.setdefault(run.atom_id, []).append(1.0 if gated else 0.0) + n = sum(len(v) for v in per_atom.values()) + count = int(sum(sum(v) for v in per_atom.values())) + return { + "n_paired": n, + "count": count, + "rate": round(count / n, 4) if n else 0.0, + "ci95": clustered_ci(per_atom), + } + + +def noise_floor(runs, *, baseline_arm: str, scoring: str) -> dict | None: + """Flip rates between two no-artifact answers of the same probe. + + ``added`` here is wrong-then-right and ``degraded`` right-then-wrong with no artifact + at all; a with-artifact rate inside this floor is not evidence. ``None`` without the + repeat arm. + """ + repeat = _usable(runs, REPEAT_ARM) + if not repeat: + return None + baseline = _usable(runs, baseline_arm) + added = degraded = n = 0 + flips: dict[str, list[float]] = {} + for key, run in repeat.items(): + before = baseline.get(key) + if before is None: + continue + n += 1 + first, second = before.correct(scoring), run.correct(scoring) + added += second and not first + degraded += first and not second + flips.setdefault(run.atom_id, []).append(1.0 if second and not first else 0.0) + if not n: + return None + return { + "n_paired": n, + "draws": 1, + "added": round(added / n, 4), + "degraded": round(degraded / n, 4), + "ci95_added": clustered_ci(flips), + } + + +def control_fields(runs, *, baseline_arm: str, with_arm: str, scoring: str, + rates: dict) -> dict: + """The control keys :func:`~dokimi.novelty.measure.paired_outcomes` adds when the + placebo or the repeat arm ran; empty otherwise, so old outputs are unchanged.""" + fields: dict = {} + gated = placebo_gated(runs, baseline_arm=baseline_arm, with_arm=with_arm, scoring=scoring) + if gated is not None: + fields["added_over_placebo"] = gated + floor = noise_floor(runs, baseline_arm=baseline_arm, scoring=scoring) + if floor is not None: + fields["noise_floor"] = floor + fields["added_net_of_noise"] = round(rates["added"] - floor["added"], 4) + fields["degraded_within_noise"] = rates["degraded"] <= floor["degraded"] + return fields diff --git a/dokimi/dokimi/novelty/measure.py b/dokimi/dokimi/novelty/measure.py index 78b19491a..8d4e09b33 100644 --- a/dokimi/dokimi/novelty/measure.py +++ b/dokimi/dokimi/novelty/measure.py @@ -30,6 +30,7 @@ from dokimi.utils import InputError from .atoms import Atom from .client import LLMClient, same_client +from .controls import REPEAT_ARM, clustered_ci, control_fields from .probes import Probe # Judge verdict scale. Deliberately finer than the collapses below so a rater @@ -55,7 +56,11 @@ # controls exist because a with-artifact gain can otherwise be a prompt-length artifact # (placebo) or a narrative that is not actually load-bearing (shuffled). BASELINE_ARM = "no_artifact" -ARMS = (BASELINE_ARM, "full", "placebo", "shuffled") +# ``no_artifact_repeat`` answers the baseline again as a new draw: its noise floor. +ARMS = (BASELINE_ARM, "full", "placebo", "shuffled", REPEAT_ARM) +# How the context is introduced; ``neutral`` says it may be irrelevant, the control for +# placebo abstention caused by the "reference material" wording. +FRAMINGS = ("reference", "neutral") _ANSWER_SYSTEM = ( "Answer the question directly and concisely. If you do not know, say you do not know " @@ -252,6 +257,7 @@ def run_measurement( allow_self_judge: bool = False, placebo_text: str | None = None, seed: int = 0, + framing: str = "reference", ) -> MeasureResult: """Answer every probe in every arm and grade the answers. @@ -269,12 +275,16 @@ def run_measurement( far *below* the no-artifact baseline on two models (0.104 vs 0.352), i.e. irrelevant context actively harms them. seed: Seed for the ``shuffled`` control, so the control is reproducible. + framing: One of :data:`FRAMINGS`; how the context is introduced in the prompt. """ check_self_judge(judge, answerer, allow_self_judge) check_arms(arms, placebo_text) + if framing not in FRAMINGS: + raise InputError(f"unknown context framing {framing!r}; known: {list(FRAMINGS)}") contexts = { BASELINE_ARM: None, + REPEAT_ARM: None, "full": artifact_text, "placebo": placebo_text, "shuffled": shuffle_text(artifact_text, seed=seed), @@ -284,7 +294,10 @@ def run_measurement( def one(job: tuple[Probe, str]) -> ProbeRun: probe, arm = job try: - response, confidence = _answer(probe.question, contexts.get(arm), answerer) + response, confidence = _answer( + probe.question, contexts.get(arm), answerer, + framing=framing, draw=1 if arm == REPEAT_ARM else 0, + ) except Exception as exc: # a provider failure is a missing datum, not a wrong answer response, confidence = f"", None verdict = ERRORED @@ -302,15 +315,27 @@ def one(job: tuple[Probe, str]) -> ProbeRun: return MeasureResult(runs=runs, arms=tuple(arms), with_arm="full") -def _answer(question: str, context: str | None, answerer: LLMClient) -> tuple[str, float | None]: - """Answer one probe. With ``context=None`` the model sees only the question.""" +# Named ``*_PROMPT`` so the run's ``prompts_sha256`` covers them. +_CONTEXT_REFERENCE_PROMPT = "Reference material:" +_CONTEXT_NEUTRAL_PROMPT = "Below is a document. It may or may not be relevant to the question." +_CONTEXT_PREAMBLES = {"reference": _CONTEXT_REFERENCE_PROMPT, "neutral": _CONTEXT_NEUTRAL_PROMPT} + + +def _answer( + question: str, context: str | None, answerer: LLMClient, + *, framing: str = "reference", draw: int = 0, +) -> tuple[str, float | None]: + """Answer one probe. With ``context=None`` the model sees only the question. + + ``draw`` > 0 asks for a fresh sample of a prompt already answered (the repeat arm). + """ if context: # The artifact goes in as a stable cache prefix-shaped preamble; the question is # unchanged between arms so the pair differs by context alone. - prompt = f"Reference material:\n\n{context}\n\n---\n\nQuestion: {question}" + prompt = f"{_CONTEXT_PREAMBLES[framing]}\n\n{context}\n\n---\n\nQuestion: {question}" else: prompt = f"Question: {question}" - reply = answerer.text(prompt, system=_ANSWER_SYSTEM, accept=_is_complete_answer) + reply = answerer.text(prompt, system=_ANSWER_SYSTEM, accept=_is_complete_answer, draw=draw) if not reply.strip(): # No answer is no datum: graded, it would read as a wrong one. A reasoning model # that spends its output ceiling before answering returns this. @@ -406,6 +431,8 @@ def paired_outcomes( total = sum(counts.values()) rates = {k: round(v / total, 4) if total else 0.0 for k, v in counts.items()} k0_rows = [r for r in baseline.values()] + added_by_atom = {a: [1.0 if n == "added" else 0.0 for n in names] + for a, names in per_atom.items()} return { "scoring": scoring, "with_arm": with_arm, @@ -414,10 +441,13 @@ def paired_outcomes( "counts": counts, **rates, "k0": round(sum(r.correct(scoring) for r in k0_rows) / len(k0_rows), 4) if k0_rows else 0.0, - "ci95_added": _bootstrap_ci([1.0 if n == "added" else 0.0 - for atom in per_atom.values() for n in atom]), + # Atoms are resampled: the paraphrases of one atom are not independent probes. + "ci95_added": clustered_ci(added_by_atom), + "ci_cluster": "atom", "atoms_added": sorted(a for a, names in per_atom.items() if "added" in names), "atoms_degraded": sorted(a for a, names in per_atom.items() if "degraded" in names), + **control_fields(runs, baseline_arm=BASELINE_ARM, with_arm=with_arm, + scoring=scoring, rates=rates), } @@ -535,11 +565,9 @@ def shuffle_text(text: str, *, seed: int = 0) -> str: Seeded from a string rather than :func:`hash`, because Python salts string hashing per process and a control that changes between runs is not a control. - Expect this arm to score at least as high as ``full``: it did on four of five models in - the study, and a behavioural retest on 71 tasks whose graded requirements *name* an - ordering kept the two level (+0.022 on the ordering items, 95% CI -0.027 to +0.072). A - probe cannot punish a scrambled document, so treat a ``shuffled`` win as the expected - reading and not as a bug in the shuffle. + A probe asks for one fact, so it cannot punish a scrambled document: expect this arm + near ``full``. The study's shuffled arm scored higher, but it injected more files + than ``full`` did, so that result does not carry over to this arm. """ from .spans import segment @@ -560,18 +588,3 @@ def _pearson(xs: list[float], ys: list[float]) -> float | None: return None return round(sum(a * b for a, b in zip(dx, dy, strict=True)) / den, 4) - -def _bootstrap_ci( - values: list[float], *, iterations: int = 400, seed: int = 0 -) -> list[float] | None: - """Percentile bootstrap CI, so a rate from 40 probes is not read like one from 400.""" - if len(values) < 5: - return None - rng = random.Random(f"novelty-ci-{seed}") - n = len(values) - means = sorted( - sum(values[rng.randrange(n)] for _ in range(n)) / n for _ in range(iterations) - ) - lo = means[int(0.025 * iterations)] - hi = means[min(iterations - 1, int(0.975 * iterations))] - return [round(lo, 4), round(hi, 4)] diff --git a/dokimi/scripts/growth_baseline.json b/dokimi/scripts/growth_baseline.json index 9162bbadb..48aa3da4a 100644 --- a/dokimi/scripts/growth_baseline.json +++ b/dokimi/scripts/growth_baseline.json @@ -1,6 +1,6 @@ { "files": { - "dokimi/commands/novelty.py": {"lines": 1021, "rationale": "estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, + "dokimi/commands/novelty.py": {"lines": 1026, "rationale": "--context-framing and the no_artifact_repeat arm help (review controls); estimate-only projection and W1 routing cost output (#164) from main, plus the #172 routing field, --all-vs-all, tied/unscored caveat and blind-skill warning, --all-vs-all with --repeats (noise and cost block); routing output in commands/novelty_routing_report.py; file split tracked separately"}, "dokimi/harnesses/benchflow.py": {"lines": 869}, "dokimi/harnesses/docker_solver.py": {"lines": 897, "rationale": "pre-existing when the gate was introduced"}, "dokimi/llm.py": {"lines": 1032, "rationale": "grew on main in #180 (replay a stored retry) after #93 seeded this baseline; split it in its own PR"}, diff --git a/dokimi/tests/test_novelty_controls.py b/dokimi/tests/test_novelty_controls.py new file mode 100644 index 000000000..f2a32ea79 --- /dev/null +++ b/dokimi/tests/test_novelty_controls.py @@ -0,0 +1,111 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Placebo gating, the noise floor, atom-clustered CIs and context framing.""" + +import json + +import pytest + +from dokimi.novelty.atoms import Atom +from dokimi.novelty.client import LLMClient +from dokimi.novelty.controls import clustered_ci +from dokimi.novelty.measure import ProbeRun, paired_outcomes, run_measurement +from dokimi.novelty.probes import Probe +from dokimi.utils import InputError + +RIGHT, WRONG = "correct_complete", "wrong_other" + + +def _run(atom_id: str, arm: str, verdict: str, question: str = "q") -> ProbeRun: + return ProbeRun(atom_id=atom_id, probe_kind="direct", arm=arm, question=question, + response="r", verdict=verdict) + + +def test_added_over_placebo_drops_gains_the_placebo_also_gets(): + runs = [ + _run("a", "no_artifact", WRONG), _run("a", "full", RIGHT), _run("a", "placebo", WRONG), + _run("b", "no_artifact", WRONG), _run("b", "full", RIGHT), _run("b", "placebo", RIGHT), + ] + paired = paired_outcomes(runs) + assert paired["added"] == 1.0 + assert paired["added_over_placebo"]["count"] == 1 + assert paired["added_over_placebo"]["rate"] == 0.5 + assert paired["added_over_placebo"]["n_paired"] == 2 + + +def test_without_control_arms_the_output_has_no_control_keys(): + paired = paired_outcomes([_run("a", "no_artifact", WRONG), _run("a", "full", RIGHT)]) + assert "added_over_placebo" not in paired + assert "noise_floor" not in paired + assert paired["ci_cluster"] == "atom" + + +def test_noise_floor_nets_added_and_flags_degraded_inside_it(): + # With no artifact, a0-a1 flip wrong->right and a9 right->wrong between the two draws. + base = [WRONG] * 8 + [RIGHT, RIGHT] + repeat = [RIGHT, RIGHT] + [WRONG] * 6 + [RIGHT, WRONG] + full = [RIGHT] * 5 + [WRONG] * 3 + [RIGHT, WRONG] + runs = [_run(f"a{i}", arm, verdicts[i]) + for arm, verdicts in (("no_artifact", base), ("no_artifact_repeat", repeat), + ("full", full)) + for i in range(10)] + paired = paired_outcomes(runs) + floor = paired["noise_floor"] + assert (floor["n_paired"], floor["added"], floor["degraded"]) == (10, 0.2, 0.1) + assert floor["draws"] == 1 and floor["ci95_added"] is not None + assert paired["added"] == 0.5 + assert paired["added_net_of_noise"] == 0.3 + assert paired["degraded"] == 0.1 + assert paired["degraded_within_noise"] is True + + +def test_clustered_ci_is_wider_than_treating_paraphrases_as_independent(): + # Each atom's three paraphrases succeed or fail together. + per_atom = {f"a{i}": [1.0] * 3 if i % 2 else [0.0] * 3 for i in range(10)} + flat = {f"p{i}": [v] for i, v in enumerate(x for vs in per_atom.values() for x in vs)} + lo, hi = clustered_ci(per_atom) + flat_lo, flat_hi = clustered_ci(flat) + assert hi - lo > flat_hi - flat_lo + assert clustered_ci({"a": [1.0], "b": [0.0]}) is None + + +def _clients(seen): + def answer(prompt, system): + seen.append(prompt) + return "x\nconfidence: 0.5", None + judge = LLMClient(model="b", transport=lambda p, s: (json.dumps({"verdict": RIGHT}), None)) + return LLMClient(model="a", transport=answer), judge + + +def _one_probe(): + atom = Atom(statement="s", question="Q?", answer="yes", span_ids=("s0001",)) + return [atom], [Probe(atom_id=atom.id, kind="direct", question="Q?", expected="yes")] + + +def test_neutral_framing_says_the_text_may_be_irrelevant(): + seen: list[str] = [] + answerer, judge = _clients(seen) + atoms, probes = _one_probe() + run_measurement(atoms, probes, "DOC", answerer=answerer, judge=judge, + concurrency=1, framing="neutral") + with_context = [p for p in seen if "DOC" in p] + assert with_context and with_context[0].startswith("Below is a document. It may or may not") + with pytest.raises(InputError, match="framing"): + run_measurement(atoms, probes, "DOC", answerer=answerer, judge=judge, framing="odd") + + +def test_repeat_arm_asks_the_baseline_again_as_a_new_draw(monkeypatch): + draws: list[tuple[str, int]] = [] + + def transport(self, prompt, system, *, accept=None, draw=0): + draws.append((prompt, draw)) + return "x\nconfidence: 0.5", None + + monkeypatch.setattr(LLMClient, "_default_transport", transport) + judge = LLMClient(model="b", transport=lambda p, s: (json.dumps({"verdict": RIGHT}), None)) + atoms, probes = _one_probe() + result = run_measurement(atoms, probes, "DOC", answerer=LLMClient(model="a"), judge=judge, + concurrency=1, arms=("no_artifact", "full", "no_artifact_repeat")) + assert sorted(d for p, d in draws if "DOC" not in p) == [0, 1] + assert result.paired("lenient")["noise_floor"]["n_paired"] == 1