Repository navigation
novelty: placebo-gated added, noise floor, atom-clustered CI, context framing #418
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| ### Added | ||
|
|
||
| - **Novelty review controls.** These are the checks a reviewer asked of the novelty 2x2. | ||
| - `paired` now reports `added_over_placebo` when the `placebo` arm ran: right with the artifact, wrong without it, and wrong with the placebo. | ||
| - A new `no_artifact_repeat` arm asks the baseline again as a new cache draw. With it, `paired` reports `noise_floor`, `added_net_of_noise` and `degraded_within_noise`. | ||
| - `novelty analyze` and `novelty probe` take `--context-framing reference|neutral`, which controls for placebo abstention caused by the prompt. | ||
|
|
||
| ### Changed | ||
|
|
||
| - **`ci95_added` resamples atoms rather than probes** (`ci_cluster: "atom"`). Paraphrases of one atom are correlated, so the old interval was too narrow. | ||
| - The `shuffle_text` docstring no longer cites the study's shuffled-arm win, because that arm injected more files than `full` did. | ||
| - `prompts_sha256` now also covers the two context preambles, so its value differs from earlier runs. | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| # Copyright (C) 2026 Intel Corporation | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
|
|
||
| """Controls that bound the novelty 2x2: placebo gating, a noise floor, atom-clustered CIs. | ||
|
|
||
| Each answers a reviewer question the raw ``added`` rate cannot: is the gain just context | ||
| (placebo), is it above run-to-run noise (a repeat of the baseline), and how wide is the | ||
| interval when the paraphrases of one atom are not independent (resample atoms). | ||
| """ | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import random | ||
|
|
||
| # A second no-artifact answer of the same probe, sampled as a new draw. Its flips against | ||
| # the baseline are what ``added`` and ``degraded`` would be if the artifact did nothing. | ||
| REPEAT_ARM = "no_artifact_repeat" | ||
| PLACEBO_ARM = "placebo" | ||
|
|
||
|
|
||
| def _key(run) -> tuple[str, str, str]: | ||
| return (run.atom_id, run.probe_kind, run.question) | ||
|
|
||
|
|
||
| def _usable(runs, arm: str) -> dict: | ||
| return {_key(r): r for r in runs | ||
| if r.arm == arm and r.probe_kind != "reversal" and not r.errored} | ||
|
Contributor
There was a problem hiding this comment. The guards and the interval bounds that decide reported measurements are untested. Mutation check, each passing the full new test file and the neighbouring novelty tests:
Suggestion: add cases with an errored probe in the placebo and repeat arms, a case with |
||
|
|
||
|
|
||
| def clustered_ci( | ||
| per_atom: dict[str, list[float]], *, iterations: int = 400, seed: int = 0 | ||
| ) -> list[float] | None: | ||
| """Percentile bootstrap of a probe rate that resamples atoms, not probes. | ||
|
|
||
| The paraphrases of one atom succeed or fail together, so a probe-level bootstrap | ||
| reads three correlated wordings as three data points and comes out too narrow. | ||
| """ | ||
| clusters = [values for values in per_atom.values() if values] | ||
| if len(clusters) < 5: | ||
| return None | ||
| rng = random.Random(f"novelty-ci-{seed}") | ||
| n = len(clusters) | ||
| means = [] | ||
| for _ in range(iterations): | ||
| picked = [clusters[rng.randrange(n)] for _ in range(n)] | ||
| total = sum(len(c) for c in picked) | ||
| means.append(sum(sum(c) for c in picked) / total) | ||
| means.sort() | ||
| lo = means[int(0.025 * iterations)] | ||
| hi = means[min(iterations - 1, int(0.975 * iterations))] | ||
| return [round(lo, 4), round(hi, 4)] | ||
|
|
||
|
|
||
| def placebo_gated(runs, *, baseline_arm: str, with_arm: str, scoring: str) -> dict | None: | ||
| """``added`` that also beats the placebo: right with the artifact, wrong without it, | ||
| and wrong with domain-mismatched text of the same length. | ||
|
|
||
| ``None`` when no placebo arm ran. Only probes answered in all three arms count. | ||
| """ | ||
| placebo = _usable(runs, PLACEBO_ARM) | ||
| if not placebo or with_arm == PLACEBO_ARM: | ||
| return None | ||
| baseline, with_runs = _usable(runs, baseline_arm), _usable(runs, with_arm) | ||
| per_atom: dict[str, list[float]] = {} | ||
| for key, run in with_runs.items(): | ||
| before, ctrl = baseline.get(key), placebo.get(key) | ||
| if before is None or ctrl is None: | ||
| continue | ||
| gated = run.correct(scoring) and not before.correct(scoring) and not ctrl.correct(scoring) | ||
| per_atom.setdefault(run.atom_id, []).append(1.0 if gated else 0.0) | ||
| n = sum(len(v) for v in per_atom.values()) | ||
| count = int(sum(sum(v) for v in per_atom.values())) | ||
| return { | ||
| "n_paired": n, | ||
| "count": count, | ||
| "rate": round(count / n, 4) if n else 0.0, | ||
|
Contributor
There was a problem hiding this comment.
With Suggestion: |
||
| "ci95": clustered_ci(per_atom), | ||
| } | ||
|
|
||
|
|
||
| def noise_floor(runs, *, baseline_arm: str, scoring: str) -> dict | None: | ||
| """Flip rates between two no-artifact answers of the same probe. | ||
|
|
||
| ``added`` here is wrong-then-right and ``degraded`` right-then-wrong with no artifact | ||
| at all; a with-artifact rate inside this floor is not evidence. ``None`` without the | ||
| repeat arm. | ||
| """ | ||
| repeat = _usable(runs, REPEAT_ARM) | ||
| if not repeat: | ||
| return None | ||
| baseline = _usable(runs, baseline_arm) | ||
| added = degraded = n = 0 | ||
| flips: dict[str, list[float]] = {} | ||
| for key, run in repeat.items(): | ||
| before = baseline.get(key) | ||
| if before is None: | ||
| continue | ||
| n += 1 | ||
| first, second = before.correct(scoring), run.correct(scoring) | ||
| added += second and not first | ||
| degraded += first and not second | ||
| flips.setdefault(run.atom_id, []).append(1.0 if second and not first else 0.0) | ||
| if not n: | ||
| return None | ||
| return { | ||
| "n_paired": n, | ||
| "draws": 1, | ||
| "added": round(added / n, 4), | ||
| "degraded": round(degraded / n, 4), | ||
| "ci95_added": clustered_ci(flips), | ||
| } | ||
|
|
||
|
|
||
| def control_fields(runs, *, baseline_arm: str, with_arm: str, scoring: str, | ||
| rates: dict) -> dict: | ||
| """The control keys :func:`~dokimi.novelty.measure.paired_outcomes` adds when the | ||
| placebo or the repeat arm ran; empty otherwise, so old outputs are unchanged.""" | ||
| fields: dict = {} | ||
| gated = placebo_gated(runs, baseline_arm=baseline_arm, with_arm=with_arm, scoring=scoring) | ||
| if gated is not None: | ||
| fields["added_over_placebo"] = gated | ||
| floor = noise_floor(runs, baseline_arm=baseline_arm, scoring=scoring) | ||
| if floor is not None: | ||
| fields["noise_floor"] = floor | ||
| fields["added_net_of_noise"] = round(rates["added"] - floor["added"], 4) | ||
| fields["degraded_within_noise"] = rates["degraded"] <= floor["degraded"] | ||
|
Comment on lines
+125
to
+126
There was a problem hiding this comment.
Contributor
There was a problem hiding this comment. Agreed, and the empty case is worse: when every |
||
| return fields | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The clustered interval needs 5 atoms where the old one needed 5 probes, and the fragment does not say so.
A run of 4 atoms with 3 paraphrases each used to get
ci95_added; now it getsnull. The new intervals are also wider, soci95_addedfrom older cells is not comparable with new ones.Suggestion: add both to this entry, so a reader knows what to re-run.