feat: Create backfill function for seal of reliability

api/src/shared/common/seal_criteria.py

@@ -14,10 +14,14 @@
 
 from datetime import datetime, timedelta
 from enum import Enum
-from typing import Dict, Final, Iterable, Optional, Tuple
+from typing import Dict, Final, Iterable, Optional, Tuple, TypeAlias
 
 from shared.common.error_handling import raise_internal_http_error, unknown_seal_criterion
 
+# A criterion named by its `SealCriterionName` value, as it arrives in a payload and as it
+# is stored in `seal_criterion.criterion`.
+CriterionNameStr: TypeAlias = str
+
 
 class SealCriterionName(str, Enum):
     """The six seal criteria. Values match the `seal_criterion_name` DB enum."""

api/src/shared/db_models/reliability_criterion_impl.py

@@ -14,6 +14,14 @@
 # The API `status` values are the `seal_criterion_status` DB enum verbatim, so a stored status is
 # served as-is with no translation - `CriterionStatus` is a `str` enum over exactly those values,
 # shared with the nightly job so the two cannot drift apart.
+#
+# `status` serves `confirmed_status`, the debounced verdict, which is the one `has_seal` is rolled
+# up from - so a client can always explain the seal it is shown. `observed_status`, the raw daily
+# check, never reaches the response on its own: the one thing it says that the debounced value does
+# not - a failure running right now - is carried by `in_grace_period` instead. This is the shape
+# specified in the #1789 spike; #1799 shipped `observed_status` here by mistake, which reported a
+# criterion inside its grace period as `fail` and could report a criterion that is the sole reason
+# a seal was withdrawn as `unknown`.
 
 
 class ReliabilityCriterionImpl(ReliabilityCriterion):
@@ -57,7 +65,7 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio
 
         criterion = resolve_criterion(criterion_row.criterion)
         now = datetime.now(timezone.utc)
-        status = criterion_row.observed_status
+        status = criterion_row.confirmed_status
 
         # `not_applicable` (withdrawn for this feed) and `never_evaluated` (no verdict ever) do not
         # participate in the seal, so they carry no grace period, no probation and no windows - just
@@ -79,12 +87,14 @@ def from_orm(cls, criterion_row: SealCriterionOrm | None) -> ReliabilityCriterio
         on_probation = probation_start is not None
 
         # A failing check still inside its grace period is not yet counting against the seal: the
-        # daily check reads `fail` but the debounced status is still `pass`. Grace does not apply
-        # during probation: a failure then restarts the probation clock outright, so there is
-        # nothing left for grace to protect.
+        # daily check reads `fail` but the debounced status is still `pass`. Both columns are read
+        # here rather than the served `status`, which is the debounced one and so is `pass` in
+        # exactly this case - that is what makes `in_grace_period` true only alongside `pass`, as
+        # #1789 specifies. Grace does not apply during probation: a failure then restarts the
+        # probation clock outright, so there is nothing left for grace to protect.
         in_grace_period = (
             grace_period is not None
-            and status == CriterionStatus.FAIL
+            and criterion_row.observed_status == CriterionStatus.FAIL
             and criterion_row.confirmed_status == CriterionStatus.PASS
             and not on_probation
         )

api/tests/unittest/models/test_feed_reliability_report_impl.py

@@ -108,7 +108,9 @@ def test_mixed_criteria(self):
         assert criteria["official"].status == CriterionStatus.PASS.value
         assert criteria["available"].status == CriterionStatus.FAIL.value
         assert criteria["available"].in_grace_period is False
-        assert criteria["compliant"].status == CriterionStatus.FAIL.value
+        # Failing its daily check but still inside grace, so it is not what cost the seal:
+        # `available` is. It reads `pass`, at risk.
+        assert criteria["compliant"].status == CriterionStatus.PASS.value
         assert criteria["compliant"].in_grace_period is True
 
     def test_evaluated_at_is_latest_across_criteria(self):

api/tests/unittest/models/test_reliability_criterion_impl.py

@@ -84,12 +84,16 @@ def test_never_evaluated_status_passes_through(self):
         assert result.in_grace_period is False
         assert result.on_probation is False
 
-    def test_unknown_status_passes_through(self):
-        """`unknown` (inputs missing this evaluation) is reported as its own status, not a failure."""
-        row = make_row(criterion=SealCriterionName.AVAILABLE, observed_status="unknown", confirmed_status="pass")
+    def test_an_unevaluable_check_serves_the_verdict_that_still_stands(self):
+        """`unknown` is a run that reached no verdict, so it never reaches the client.
+
+        The nightly job leaves `confirmed_status` alone on such a day, and that is what the seal
+        is decided on - so the API reports it, not the fact that one run could not look.
+        """
+        row = make_row(criterion=SealCriterionName.AVAILABLE, observed_status="unknown", confirmed_status="fail")
         result = ReliabilityCriterionImpl.from_orm(row)
 
-        assert result.status == CriterionStatus.UNKNOWN.value
+        assert result.status == CriterionStatus.FAIL.value
         assert result.in_grace_period is False
 
     def test_not_applicable_status_is_withdrawn(self):
@@ -111,9 +115,11 @@ def test_not_applicable_status_is_withdrawn(self):
         assert result.probation_ends_at is None
 
     def test_failing_inside_grace_period(self):
-        """A failure the grace period is still holding reports `fail` with `in_grace_period`.
+        """A failure the grace period is still holding reports `pass` with `in_grace_period`.
 
-        The seal is not withdrawn yet, and the countdown says when it would be.
+        Grace is not a failing state (#1789): the criterion still counts towards the seal, so it
+        still reads `pass`, and the flag is what marks it at risk. The countdown says when it
+        would stop passing.
         """
         first_failure = NOW - timedelta(days=10)
         row = make_row(
@@ -124,7 +130,7 @@ def test_failing_inside_grace_period(self):
         )
         result = ReliabilityCriterionImpl.from_orm(row)
 
-        assert result.status == CriterionStatus.FAIL.value
+        assert result.status == CriterionStatus.PASS.value
         assert result.in_grace_period is True
         assert result.grace_period_ends_at == first_failure + GRACE_PERIODS[SealCriterionName.COMPLIANT]
         assert result.first_failure_at == first_failure
@@ -134,7 +140,8 @@ def test_grace_exempt_criterion_is_never_in_grace(self):
         """A criterion with no grace period reports the failure straight away.
 
         `fresh_continuous` has no grace period, so even a row whose `confirmed_status` still reads
-        `pass` must not be served as being under grace - there would be no end date to report.
+        `pass` must not be served as being under grace - there would be no end date to report. The
+        served status is the stored `confirmed_status`, whatever the daily check said.
         """
         row = make_row(
             criterion=SealCriterionName.FRESH_CONTINUOUS,
@@ -145,7 +152,7 @@ def test_grace_exempt_criterion_is_never_in_grace(self):
         )
         result = ReliabilityCriterionImpl.from_orm(row)
 
-        assert result.status == CriterionStatus.FAIL.value
+        assert result.status == CriterionStatus.PASS.value
         assert result.in_grace_period is False
         assert result.grace_period_ends_at is None
 
@@ -189,7 +196,7 @@ def test_grace_does_not_apply_during_probation(self):
         )
         result = ReliabilityCriterionImpl.from_orm(row)
 
-        assert result.status == CriterionStatus.FAIL.value
+        assert result.status == CriterionStatus.PASS.value
         assert result.on_probation is True
         assert result.in_grace_period is False
         assert result.grace_period_ends_at is None
@@ -247,3 +254,21 @@ def test_elapsed_grace_period_reports_no_end_date(self):
 
         assert result.in_grace_period is True
         assert result.grace_period_ends_at is None
+
+    def test_the_served_status_is_the_one_the_seal_is_decided_on(self):
+        """The contract #1789 sets: `status` explains `has_seal`, so it is `confirmed_status`.
+
+        A criterion whose daily check fails while the debounced verdict still passes must not be
+        served as failing, or a client cannot reconcile the criteria it is shown with the seal
+        it is shown beside them.
+        """
+        row = make_row(
+            observed_status="fail",
+            confirmed_status="pass",
+            first_observed_failure_at=NOW - timedelta(days=1),
+            last_observed_failure_at=NOW,
+        )
+        assert ReliabilityCriterionImpl.from_orm(row).status == CriterionStatus.PASS.value
+
+        row = make_row(observed_status="pass", confirmed_status="fail", last_confirmed_failure_at=NOW)
+        assert ReliabilityCriterionImpl.from_orm(row).status == CriterionStatus.FAIL.value

api/tests/unittest/test_feeds.py

@@ -514,8 +514,9 @@ def test_gtfs_feed_reliability_with_criteria(client: TestClient):
     assert body["has_seal"] is False
     assert body["lost_at"] is not None
     assert by_name["official"]["status"] == "pass"
-    # Failing but inside its 30-day grace window: still at-risk rather than a confirmed loss.
-    assert by_name["compliant"]["status"] == "fail"
+    # Failing its daily check but inside its 30-day grace window, so it still counts towards the
+    # seal: `pass` plus the at-risk flag, rather than a confirmed loss.
+    assert by_name["compliant"]["status"] == "pass"
     assert by_name["compliant"]["in_grace_period"] is True
     assert by_name["compliant"]["grace_period_ends_at"] is not None
     # Passing its check yet still serving probation, which is why the feed has no seal.

docs/DatabaseCatalogAPI.yaml

@@ -1203,13 +1203,14 @@ components:
       description: >
         One criterion's contribution to the Seal of Reliability.
 
-        `status` is the criterion's own check at the last evaluation, undebounced, so a criterion
-        can read `fail` while the feed still holds the seal - that is the at-risk state, and
-        `in_grace_period` distinguishes it from a confirmed failure. Conversely a criterion can
-        read `pass` while `on_probation` is true, in which case it still does not count towards
-        the seal. The three states a client renders are therefore: healthy (`pass`), at risk
-        (`fail` with `in_grace_period`), and failing (`fail` without it) - with `on_probation`
-        as an independent flag on top.
+        `status` is the criterion's debounced verdict - the one the seal is decided on, so a
+        client can always explain the `has_seal` beside it. A criterion failing its daily check
+        but still inside its grace period reads `pass` with `in_grace_period` true: grace is not
+        a failing state, it is the warning before one. Conversely a criterion can read `pass`
+        while `on_probation` is true, in which case it still does not count towards the seal.
+        The three states a client renders are therefore: healthy (`pass`), at risk (`pass` with
+        `in_grace_period`), and failing (`fail`) - with `on_probation` as an independent flag
+        on top.
       type: object
       required:
         - criterion
@@ -1237,14 +1238,15 @@ components:
           example: compliant
         status:
           description: >
-            The criterion's verdict at the last evaluation, with no grace period applied.
-              * `pass` - the check passed.
-              * `fail` - the check failed. The seal is only withdrawn once the failure outlasts
-                the criterion's grace period, so check `in_grace_period` before presenting this
-                as a loss.
-              * `unknown` - the criterion was evaluated but its inputs were missing, so no verdict
-                could be reached this time. It is skipped when deciding the seal rather than counted
-                as a failure.
+            The criterion's debounced verdict: what it contributes to the seal, grace period
+            already applied.
+              * `pass` - the criterion is not counting against the seal. Either its check passed,
+                or the check failed and the failure is still inside the criterion's grace period,
+                which `in_grace_period` tells apart.
+              * `fail` - the failure is confirmed and the criterion is withholding the seal.
+              * `unknown` - not produced. A run whose inputs were missing reaches no verdict and
+                leaves this value untouched, so the last verdict stands. Listed only because the
+                underlying column can hold it.
               * `not_applicable` - the criterion does not apply to this feed (for example a
                 coverage criterion on a seasonal feed) and is withdrawn from the seal entirely.
               * `never_evaluated` - the criterion has produced no verdict for this feed yet. It is
@@ -1259,10 +1261,12 @@ components:
           example: fail
         in_grace_period:
           description: >
-            Whether a failing check is still inside the criterion's grace period, and so is not
-            yet counting against the seal. Can only be true while `status` is `fail`, and is
-            always false while `on_probation` is true, since a failure during probation restarts
-            probation outright rather than being absorbed.
+            Whether the criterion's daily check is currently failing but the failure is still
+            inside its grace period, and so is not yet counting against the seal. This is the
+            at-risk state, and the only thing in the response that reports the raw daily check.
+            Can only be true while `status` is `pass`, and is always false while `on_probation`
+            is true, since a failure during probation restarts probation outright rather than
+            being absorbed.
           type: boolean
           example: true
         grace_period_ends_at: