Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ Main continuous integration job. Builds CCF for all target platforms, runs unit,

The Virtual A, B, and C jobs target `gha-vmss-d16av7-ci`, `gha-vmss-d16av7-ci-b`, and `gha-vmss-d16av7-ci-c`, respectively, to distribute demand across the regional pools.

End to end tests record each network's CCF consensus protocol trace and upload it with their logs.
The `Replay e2e consensus traces` job then replays every complete capture from the Virtual jobs against the Lean model with `tests/cft_trace_validation.py`, and fails if one is rejected.

File: `ci.yml`
3rd party dependencies: elan

Expand Down
3 changes: 3 additions & 0 deletions .github/workflows/ci-al4.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
Expand Down Expand Up @@ -139,6 +140,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
Expand Down Expand Up @@ -201,6 +203,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
49 changes: 49 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
Expand Down Expand Up @@ -178,6 +179,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
Expand Down Expand Up @@ -246,6 +248,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
Expand Down Expand Up @@ -344,6 +347,52 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()

cft_replay:
name: "Replay e2e consensus traces"
needs: [vmss-virtual-a, vmss-virtual-b, vmss-virtual-c]
runs-on: ubuntu-26.04
timeout-minutes: 30
defaults:
run:
shell: bash

steps:
- name: Install prerequisites
run: |
set -euo pipefail
sudo apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y elan

- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Install Lean
run: |
set -euo pipefail
elan run --install "$(cat lean/lean-toolchain)" lean --version

- name: Build CCF consensus replay
working-directory: lean/consensus
run: |
set -euo pipefail
lake build --wfail consensus-replay

# RUNNER_TEMP can persist between jobs on reused self-hosted runners; drop files from earlier runs.
- name: Clear stale e2e logs
run: rm -rf "$RUNNER_TEMP/e2e-logs"

- name: Download e2e logs
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: logs-azurelinux-virtual-*
path: ${{ runner.temp }}/e2e-logs

- name: Validate e2e consensus traces
run: |
set -euo pipefail
python3 tests/cft_trace_validation.py \
lean/consensus/.lake/build/bin/consensus-replay "$RUNNER_TEMP/e2e-logs"
2 changes: 2 additions & 0 deletions .github/workflows/coverage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
Expand Down Expand Up @@ -302,6 +303,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
if: success() || failure()
9 changes: 9 additions & 0 deletions .github/workflows/lean-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,15 @@ jobs:
echo "::endgroup::"
done

- name: Check CCF consensus trace fixture order independence
if: inputs.package == 'consensus'
run: |
set -euo pipefail
shopt -s failglob
python3 lean/consensus/replay/check-order.py \
lean/consensus/.lake/build/bin/consensus-replay \
lean/consensus/replay/fixtures/*/*.ndjson

- name: Check recovery trace fixtures against their schema
if: inputs.package == 'disaster-recovery'
run: |
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/lean.yml
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,14 @@ jobs:
echo "::endgroup::"
done

- name: Check CCF consensus trace order independence
run: |
set -euo pipefail
shopt -s failglob
python3 lean/consensus/replay/check-order.py \
lean/consensus/.lake/build/bin/consensus-replay \
"$RUNNER_TEMP"/cft-scenario-traces/*.ndjson

models:
name: Lean models and proofs
needs:
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/long-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore

Expand Down Expand Up @@ -131,6 +132,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore

Expand Down Expand Up @@ -211,6 +213,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
build_fuzz/fuzz-artifacts/**
if-no-files-found: ignore
Expand Down Expand Up @@ -270,6 +273,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore

Expand Down Expand Up @@ -350,5 +354,6 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore
1 change: 1 addition & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ jobs:
build/workspace/*/*.ledger/*
build/workspace/*/*.ledger.*/*
build/workspace/*/stack_trace
build/workspace/*_trace/*/*
build/workspace/**/openapi_coverage.json
if-no-files-found: ignore

Expand Down
2 changes: 2 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1776,6 +1776,8 @@ if(BUILD_TESTS)
LABEL e2e
BUCKET bucket_a
BUILD_DEPENDS js_generic
# Released nodes do not support the trace capture sink
NO_CFT_TRACE
ADDITIONAL_ARGS
${LTS_TEST_ARGS}
--constitution
Expand Down
12 changes: 10 additions & 2 deletions cmake/common.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ function(add_e2e_test)
cmake_parse_arguments(
PARSE_ARGV 0
PARSED_ARGS
"DETECT_DEADLOCKS"
"DETECT_DEADLOCKS;NO_CFT_TRACE"
"NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET;TSAN_SUPPRESSIONS"
"CONSTITUTION;ADDITIONAL_ARGS;BUILD_DEPENDS;CONFIGURATIONS"
)
Expand All @@ -228,12 +228,20 @@ function(add_e2e_test)
set(PARSED_ARGS_PERF_LABEL ${PARSED_ARGS_NAME})
endif()

# Record each network's CCF consensus protocol trace for validation.
# Benchmarks expect their own collector to be the nodes' only sink.
set(CFT_TRACE_ARGS --capture-cft-trace)
if(PARSED_ARGS_NO_CFT_TRACE OR "${PARSED_ARGS_LABEL}" STREQUAL "perf")
set(CFT_TRACE_ARGS)
endif()

add_test(
NAME ${PARSED_ARGS_NAME}
COMMAND
${PYTHON_WRAPPER} ${PARSED_ARGS_PYTHON_SCRIPT} -b . --label
${PARSED_ARGS_NAME} ${CCF_NETWORK_TEST_ARGS} ${PARSED_ARGS_CONSTITUTION}
${PARSED_ARGS_ADDITIONAL_ARGS} --tick-ms ${NODE_TICK_MS}
${PARSED_ARGS_ADDITIONAL_ARGS} ${CFT_TRACE_ARGS} --tick-ms
${NODE_TICK_MS}
CONFIGURATIONS ${PARSED_ARGS_CONFIGURATIONS}
)
if(NOT PARSED_ARGS_BUILD_DEPENDS)
Expand Down
2 changes: 1 addition & 1 deletion cmake/gersemi_definitions.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ function(add_e2e_test)
cmake_parse_arguments(
PARSE_ARGV 0
PARSED_ARGS
"DETECT_DEADLOCKS"
"DETECT_DEADLOCKS;NO_CFT_TRACE"
"NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET;TSAN_SUPPRESSIONS"
"CONSTITUTION;ADDITIONAL_ARGS;BUILD_DEPENDS;CONFIGURATIONS"
)
Expand Down
1 change: 1 addition & 0 deletions lean/consensus/Consensus.lean
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ public import Consensus.Replay.Reduction

import Consensus.Tests.CanonicalTraces
import Consensus.Tests.ReplayDelivery
import Consensus.Tests.ReplayStreams
40 changes: 35 additions & 5 deletions lean/consensus/Consensus/Replay/Input.lean
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ module

public import Consensus.Replay.Interpreter

import Std.Data.HashMap

public section

namespace Consensus.Replay
Expand Down Expand Up @@ -72,6 +74,8 @@ def readLog (path : System.FilePath) : IO (Except String (Array Record)) := do
structure Event where
record : Record
command : Option Record
/-- `h_ts`, the emission order within the record's process; other tags share the counter. -/
time : Nat
function : String
node : String
message : Json
Expand All @@ -81,13 +85,33 @@ deriving Inhabited
def Event.origin (event : Event) (rule : String) : Origin :=
{ file := event.record.file, line := event.record.line, rule }

/-- Project events, carry command provenance, and collect node identities. -/
/--
Order each process's records by `h_ts`, carry command provenance within a process,
split events into per-node streams sorted by node, and collect node identities.
-/
def prepareEvents (records : Array Record)
: Except String (Array Event × List String) := do
: Except String (Array (Array Event) × List String) := do
let keyed ←
records.mapM
fun record =>
(do
return (← string record.value "process_id", ← nat record.value "h_ts", record))
|>.mapError (s!"{record.location}: " ++ ·)
let keyed :=
keyed.toList.mergeSort fun (p, t, _) (q, u, _) => p < q || (p == q && t <= u)
let mut events := #[]
let mut previous : Option (String × Nat × Record) := none
let mut command : Option Record := none
let mut processes : Std.HashMap String String := {}
let mut declared : List String := []
for record in records do
for (process, time, record) in keyed do
if let some (last, lastTime, earlier) := previous then
if last == process && lastTime == time then
throw
s!"{record.location}: h_ts {time} of process '{process}' repeats {earlier.location}"
if last != process then
command := none
previous := some (process, time, record)
try
let message ← field record.value "msg"
if (optional message "cmd").isSome then
Expand All @@ -101,14 +125,20 @@ def prepareEvents (records : Array Record)
field message "state"
let node ←
string (if function == "retired_committed_tx" then message else state) "node_id"
events := events.push { record, command, function, node, message, state }
if let some other := processes[node]? then
require (other == process)
s!"node '{node}' is traced by processes '{other}' and '{process}'"
processes := processes.insert node process
events := events.push { record, command, time, function, node, message, state }
declared := node :: declared
if function == "retired_committed_tx" then
declared := (← string message "retired_node_id") :: declared
if function == "add_configuration" then
declared := (← configuration message) ++ declared
catch error =>
throw s!"{record.location}: {error}"
return (events, declared.mergeSort.eraseDups)
let streams :=
processes.keys.mergeSort.toArray.map fun node => events.filter (·.node == node)
return (streams, declared.mergeSort.eraseDups)

end Consensus.Replay
Loading