From 1189887cdd1ca19899ee21ed23674f1dfb74f947 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 12 Jun 2024 15:05:44 +0200 Subject: [PATCH 1/8] Trigger own assignments in approval-voting --- polkadot/node/subsystem-bench/src/lib/approval/mod.rs | 9 ++++++++- .../node/subsystem-bench/src/lib/mock/runtime_api.rs | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 5c0c65b11cdb..4a68c95477ca 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -60,7 +60,7 @@ use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_overseer::Handle as OverseerHandleReal; use polkadot_primitives::{ BlockNumber, CandidateEvent, CandidateIndex, CandidateReceipt, Hash, Header, Slot, - ValidatorIndex, + ValidatorIndex, ASSIGNMENT_KEY_TYPE_ID, }; use prometheus::Registry; use sc_keystore::LocalKeystore; @@ -68,6 +68,7 @@ use sc_service::SpawnTaskHandle; use serde::{Deserialize, Serialize}; use sp_consensus_babe::Epoch as BabeEpoch; use sp_core::H256; +use sp_keystore::Keystore; use std::{ cmp::max, collections::{HashMap, HashSet}, @@ -785,6 +786,12 @@ fn build_overseer( let db: polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); let keystore = LocalKeystore::in_memory(); + keystore + .sr25519_generate_new( + ASSIGNMENT_KEY_TYPE_ID, + Some(state.test_authorities.key_seeds.first().unwrap().as_str()), + ) + .unwrap(); let system_clock = PastSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); diff --git a/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs index be9dbd55cb6f..5418db4a94a1 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs @@ -28,7 +28,7 @@ use polkadot_node_subsystem_types::OverseerSignal; use polkadot_primitives::{ node_features, AsyncBackingParams, CandidateEvent, CandidateReceipt, CoreState, GroupIndex, GroupRotationInfo, IndexedVec, NodeFeatures, OccupiedCore, ScheduledCore, SessionIndex, - SessionInfo, ValidatorIndex, + SessionInfo, ValidationCode, ValidatorIndex, }; use sp_consensus_babe::Epoch as BabeEpoch; use sp_core::H256; @@ -288,6 +288,13 @@ impl MockRuntimeApi { }; tx.send(Ok((groups, group_rotation_info))).unwrap(); }, + RuntimeApiMessage::Request( + _parent, + RuntimeApiRequest::ValidationCodeByHash(_, tx), + ) => { + let validation_code = ValidationCode(Vec::new()); + tx.send(Ok(Some(validation_code))).unwrap(); + }, // Long term TODO: implement more as needed. message => { unimplemented!("Unexpected runtime-api message: {:?}", message) From e29a819651712d1a213d6442db07f222a4b92028 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 12 Jun 2024 15:17:09 +0200 Subject: [PATCH 2/8] Estimate the network capacity --- .../subsystem-bench/examples/capacity.yaml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 polkadot/node/subsystem-bench/examples/capacity.yaml diff --git a/polkadot/node/subsystem-bench/examples/capacity.yaml b/polkadot/node/subsystem-bench/examples/capacity.yaml new file mode 100644 index 000000000000..24e8c7c14c89 --- /dev/null +++ b/polkadot/node/subsystem-bench/examples/capacity.yaml @@ -0,0 +1,66 @@ +# Estimate CPU requirements on a network +# 500 validators +# 100 cores +# 5 backing validators per core + + +TestConfiguration: +# Approval voting best case scenario +- objective: !ApprovalVoting + coalesce_mean: 3.0 + coalesce_std_dev: 1.0 + enable_assignments_v2: true + last_considered_tranche: 89 + stop_when_approved: true + coalesce_tranche_diff: 12 + num_no_shows_per_candidate: 10 + workdir_prefix: "/tmp/" + n_validators: 500 + n_cores: 50 + max_validators_per_core: 5 + min_pov_size: 5120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + connectivity: 100 + latency: null + num_blocks: 3 + +- objective: DataAvailabilityWrite + n_validators: 500 + n_cores: 100 + max_validators_per_core: 5 + min_pov_size: 5120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + connectivity: 100 + latency: null + num_blocks: 3 + +- objective: !DataAvailabilityRead + strategy: FullFromBackers + fetch_from_backers: true + n_validators: 500 + n_cores: 6 + max_validators_per_core: 5 + min_pov_size: 5120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + connectivity: 100 + latency: null + num_blocks: 3 + + +- objective: StatementDistribution + n_validators: 500 + n_cores: 100 + max_validators_per_core: 5 + min_pov_size: 5120 + max_pov_size: 5120 + peer_bandwidth: 524288000000 + bandwidth: 524288000000 + connectivity: 100 + latency: null + num_blocks: 3 From a0567e7978fde16541d2be322dc7e7dc9bbfbfd4 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 12 Jun 2024 15:25:33 +0200 Subject: [PATCH 3/8] Revert "Trigger own assignments in approval-voting" This reverts commit 1189887cdd1ca19899ee21ed23674f1dfb74f947. --- polkadot/node/subsystem-bench/src/lib/approval/mod.rs | 9 +-------- .../node/subsystem-bench/src/lib/mock/runtime_api.rs | 9 +-------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs index 4a68c95477ca..5c0c65b11cdb 100644 --- a/polkadot/node/subsystem-bench/src/lib/approval/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/approval/mod.rs @@ -60,7 +60,7 @@ use polkadot_node_subsystem_util::metrics::Metrics; use polkadot_overseer::Handle as OverseerHandleReal; use polkadot_primitives::{ BlockNumber, CandidateEvent, CandidateIndex, CandidateReceipt, Hash, Header, Slot, - ValidatorIndex, ASSIGNMENT_KEY_TYPE_ID, + ValidatorIndex, }; use prometheus::Registry; use sc_keystore::LocalKeystore; @@ -68,7 +68,6 @@ use sc_service::SpawnTaskHandle; use serde::{Deserialize, Serialize}; use sp_consensus_babe::Epoch as BabeEpoch; use sp_core::H256; -use sp_keystore::Keystore; use std::{ cmp::max, collections::{HashMap, HashSet}, @@ -786,12 +785,6 @@ fn build_overseer( let db: polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter = polkadot_node_subsystem_util::database::kvdb_impl::DbAdapter::new(db, &[]); let keystore = LocalKeystore::in_memory(); - keystore - .sr25519_generate_new( - ASSIGNMENT_KEY_TYPE_ID, - Some(state.test_authorities.key_seeds.first().unwrap().as_str()), - ) - .unwrap(); let system_clock = PastSystemClock::new(SystemClock {}, state.delta_tick_from_generated.clone()); diff --git a/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs index 5418db4a94a1..be9dbd55cb6f 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs @@ -28,7 +28,7 @@ use polkadot_node_subsystem_types::OverseerSignal; use polkadot_primitives::{ node_features, AsyncBackingParams, CandidateEvent, CandidateReceipt, CoreState, GroupIndex, GroupRotationInfo, IndexedVec, NodeFeatures, OccupiedCore, ScheduledCore, SessionIndex, - SessionInfo, ValidationCode, ValidatorIndex, + SessionInfo, ValidatorIndex, }; use sp_consensus_babe::Epoch as BabeEpoch; use sp_core::H256; @@ -288,13 +288,6 @@ impl MockRuntimeApi { }; tx.send(Ok((groups, group_rotation_info))).unwrap(); }, - RuntimeApiMessage::Request( - _parent, - RuntimeApiRequest::ValidationCodeByHash(_, tx), - ) => { - let validation_code = ValidationCode(Vec::new()); - tx.send(Ok(Some(validation_code))).unwrap(); - }, // Long term TODO: implement more as needed. message => { unimplemented!("Unexpected runtime-api message: {:?}", message) From 0a868ed287af03b684f41ba9e66605b7b81a27ed Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 12 Jun 2024 15:27:50 +0200 Subject: [PATCH 4/8] Add a job --- .gitlab/pipeline/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index d171a8a19426..09fcf021cd2f 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -637,3 +637,10 @@ subsystem-benchmark-statement-distribution: script: - cargo bench -p polkadot-statement-distribution --bench statement-distribution-regression-bench --features subsystem-benchmarks allow_failure: true + + subsystem-benchmark-estimate-capacity: + extends: + - .subsystem-benchmark-template + script: + - cargo run -p polkadot-subsystem-bench --release -- polkadot/node/subsystem-bench/examples/capacity.yaml + allow_failure: true From 457d95c89060a04d6c9b356d5a1d8282320daf9a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 12 Jun 2024 16:18:00 +0200 Subject: [PATCH 5/8] Fix yaml --- .gitlab/pipeline/test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 09fcf021cd2f..17f957de86a9 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -638,9 +638,9 @@ subsystem-benchmark-statement-distribution: - cargo bench -p polkadot-statement-distribution --bench statement-distribution-regression-bench --features subsystem-benchmarks allow_failure: true - subsystem-benchmark-estimate-capacity: - extends: - - .subsystem-benchmark-template - script: - - cargo run -p polkadot-subsystem-bench --release -- polkadot/node/subsystem-bench/examples/capacity.yaml - allow_failure: true +subsystem-benchmark-estimate-capacity: + extends: + - .subsystem-benchmark-template + script: + - cargo run -p polkadot-subsystem-bench --release -- polkadot/node/subsystem-bench/examples/capacity.yaml + allow_failure: true From a196c111104de0a3255941a08de32c8d8173c70d Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 13 Jun 2024 09:18:01 +0200 Subject: [PATCH 6/8] Add comments --- polkadot/node/subsystem-bench/examples/capacity.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/capacity.yaml b/polkadot/node/subsystem-bench/examples/capacity.yaml index 24e8c7c14c89..9efad2e64e96 100644 --- a/polkadot/node/subsystem-bench/examples/capacity.yaml +++ b/polkadot/node/subsystem-bench/examples/capacity.yaml @@ -3,7 +3,6 @@ # 100 cores # 5 backing validators per core - TestConfiguration: # Approval voting best case scenario - objective: !ApprovalVoting @@ -15,8 +14,8 @@ TestConfiguration: coalesce_tranche_diff: 12 num_no_shows_per_candidate: 10 workdir_prefix: "/tmp/" + n_cores: 50 # 50 cores occupied n_validators: 500 - n_cores: 50 max_validators_per_core: 5 min_pov_size: 5120 max_pov_size: 5120 @@ -27,8 +26,8 @@ TestConfiguration: num_blocks: 3 - objective: DataAvailabilityWrite - n_validators: 500 n_cores: 100 + n_validators: 500 max_validators_per_core: 5 min_pov_size: 5120 max_pov_size: 5120 @@ -41,8 +40,8 @@ TestConfiguration: - objective: !DataAvailabilityRead strategy: FullFromBackers fetch_from_backers: true + n_cores: 6 # we trigger 6 own assignments during approval-voting benchmark n_validators: 500 - n_cores: 6 max_validators_per_core: 5 min_pov_size: 5120 max_pov_size: 5120 @@ -54,8 +53,8 @@ TestConfiguration: - objective: StatementDistribution - n_validators: 500 n_cores: 100 + n_validators: 500 max_validators_per_core: 5 min_pov_size: 5120 max_pov_size: 5120 From 6f292b00b8f6221d17c608a2c620cc4c064f8a86 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 13 Jun 2024 10:02:50 +0200 Subject: [PATCH 7/8] Update config --- polkadot/node/subsystem-bench/examples/capacity.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/capacity.yaml b/polkadot/node/subsystem-bench/examples/capacity.yaml index 9efad2e64e96..6d475e8898be 100644 --- a/polkadot/node/subsystem-bench/examples/capacity.yaml +++ b/polkadot/node/subsystem-bench/examples/capacity.yaml @@ -14,7 +14,7 @@ TestConfiguration: coalesce_tranche_diff: 12 num_no_shows_per_candidate: 10 workdir_prefix: "/tmp/" - n_cores: 50 # 50 cores occupied + n_cores: 100 n_validators: 500 max_validators_per_core: 5 min_pov_size: 5120 @@ -40,7 +40,7 @@ TestConfiguration: - objective: !DataAvailabilityRead strategy: FullFromBackers fetch_from_backers: true - n_cores: 6 # we trigger 6 own assignments during approval-voting benchmark + n_cores: 7 # in average we trigger 7 own assignments during approval-voting benchmark n_validators: 500 max_validators_per_core: 5 min_pov_size: 5120 From f2fa53aab2f969ea7c235e72cadbee0d98ea4f15 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 13 Jun 2024 10:18:23 +0200 Subject: [PATCH 8/8] Update config --- polkadot/node/subsystem-bench/examples/capacity.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/examples/capacity.yaml b/polkadot/node/subsystem-bench/examples/capacity.yaml index 6d475e8898be..41ee8b7ae2db 100644 --- a/polkadot/node/subsystem-bench/examples/capacity.yaml +++ b/polkadot/node/subsystem-bench/examples/capacity.yaml @@ -12,7 +12,7 @@ TestConfiguration: last_considered_tranche: 89 stop_when_approved: true coalesce_tranche_diff: 12 - num_no_shows_per_candidate: 10 + num_no_shows_per_candidate: 3 workdir_prefix: "/tmp/" n_cores: 100 n_validators: 500 @@ -40,7 +40,7 @@ TestConfiguration: - objective: !DataAvailabilityRead strategy: FullFromBackers fetch_from_backers: true - n_cores: 7 # in average we trigger 7 own assignments during approval-voting benchmark + n_cores: 10 # in average we trigger 7 own assignments during approval-voting benchmark and extra 3 for 3 no-shows n_validators: 500 max_validators_per_core: 5 min_pov_size: 5120