From e900f03eb3460c11bc5de4f6ebf1813ccd4bc089 Mon Sep 17 00:00:00 2001 From: Watchmaker Date: Mon, 22 Jul 2024 12:56:17 -0700 Subject: [PATCH 01/14] Removing from the "main" branch the doc to run subtensor locally As per this Discord thread: https://discord.com/channels/799672011265015819/1260678915186495509 Docs issue: https://github.com/opentensor/developer-docs/issues/221 --- docs/running-subtensor-locally.md | 205 +----------------------------- 1 file changed, 1 insertion(+), 204 deletions(-) diff --git a/docs/running-subtensor-locally.md b/docs/running-subtensor-locally.md index 505fe2fb5..82bb87356 100644 --- a/docs/running-subtensor-locally.md +++ b/docs/running-subtensor-locally.md @@ -1,206 +1,3 @@ # Running subtensor node locally -- [Method 1: Using Docker](#method-1-using-docker) -- [Method 2: Using Source Code](#method-2-using-source-code) -- [Running on Cloud](#running-on-cloud) - -## Method 1: Using Docker - -To run a subtensor node with Docker, follow the below steps. - -If you are already running a subtensor node using Docker, then go directly to [Step 5 Prepare to Run](#step-5-prepare-to-run) to restart the Docker container. The below steps 1 through 4 are for first time users only. - -### Step 1: Install git - -Make sure you installed `git` on your machine. See [GitHub docs](https://docs.github.com/en/get-started). - -### Step 2: Install Docker - -Follow Docker's [official installation guides](https://docs.docker.com/engine/install/) and install Docker. - -**Run Docker first** -Before you proceed, make sure that Docker is running. - -### Step 3: Clone the subtensor repo - -Clone the subtensor repo: - -```bash -git clone https://github.com/opentensor/subtensor.git -``` - -### Step 4: Go into subtensor directory - -Then `cd` into the subtensor directory: - -```bash -cd subtensor -``` - -### Step 5: Prepare to run - -Execute the below three commands in this order: - -Make sure you are on the `main` branch. If not, switch to it: - -```bash -git checkout main -``` - -Pull the latest `main` branch contents: - -```bash -git pull -``` - -Stop the currently running Docker containers: - -```bash -docker compose down --volumes -``` - -### Run a lite node on mainchain - -To run a lite node connected to the Bittensor mainchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network mainnet --node-type lite -``` - -### Run an archive node on mainchain - -To run an archive node connected to the Bittensor mainchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network mainnet --node-type archive -``` - -### Run a lite node on testchain - -To run a lite node connected to the Bittensor testchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network testnet --node-type lite -``` - -### Run an archive node on testchain - -To run an archive node connected to the Bittensor testchain, run the below command. - -```bash -sudo ./scripts/run/subtensor.sh -e docker --network testnet --node-type archive -``` - ---- - -## Method 2: Using Source Code - -To install and run a subtensor node by compiling the source code, follow the below steps. - -## Install basic packages - -Install the basic requirements by running the below commands on a Linux terminal. - -```bash title="On Linux" -sudo apt-get update -sudo apt install build-essential -sudo apt-get install clang -sudo apt-get install curl -sudo apt-get install git -sudo apt-get install make -sudo apt install --assume-yes git clang curl libssl-dev protobuf-compiler -sudo apt install --assume-yes git clang curl libssl-dev llvm libudev-dev make protobuf-compiler -``` - -## Install Rust - -Next, install Rust and update the environment by running the following commands: - -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source ~/.cargo/env -``` - -Next, install Rust toolchain: - -```bash -rustup default stable -rustup update -rustup target add wasm32-unknown-unknown -rustup toolchain install nightly -rustup target add --toolchain nightly wasm32-unknown-unknown -``` - -## Compile subtensor code - -Next, to compile the subtensor source code, follow the below steps: - -Clone the subtensor repo: - -```bash -git clone https://github.com/opentensor/subtensor.git -``` - -`cd` into the subtensor directory: - -```bash -cd subtensor -``` - -Make sure you are on the `main` branch. If not, switch to it: - -```bash -git checkout main -``` - -Remove previous chain state: - -```bash -rm -rf /tmp/blockchain -``` - -Install subtensor by compiling with `cargo`: - -```bash -cargo build --profile production --features=runtime-benchmarks -``` - -## Run the subtensor node - -You can now run the public subtensor node either as a lite node or as an archive node. See below: - -### Lite node on mainchain - -To run a lite node connected to the mainchain, execute the below command (note the `--sync=warp` flag which runs the subtensor node in lite mode): - -```bash title="With --sync=warp setting, for lite node" -./target/production/node-subtensor --chain raw_spec.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -### Archive node on mainchain - -To run an archive node connected to the mainchain, execute the below command (note the `--sync=full` which syncs the node to the full chain and `--pruning archive` flags, which disables the node's automatic pruning of older historical data): - -```bash title="With --sync=full and --pruning archive setting, for archive node" -./target/production/node-subtensor --chain raw_spec.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /ip4/13.58.175.193/tcp/30333/p2p/12D3KooWDe7g2JbNETiKypcKT1KsCEZJbTzEHCn8hpd4PHZ6pdz5 --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -### Lite node on testchain - -To run a lite node connected to the testchain, execute the below command: - -```bash title="With bootnodes set to testnet and --sync=warp setting, for lite node." -./target/production/node-subtensor --chain raw_testspec.json --base-path /tmp/blockchain --sync=warp --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -### Archive node on testchain - -To run an archive node connected to the testchain, execute the below command: - -```bash title="With bootnodes set to testnet and --sync=full and --pruning archive setting, for archive node" -./target/production/node-subtensor --chain raw_testspec.json --base-path /tmp/blockchain --sync=full --pruning archive --port 30333 --max-runtime-instances 32 --rpc-max-response-size 2048 --rpc-cors all --rpc-port 9944 --bootnodes /dns/bootnode.test.finney.opentensor.ai/tcp/30333/p2p/12D3KooWPM4mLcKJGtyVtkggqdG84zWrd7Rij6PGQDoijh1X86Vr --no-mdns --in-peers 8000 --out-peers 8000 --prometheus-external --rpc-external -``` - -## Running on cloud - -We have not tested these installation scripts on any cloud service. In addition, if you are using Runpod cloud service, then note that this service is already [containerized](https://docs.runpod.io/pods/overview). Hence, the only option available to you is to compile from the source, as described in the above [Method 2: Using Source Code](#method-2-using-source-code) section. Note that these scripts have not been tested on Runpod. +See the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). From 55e6d0681d7521eaf314d6b6c6643fd0f54a4ae8 Mon Sep 17 00:00:00 2001 From: Watchmaker Date: Mon, 22 Jul 2024 13:11:09 -0700 Subject: [PATCH 02/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 47639b640..30f36ffab 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Requirements: ## For Subnet Development -If you are developing and testing subnet incentive mechanism, you will need to run a local subtensor node. Follow the detailed step-by-step instructions provided in the document [Running subtensor locally](./docs/running-subtensor-locally.md) to run either a lite node or an archive node. Also see the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). +If you are developing and testing subnet incentive mechanism, you will need to run a local subtensor node. Follow the detailed step-by-step instructions provided in the [**Subtensor Nodes** section in Bittensor Developer Documentation](https://docs.bittensor.com/subtensor-nodes). ### Lite node vs Archive node From aec052a099de7f3977964a3d6117b8222cc351a2 Mon Sep 17 00:00:00 2001 From: Unconst <32490803+unconst@users.noreply.github.com> Date: Tue, 23 Jul 2024 12:38:54 -0500 Subject: [PATCH 03/14] Update CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 5fefbd608..ffc01511b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @sacha-l @lisa-parity +* @unconst From 3b1352dafddff722709524166223484f5ad5f6bf Mon Sep 17 00:00:00 2001 From: Keith Date: Thu, 25 Jul 2024 16:55:52 +0200 Subject: [PATCH 04/14] Only run spec version check when the skip label doesn't exist --- .github/workflows/check-finney.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/check-finney.yml b/.github/workflows/check-finney.yml index 4bb12caf2..2428487b7 100644 --- a/.github/workflows/check-finney.yml +++ b/.github/workflows/check-finney.yml @@ -11,6 +11,7 @@ jobs: check-spec-version: name: Check spec_version bump runs-on: SubtensorCI + if: !contains(github.event.issue.labels.*.name, 'no-spec-version-bump') steps: - name: Dependencies run: | From 58cc59b8f2f482243305f233677949510e11fef0 Mon Sep 17 00:00:00 2001 From: Keith Date: Thu, 25 Jul 2024 17:01:01 +0200 Subject: [PATCH 05/14] Fix syntax error --- .github/workflows/check-finney.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-finney.yml b/.github/workflows/check-finney.yml index 2428487b7..ac14dd601 100644 --- a/.github/workflows/check-finney.yml +++ b/.github/workflows/check-finney.yml @@ -11,7 +11,7 @@ jobs: check-spec-version: name: Check spec_version bump runs-on: SubtensorCI - if: !contains(github.event.issue.labels.*.name, 'no-spec-version-bump') + if: ${{ !contains(github.event.issue.labels.*.name, 'no-spec-version-bump') }} steps: - name: Dependencies run: | From 31d6fe3512048deb81edd8ac79a9f49f2653dee6 Mon Sep 17 00:00:00 2001 From: Keith Date: Tue, 30 Jul 2024 23:22:45 +0900 Subject: [PATCH 06/14] Fix syntax --- .github/workflows/check-finney.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-finney.yml b/.github/workflows/check-finney.yml index ac14dd601..3e9fb5994 100644 --- a/.github/workflows/check-finney.yml +++ b/.github/workflows/check-finney.yml @@ -11,7 +11,7 @@ jobs: check-spec-version: name: Check spec_version bump runs-on: SubtensorCI - if: ${{ !contains(github.event.issue.labels.*.name, 'no-spec-version-bump') }} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-spec-version-bump') }} steps: - name: Dependencies run: | From f212a59371ade5bdbd18d6870e98cdc1a063ca7a Mon Sep 17 00:00:00 2001 From: Keith Date: Wed, 31 Jul 2024 00:41:09 +0900 Subject: [PATCH 07/14] Fix clippy --- pallets/collective/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pallets/collective/src/lib.rs b/pallets/collective/src/lib.rs index 96040f99c..66c55036d 100644 --- a/pallets/collective/src/lib.rs +++ b/pallets/collective/src/lib.rs @@ -951,6 +951,7 @@ impl, I: 'static> Pallet { /// /// If not `approved`: /// - one event deposited. + /// /// Two removals, one mutation. /// Computation and i/o `O(P)` where: /// - `P` is number of active proposals From 4dcd6c490fc173b03338aed79f352e4c54b689a1 Mon Sep 17 00:00:00 2001 From: Liam Date: Mon, 5 Aug 2024 15:08:54 +0200 Subject: [PATCH 08/14] bump spec version --- runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index a4abd124f..f5ca3c5cd 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -139,7 +139,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // `spec_version`, and `authoring_version` are the same between Wasm and native. // This value is set to 100 to notify Polkadot-JS App (https://polkadot.js.org/apps) to use // the compatible custom types. - spec_version: 165, + spec_version: 166, impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1, From d0649dcdaea43a0274091937389ec3f04907f709 Mon Sep 17 00:00:00 2001 From: Liam Date: Tue, 6 Aug 2024 09:24:08 +0200 Subject: [PATCH 09/14] fix clippy collapsible match --- pallets/subtensor/src/lib.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pallets/subtensor/src/lib.rs b/pallets/subtensor/src/lib.rs index f7824e2a3..2edcb8d50 100644 --- a/pallets/subtensor/src/lib.rs +++ b/pallets/subtensor/src/lib.rs @@ -2403,18 +2403,17 @@ where _len: usize, ) -> TransactionValidity { // Check if the call is one of the balance transfer types we want to reject - if let Some(balances_call) = call.is_sub_type() { - match balances_call { - BalancesCall::transfer_allow_death { .. } - | BalancesCall::transfer_keep_alive { .. } - | BalancesCall::transfer_all { .. } => { - if Pallet::::coldkey_in_arbitration(who) { - return Err(TransactionValidityError::Invalid(InvalidTransaction::Call)); - } + match call.is_sub_type() { + Some(BalancesCall::transfer_allow_death { .. }) + | Some(BalancesCall::transfer_keep_alive { .. }) + | Some(BalancesCall::transfer_all { .. }) => { + if Pallet::::coldkey_in_arbitration(who) { + return Err(TransactionValidityError::Invalid(InvalidTransaction::Call)); } - _ => {} // Other Balances calls are allowed } + _ => {} // Other Balances calls are allowed } + match call.is_sub_type() { Some(Call::commit_weights { netuid, .. }) => { if Self::check_weights_min_stake(who) { From 47345424ba45026e9b892017525b6e20383045cb Mon Sep 17 00:00:00 2001 From: Liam Date: Tue, 6 Aug 2024 17:28:21 +0200 Subject: [PATCH 10/14] clippy --- pallets/subtensor/tests/difficulty.rs | 1 - pallets/subtensor/tests/epoch.rs | 1 - pallets/subtensor/tests/mock.rs | 4 ++++ pallets/subtensor/tests/neuron_info.rs | 1 - pallets/subtensor/tests/registration.rs | 1 - pallets/subtensor/tests/serving.rs | 2 -- pallets/subtensor/tests/staking.rs | 3 --- pallets/subtensor/tests/weights.rs | 3 --- runtime/src/lib.rs | 2 ++ 9 files changed, 6 insertions(+), 12 deletions(-) diff --git a/pallets/subtensor/tests/difficulty.rs b/pallets/subtensor/tests/difficulty.rs index 05238bc43..c3023b829 100644 --- a/pallets/subtensor/tests/difficulty.rs +++ b/pallets/subtensor/tests/difficulty.rs @@ -5,7 +5,6 @@ mod mock; use sp_core::U256; #[test] -#[cfg(not(tarpaulin))] fn test_registration_difficulty_adjustment() { new_test_ext(1).execute_with(|| { // Create Net 1 diff --git a/pallets/subtensor/tests/epoch.rs b/pallets/subtensor/tests/epoch.rs index 676b3cd35..e2b911525 100644 --- a/pallets/subtensor/tests/epoch.rs +++ b/pallets/subtensor/tests/epoch.rs @@ -2107,7 +2107,6 @@ fn test_zero_weights() { // Test that epoch assigns validator permits to highest stake uids, varies uid interleaving and stake values. #[test] -#[cfg(not(tarpaulin))] fn test_validator_permits() { let netuid: u16 = 1; let tempo: u16 = u16::MAX - 1; // high tempo to skip automatic epochs in on_initialize, use manual epochs instead diff --git a/pallets/subtensor/tests/mock.rs b/pallets/subtensor/tests/mock.rs index fc784f46f..06bca8aff 100644 --- a/pallets/subtensor/tests/mock.rs +++ b/pallets/subtensor/tests/mock.rs @@ -262,6 +262,7 @@ impl CollectiveInterface for TriumvirateVotes { } // We call pallet_collective TriumvirateCollective +#[allow(unused)] type TriumvirateCollective = pallet_collective::Instance1; impl pallet_collective::Config for Test { type RuntimeOrigin = RuntimeOrigin; @@ -279,6 +280,7 @@ impl pallet_collective::Config for Test { } // We call council members Triumvirate +#[allow(unused)] type TriumvirateMembership = pallet_membership::Instance1; impl pallet_membership::Config for Test { type RuntimeEvent = RuntimeEvent; @@ -295,6 +297,7 @@ impl pallet_membership::Config for Test { // This is a dummy collective instance for managing senate members // Probably not the best solution, but fastest implementation +#[allow(unused)] type SenateCollective = pallet_collective::Instance2; impl pallet_collective::Config for Test { type RuntimeOrigin = RuntimeOrigin; @@ -312,6 +315,7 @@ impl pallet_collective::Config for Test { } // We call our top K delegates membership Senate +#[allow(unused)] type SenateMembership = pallet_membership::Instance2; impl pallet_membership::Config for Test { type RuntimeEvent = RuntimeEvent; diff --git a/pallets/subtensor/tests/neuron_info.rs b/pallets/subtensor/tests/neuron_info.rs index 10df1c07d..3494fdc5f 100644 --- a/pallets/subtensor/tests/neuron_info.rs +++ b/pallets/subtensor/tests/neuron_info.rs @@ -15,7 +15,6 @@ fn test_get_neuron_none() { } #[test] -#[cfg(not(tarpaulin))] fn test_get_neuron_some() { new_test_ext(1).execute_with(|| { let netuid: u16 = 1; diff --git a/pallets/subtensor/tests/registration.rs b/pallets/subtensor/tests/registration.rs index bd95ae3b1..7d6e8ea65 100644 --- a/pallets/subtensor/tests/registration.rs +++ b/pallets/subtensor/tests/registration.rs @@ -539,7 +539,6 @@ fn test_burn_adjustment() { } #[test] -#[cfg(not(tarpaulin))] fn test_registration_too_many_registrations_per_block() { new_test_ext(1).execute_with(|| { let netuid: u16 = 1; diff --git a/pallets/subtensor/tests/serving.rs b/pallets/subtensor/tests/serving.rs index 41e9888cc..b87b7fd10 100644 --- a/pallets/subtensor/tests/serving.rs +++ b/pallets/subtensor/tests/serving.rs @@ -161,7 +161,6 @@ fn test_serving_set_metadata_update() { } #[test] -#[cfg(not(tarpaulin))] fn test_axon_serving_rate_limit_exceeded() { new_test_ext(1).execute_with(|| { let hotkey_account_id = U256::from(1); @@ -379,7 +378,6 @@ fn test_prometheus_serving_set_metadata_update() { } #[test] -#[cfg(not(tarpaulin))] fn test_prometheus_serving_rate_limit_exceeded() { new_test_ext(1).execute_with(|| { let hotkey_account_id = U256::from(1); diff --git a/pallets/subtensor/tests/staking.rs b/pallets/subtensor/tests/staking.rs index cb2cac4ef..12d299d8f 100644 --- a/pallets/subtensor/tests/staking.rs +++ b/pallets/subtensor/tests/staking.rs @@ -22,7 +22,6 @@ use sp_runtime::traits::SignedExtension; ************************************************************/ #[test] -#[cfg(not(tarpaulin))] fn test_add_stake_dispatch_info_ok() { new_test_ext(1).execute_with(|| { let hotkey = U256::from(0); @@ -528,7 +527,6 @@ fn test_remove_stake_rate_limit_exceeded() { } #[test] -#[cfg(not(tarpaulin))] fn test_remove_stake_dispatch_info_ok() { new_test_ext(1).execute_with(|| { let hotkey = U256::from(0); @@ -1201,7 +1199,6 @@ fn test_delegate_stake_division_by_zero_check() { } #[test] -#[cfg(not(tarpaulin))] fn test_full_with_delegating() { new_test_ext(1).execute_with(|| { let netuid = 1; diff --git a/pallets/subtensor/tests/weights.rs b/pallets/subtensor/tests/weights.rs index 2344bd425..020eb1f6b 100644 --- a/pallets/subtensor/tests/weights.rs +++ b/pallets/subtensor/tests/weights.rs @@ -21,7 +21,6 @@ use substrate_fixed::types::I32F32; // Test the call passes through the subtensor module. #[test] -#[cfg(not(tarpaulin))] fn test_set_weights_dispatch_info_ok() { new_test_ext(0).execute_with(|| { let dests = vec![1, 1]; @@ -41,7 +40,6 @@ fn test_set_weights_dispatch_info_ok() { }); } #[test] -#[cfg(not(tarpaulin))] fn test_set_rootweights_dispatch_info_ok() { new_test_ext(0).execute_with(|| { let dests = vec![1, 1]; @@ -404,7 +402,6 @@ fn test_weights_err_no_validator_permit() { // To execute this test: cargo test --package pallet-subtensor --test weights test_set_weights_min_stake_failed -- --nocapture` #[test] -#[cfg(not(tarpaulin))] fn test_set_weights_min_stake_failed() { new_test_ext(0).execute_with(|| { let dests = vec![0]; diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index a4abd124f..475f1e4a7 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -516,6 +516,7 @@ impl pallet_collective::Config for Runtime { } // We call council members Triumvirate +#[allow(unused)] type TriumvirateMembership = pallet_membership::Instance1; impl pallet_membership::Config for Runtime { type RuntimeEvent = RuntimeEvent; @@ -531,6 +532,7 @@ impl pallet_membership::Config for Runtime { } // We call our top K delegates membership Senate +#[allow(unused)] type SenateMembership = pallet_membership::Instance2; impl pallet_membership::Config for Runtime { type RuntimeEvent = RuntimeEvent; From c34d72b23579c611a3368ae2932604353794f9b6 Mon Sep 17 00:00:00 2001 From: VectorChat Date: Thu, 8 Aug 2024 16:41:13 -0500 Subject: [PATCH 11/14] neuron pruning changes, initial tests --- pallets/subtensor/src/registration.rs | 80 ++++++++--------- pallets/subtensor/src/utils.rs | 7 ++ pallets/subtensor/tests/registration.rs | 115 ++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 44 deletions(-) diff --git a/pallets/subtensor/src/registration.rs b/pallets/subtensor/src/registration.rs index 6b73f2fc3..1ece0e9ca 100644 --- a/pallets/subtensor/src/registration.rs +++ b/pallets/subtensor/src/registration.rs @@ -423,65 +423,57 @@ impl Pallet { } /// Determine which peer to prune from the network by finding the element with the lowest pruning score out of - /// immunity period. If all neurons are in immunity period, return node with lowest prunning score. - /// This function will always return an element to prune. + /// immunity period. If there is a tie for lowest pruning score, the neuron registered earliest is pruned. + /// If all neurons are in immunity period, the neuron with the lowest pruning score is pruned. If there is a tie for + /// the lowest pruning score, the immune neuron registered earliest is pruned. + /// Ties for earliest registration are broken by the neuron with the lowest uid. pub fn get_neuron_to_prune(netuid: u16) -> u16 { let mut min_score: u16 = u16::MAX; - let mut min_score_in_immunity_period = u16::MAX; - let mut uid_with_min_score = 0; - let mut uid_with_min_score_in_immunity_period: u16 = 0; + let mut min_score_in_immunity: u16 = u16::MAX; + let mut earliest_registration: u64 = u64::MAX; + let mut earliest_registration_in_immunity: u64 = u64::MAX; + let mut uid_to_prune: u16 = 0; + let mut uid_to_prune_in_immunity: u16 = 0; + let mut found_non_immune = false; let neurons_n = Self::get_subnetwork_n(netuid); if neurons_n == 0 { return 0; // If there are no neurons in this network. } - let current_block: u64 = Self::get_current_block_as_u64(); - let immunity_period: u64 = Self::get_immunity_period(netuid) as u64; - for neuron_uid_i in 0..neurons_n { - let pruning_score: u16 = Self::get_pruning_score_for_uid(netuid, neuron_uid_i); + for neuron_uid in 0..neurons_n { + let pruning_score: u16 = Self::get_pruning_score_for_uid(netuid, neuron_uid); let block_at_registration: u64 = - Self::get_neuron_block_at_registration(netuid, neuron_uid_i); - #[allow(clippy::comparison_chain)] - if min_score == pruning_score { - if current_block.saturating_sub(block_at_registration) < immunity_period { - //neuron is in immunity period - if min_score_in_immunity_period > pruning_score { - min_score_in_immunity_period = pruning_score; - uid_with_min_score_in_immunity_period = neuron_uid_i; - } - } else { - uid_with_min_score = neuron_uid_i; + Self::get_neuron_block_at_registration(netuid, neuron_uid); + let is_immune = Self::get_neuron_is_immune(netuid, neuron_uid); + + if is_immune { + if pruning_score < min_score_in_immunity + || (pruning_score == min_score_in_immunity + && block_at_registration < earliest_registration_in_immunity) + { + min_score_in_immunity = pruning_score; + earliest_registration_in_immunity = block_at_registration; + uid_to_prune_in_immunity = neuron_uid; } - } - // Find min pruning score. - else if min_score > pruning_score { - if current_block.saturating_sub(block_at_registration) < immunity_period { - //neuron is in immunity period - if min_score_in_immunity_period > pruning_score { - min_score_in_immunity_period = pruning_score; - uid_with_min_score_in_immunity_period = neuron_uid_i; - } - } else { + } else { + found_non_immune = true; + if pruning_score < min_score + || (pruning_score == min_score && block_at_registration < earliest_registration) + { min_score = pruning_score; - uid_with_min_score = neuron_uid_i; + earliest_registration = block_at_registration; + uid_to_prune = neuron_uid; } } } - if min_score == u16::MAX { - //all neuorns are in immunity period - Self::set_pruning_score_for_uid( - netuid, - uid_with_min_score_in_immunity_period, - u16::MAX, - ); - uid_with_min_score_in_immunity_period + + if found_non_immune { + Self::set_pruning_score_for_uid(netuid, uid_to_prune, u16::MAX); + uid_to_prune } else { - // We replace the pruning score here with u16 max to ensure that all peers always have a - // pruning score. In the event that every peer has been pruned this function will prune - // the last element in the network continually. - Self::set_pruning_score_for_uid(netuid, uid_with_min_score, u16::MAX); - uid_with_min_score + Self::set_pruning_score_for_uid(netuid, uid_to_prune_in_immunity, u16::MAX); + uid_to_prune_in_immunity } } diff --git a/pallets/subtensor/src/utils.rs b/pallets/subtensor/src/utils.rs index c61133e94..d12a8a01a 100644 --- a/pallets/subtensor/src/utils.rs +++ b/pallets/subtensor/src/utils.rs @@ -461,6 +461,13 @@ impl Pallet { Self::deposit_event(Event::ImmunityPeriodSet(netuid, immunity_period)); } + pub fn get_neuron_is_immune(netuid: u16, uid: u16) -> bool { + let registered_at = Self::get_neuron_block_at_registration(netuid, uid); + let current_block = Self::get_current_block_as_u64(); + let immunity_period = Self::get_immunity_period(netuid); + current_block.saturating_sub(registered_at) < u64::from(immunity_period) + } + pub fn get_min_allowed_weights(netuid: u16) -> u16 { MinAllowedWeights::::get(netuid) } diff --git a/pallets/subtensor/tests/registration.rs b/pallets/subtensor/tests/registration.rs index 7d6e8ea65..98963f61c 100644 --- a/pallets/subtensor/tests/registration.rs +++ b/pallets/subtensor/tests/registration.rs @@ -1,6 +1,9 @@ #![allow(clippy::unwrap_used)] +use std::u16; + use frame_support::traits::Currency; +use substrate_fixed::types::extra::True; use crate::mock::*; use frame_support::dispatch::{DispatchClass, DispatchInfo, GetDispatchInfo, Pays}; @@ -538,6 +541,118 @@ fn test_burn_adjustment() { }); } +#[test] +fn test_burn_registration_pruning_scenarios() { + new_test_ext(1).execute_with(|| { + let netuid: u16 = 1; + let tempo: u16 = 13; + let burn_cost = 1000; + let coldkey_account_id = U256::from(667); + let max_allowed_uids = 6; + let immunity_period = 5000; + + SubtensorModule::set_burn(netuid, burn_cost); + SubtensorModule::set_max_allowed_uids(netuid, max_allowed_uids); + SubtensorModule::set_target_registrations_per_interval(netuid, max_allowed_uids); + SubtensorModule::set_immunity_period(netuid, immunity_period); + + // SubtensorModule::set_immunity_period(netuid, immunity_period); + + add_network(netuid, tempo, 0); + + let mint_balance = burn_cost * u64::from(max_allowed_uids) + 1_000_000_000; + SubtensorModule::add_balance_to_coldkey_account(&coldkey_account_id, mint_balance); + + // Register first half of neurons + for i in 0..3 { + assert_ok!(SubtensorModule::burned_register( + <::RuntimeOrigin>::signed(coldkey_account_id), + netuid, + U256::from(i) + )); + step_block(1); + } + + // Note: pruning score is set to u16::MAX after getting neuron to prune + + // 1. Test all immune neurons + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), true); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), true); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), true); + + SubtensorModule::set_pruning_score_for_uid(netuid, 0, 100); + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 75); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 50); + + // The immune neuron with the lowest score should be pruned + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 2); + + // 2. Test tie-breaking for immune neurons + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 50); + + // Should get the oldest neuron + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // 3. Test no immune neurons + step_block(immunity_period); + + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), false); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), false); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), false); + + SubtensorModule::set_pruning_score_for_uid(netuid, 0, 100); + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 75); + + // The non-immune neuron with the lowest score should be pruned + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // 4. Test tie-breaking for non-immune neurons + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 50); + + // Should get the oldest non-immune neuron + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // 5. Test mixed immunity + // Register second batch of neurons (these will be non-immune) + for i in 3..6 { + assert_ok!(SubtensorModule::burned_register( + <::RuntimeOrigin>::signed(coldkey_account_id), + netuid, + U256::from(i) + )); + step_block(1); + } + + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 3), true); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 4), true); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 5), true); + + // Set pruning scores for all neurons + SubtensorModule::set_pruning_score_for_uid(netuid, 0, 75); // non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); // non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 2, 60); // non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 3, 40); // immune + SubtensorModule::set_pruning_score_for_uid(netuid, 4, 55); // immune + SubtensorModule::set_pruning_score_for_uid(netuid, 5, 45); // immune + + // The non-immune neuron with the lowest score should be pruned + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); + + // If we remove the lowest non-immune neuron, it should choose the next lowest non-immune + SubtensorModule::set_pruning_score_for_uid(netuid, 1, u16::MAX); + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 2); + + // If we make all non-immune neurons have high scores, it should choose the oldest non-immune neuron + SubtensorModule::set_pruning_score_for_uid(netuid, 0, u16::MAX); + SubtensorModule::set_pruning_score_for_uid(netuid, 1, u16::MAX); + SubtensorModule::set_pruning_score_for_uid(netuid, 2, u16::MAX); + assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 0); + }); +} + #[test] fn test_registration_too_many_registrations_per_block() { new_test_ext(1).execute_with(|| { From 069a33d064e5296b6a1f667879c384bb0d93a57e Mon Sep 17 00:00:00 2001 From: VectorChat Date: Thu, 8 Aug 2024 17:11:51 -0500 Subject: [PATCH 12/14] adding comments --- pallets/subtensor/src/registration.rs | 10 ++++++++++ pallets/subtensor/src/utils.rs | 2 +- pallets/subtensor/tests/registration.rs | 11 ++++++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pallets/subtensor/src/registration.rs b/pallets/subtensor/src/registration.rs index 1ece0e9ca..8c28c176e 100644 --- a/pallets/subtensor/src/registration.rs +++ b/pallets/subtensor/src/registration.rs @@ -434,6 +434,10 @@ impl Pallet { let mut earliest_registration_in_immunity: u64 = u64::MAX; let mut uid_to_prune: u16 = 0; let mut uid_to_prune_in_immunity: u16 = 0; + + // This boolean is used instead of checking if min_score == u16::MAX, to avoid the case + // where all non-immune neurons have pruning score u16::MAX + // This may be unlikely in practice. let mut found_non_immune = false; let neurons_n = Self::get_subnetwork_n(netuid); @@ -448,6 +452,9 @@ impl Pallet { let is_immune = Self::get_neuron_is_immune(netuid, neuron_uid); if is_immune { + // if the immune neuron has a lower pruning score than the minimum for immune neurons, + // or, if the pruning scores are equal and the immune neuron was registered earlier than the current minimum for immune neurons, + // then update the minimum pruning score and the uid to prune for immune neurons if pruning_score < min_score_in_immunity || (pruning_score == min_score_in_immunity && block_at_registration < earliest_registration_in_immunity) @@ -458,6 +465,9 @@ impl Pallet { } } else { found_non_immune = true; + // if the non-immune neuron has a lower pruning score than the minimum for non-immune neurons, + // or, if the pruning scores are equal and the non-immune neuron was registered earlier than the current minimum for non-immune neurons, + // then update the minimum pruning score and the uid to prune for non-immune neurons if pruning_score < min_score || (pruning_score == min_score && block_at_registration < earliest_registration) { diff --git a/pallets/subtensor/src/utils.rs b/pallets/subtensor/src/utils.rs index d12a8a01a..f88ef6865 100644 --- a/pallets/subtensor/src/utils.rs +++ b/pallets/subtensor/src/utils.rs @@ -460,7 +460,7 @@ impl Pallet { ImmunityPeriod::::insert(netuid, immunity_period); Self::deposit_event(Event::ImmunityPeriodSet(netuid, immunity_period)); } - + /// Check if a neuron is in immunity based on the current block pub fn get_neuron_is_immune(netuid: u16, uid: u16) -> bool { let registered_at = Self::get_neuron_block_at_registration(netuid, uid); let current_block = Self::get_current_block_as_u64(); diff --git a/pallets/subtensor/tests/registration.rs b/pallets/subtensor/tests/registration.rs index 98963f61c..f644680c3 100644 --- a/pallets/subtensor/tests/registration.rs +++ b/pallets/subtensor/tests/registration.rs @@ -551,13 +551,12 @@ fn test_burn_registration_pruning_scenarios() { let max_allowed_uids = 6; let immunity_period = 5000; + // Initial setup SubtensorModule::set_burn(netuid, burn_cost); SubtensorModule::set_max_allowed_uids(netuid, max_allowed_uids); SubtensorModule::set_target_registrations_per_interval(netuid, max_allowed_uids); SubtensorModule::set_immunity_period(netuid, immunity_period); - // SubtensorModule::set_immunity_period(netuid, immunity_period); - add_network(netuid, tempo, 0); let mint_balance = burn_cost * u64::from(max_allowed_uids) + 1_000_000_000; @@ -575,7 +574,7 @@ fn test_burn_registration_pruning_scenarios() { // Note: pruning score is set to u16::MAX after getting neuron to prune - // 1. Test all immune neurons + // 1. Test if all immune neurons assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), true); assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), true); assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), true); @@ -591,12 +590,13 @@ fn test_burn_registration_pruning_scenarios() { SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); SubtensorModule::set_pruning_score_for_uid(netuid, 2, 50); - // Should get the oldest neuron + // Should get the oldest neuron (i.e., neuron that was registered first) assert_eq!(SubtensorModule::get_neuron_to_prune(netuid), 1); - // 3. Test no immune neurons + // 3. Test if no immune neurons step_block(immunity_period); + // ensure all neurons are non-immune assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), false); assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), false); assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), false); @@ -626,6 +626,7 @@ fn test_burn_registration_pruning_scenarios() { step_block(1); } + // Ensure all new neurons are immune assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 3), true); assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 4), true); assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 5), true); From 15366cf21150064b4bb69d5d76d2f9cbecdfd53c Mon Sep 17 00:00:00 2001 From: VectorChat Date: Fri, 9 Aug 2024 15:41:33 -0500 Subject: [PATCH 13/14] clippy --- pallets/subtensor/tests/registration.rs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pallets/subtensor/tests/registration.rs b/pallets/subtensor/tests/registration.rs index f644680c3..6a2b9be0b 100644 --- a/pallets/subtensor/tests/registration.rs +++ b/pallets/subtensor/tests/registration.rs @@ -3,7 +3,6 @@ use std::u16; use frame_support::traits::Currency; -use substrate_fixed::types::extra::True; use crate::mock::*; use frame_support::dispatch::{DispatchClass, DispatchInfo, GetDispatchInfo, Pays}; @@ -551,6 +550,9 @@ fn test_burn_registration_pruning_scenarios() { let max_allowed_uids = 6; let immunity_period = 5000; + const IS_IMMUNE: bool = true; + const NOT_IMMUNE: bool = false; + // Initial setup SubtensorModule::set_burn(netuid, burn_cost); SubtensorModule::set_max_allowed_uids(netuid, max_allowed_uids); @@ -575,9 +577,9 @@ fn test_burn_registration_pruning_scenarios() { // Note: pruning score is set to u16::MAX after getting neuron to prune // 1. Test if all immune neurons - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), true); - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), true); - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), true); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), IS_IMMUNE); SubtensorModule::set_pruning_score_for_uid(netuid, 0, 100); SubtensorModule::set_pruning_score_for_uid(netuid, 1, 75); @@ -597,9 +599,9 @@ fn test_burn_registration_pruning_scenarios() { step_block(immunity_period); // ensure all neurons are non-immune - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), false); - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), false); - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), false); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 0), NOT_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 1), NOT_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 2), NOT_IMMUNE); SubtensorModule::set_pruning_score_for_uid(netuid, 0, 100); SubtensorModule::set_pruning_score_for_uid(netuid, 1, 50); @@ -627,9 +629,9 @@ fn test_burn_registration_pruning_scenarios() { } // Ensure all new neurons are immune - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 3), true); - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 4), true); - assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 5), true); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 3), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 4), IS_IMMUNE); + assert_eq!(SubtensorModule::get_neuron_is_immune(netuid, 5), IS_IMMUNE); // Set pruning scores for all neurons SubtensorModule::set_pruning_score_for_uid(netuid, 0, 75); // non-immune From 681097b529dda69acd91cdbcb3753114fef57be8 Mon Sep 17 00:00:00 2001 From: VectorChat Date: Fri, 9 Aug 2024 21:41:54 -0500 Subject: [PATCH 14/14] more clippy --- pallets/subtensor/tests/registration.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/pallets/subtensor/tests/registration.rs b/pallets/subtensor/tests/registration.rs index 6a2b9be0b..536aa2688 100644 --- a/pallets/subtensor/tests/registration.rs +++ b/pallets/subtensor/tests/registration.rs @@ -1,7 +1,5 @@ #![allow(clippy::unwrap_used)] -use std::u16; - use frame_support::traits::Currency; use crate::mock::*;