[paused] feat: snos job

CHANGELOG.md

@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Added
 
+- `SnosJob` implementation
+
 ## Changed
 
 ## Removed

Cargo.toml

@@ -30,6 +30,7 @@ axum-macros = "0.4.1"
 color-eyre = "0.6.2"
 dotenvy = "0.15.7"
 futures = "0.3.30"
+indexmap = "2.1.0"
 mongodb = { version = "2.8.1" }
 omniqueue = { version = "0.2.0" }
 reqwest = { version = "0.11.24" }
@@ -57,17 +58,22 @@ itertools = "0.13.0"
 mockall = "0.12.1"
 testcontainers = "0.18.0"
 
+# Bmlockifier, using the same version than Madara
+blockifier = { git = "https://github.com/Moonsong-Labs/blockifier", branch = "msl/derive-clone", features = [
+  "testing",
+] }
+
 # Cairo VM
 cairo-vm = { git = "https://github.com/lambdaclass/cairo-vm", features = [
   "extensive_hints",
   "cairo-1-hints",
 ] }
 
-# TODO: we currently use the Moonsong fork & the os-output-serde branch so we
-# can deserialize our snos input json into a StarknetOsInput struct.
-# TODO: update back to the main repo once it's merged
-# Sharp (Starkware)
-snos = { git = "https://github.com/keep-starknet-strange/snos" }
+# Snos
+snos = { git = "https://github.com/keep-starknet-strange/snos", branch = "main" }
+
+# Starknet API
+starknet_api = { version = "=0.10", features = ["testing"] }
 
 # Madara prover API
 madara-prover-common = { git = "https://github.com/Moonsong-Labs/madara-prover-api", branch = "od/use-latest-cairo-vm" }

crates/orchestrator/Cargo.toml

@@ -19,6 +19,7 @@ aws-config = { version = "1.1.7", features = ["behavior-version-latest"] }
 aws-sdk-s3 = { version = "1.38.0", features = ["behavior-version-latest"] }
 axum = { workspace = true, features = ["macros"] }
 axum-macros = { workspace = true }
+blockifier = { workspace = true }
 bytes = "1.6.0"
 cairo-vm = { workspace = true }
 color-eyre = { workspace = true }
@@ -28,6 +29,7 @@ ethereum-da-client = { workspace = true, optional = true }
 ethereum-settlement-client = { workspace = true }
 futures = { workspace = true }
 hex = { workspace = true }
+indexmap = { workspace = true }
 lazy_static = { workspace = true }
 log = "0.4.21"
 majin-blob-core = { git = "https://github.com/AbdelStark/majin-blob", branch = "main" }
@@ -47,6 +49,7 @@ snos = { workspace = true }
 starknet = { workspace = true }
 starknet-core = "0.9.0"
 starknet-settlement-client = { workspace = true }
+starknet_api = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
 tracing = { workspace = true }

crates/orchestrator/src/data_storage/mod.rs

@@ -12,6 +12,7 @@ use mockall::automock;
 /// The proposed storage format is :
 /// ----s3
 ///     ----<block_number>
+///         ----<cairo_pie.json>
 ///         ----<snos_output.json>
 ///         ----<kzg.txt>
 #[automock]

crates/orchestrator/src/jobs/constants.rs

@@ -8,3 +8,5 @@ pub const JOB_METADATA_STATE_UPDATE_BLOCKS_TO_SETTLE_KEY: &str = "blocks_number_
 pub const JOB_METADATA_STATE_UPDATE_FETCH_FROM_TESTS: &str = "fetch_from_test_data";
 pub const JOB_METADATA_STATE_UPDATE_ATTEMPT_PREFIX: &str = "attempt_tx_hashes_";
 pub const JOB_METADATA_STATE_UPDATE_LAST_FAILED_BLOCK_NO: &str = "last_failed_block_no";
+
+pub const JOB_METADATA_SNOS_BLOCK: &str = "block_number_to_run";

crates/orchestrator/src/jobs/snos_job/dummy_state.rs

@@ -0,0 +1,84 @@
+//! A Dummy state that does nothing.
+//! It just implements the State and StateReader trait provided by Blockifier.
+//!
+//! This module needs to be deleted as soon as we can import the structure from
+//! Madara code.
+//! Currently, we have version conflicts between snos <=> deoxys <=> cairo-vm.
+//! This is an issue that needs to be tackled on its own.
+
+use std::collections::HashSet;
+
+use blockifier::execution::contract_class::ContractClass;
+use blockifier::state::cached_state::CommitmentStateDiff;
+use blockifier::state::errors::StateError;
+use blockifier::state::state_api::{State, StateReader, StateResult};
+use indexmap::IndexMap;
+use starknet_api::core::{ClassHash, CompiledClassHash, ContractAddress, Nonce};
+use starknet_api::hash::StarkFelt;
+use starknet_api::state::StorageKey;
+
+pub struct DummyState;
+
+impl StateReader for DummyState {
+    fn get_storage_at(&mut self, _contract_address: ContractAddress, _key: StorageKey) -> StateResult<StarkFelt> {
+        Ok(StarkFelt::ZERO)
+    }
+
+    fn get_nonce_at(&mut self, _contract_address: ContractAddress) -> StateResult<Nonce> {
+        Ok(Nonce::default())
+    }
+
+    fn get_class_hash_at(&mut self, _contract_address: ContractAddress) -> StateResult<ClassHash> {
+        Ok(ClassHash::default())
+    }
+
+    fn get_compiled_contract_class(&mut self, _class_hash: ClassHash) -> StateResult<ContractClass> {
+        Err(StateError::OutOfRangeContractAddress)
+    }
+
+    fn get_compiled_class_hash(&mut self, _class_hash: ClassHash) -> StateResult<CompiledClassHash> {
+        Ok(CompiledClassHash::default())
+    }
+}
+
+impl State for DummyState {
+    fn set_storage_at(
+        &mut self,
+        _contract_address: ContractAddress,
+        _key: StorageKey,
+        _value: StarkFelt,
+    ) -> StateResult<()> {
+        Ok(())
+    }
+
+    fn increment_nonce(&mut self, _contract_address: ContractAddress) -> StateResult<()> {
+        Ok(())
+    }
+
+    fn set_class_hash_at(&mut self, _contract_address: ContractAddress, _class_hash: ClassHash) -> StateResult<()> {
+        Ok(())
+    }
+
+    fn set_contract_class(&mut self, _class_hash: ClassHash, _contract_class: ContractClass) -> StateResult<()> {
+        Ok(())
+    }
+
+    fn set_compiled_class_hash(
+        &mut self,
+        _class_hash: ClassHash,
+        _compiled_class_hash: CompiledClassHash,
+    ) -> StateResult<()> {
+        Ok(())
+    }
+
+    fn to_state_diff(&mut self) -> CommitmentStateDiff {
+        CommitmentStateDiff {
+            address_to_class_hash: IndexMap::default(),
+            address_to_nonce: IndexMap::default(),
+            storage_updates: IndexMap::default(),
+            class_hash_to_compiled_class_hash: IndexMap::default(),
+        }
+    }
+
+    fn add_visited_pcs(&mut self, _class_hash: ClassHash, _pcs: &HashSet<usize>) {}
+}

crates/orchestrator/src/jobs/snos_job/mod.rs

@@ -1,13 +1,35 @@
+mod dummy_state;
+
 use std::collections::HashMap;
+use std::num::NonZeroU128;
 
 use async_trait::async_trait;
+use blockifier::block::{pre_process_block, BlockInfo, BlockNumberHashPair, GasPrices};
+use blockifier::context::{ChainInfo, FeeTokenAddresses};
+use blockifier::versioned_constants::VersionedConstants;
+use cairo_vm::types::layout_name::LayoutName;
+use cairo_vm::Felt252;
+use color_eyre::eyre::eyre;
 use color_eyre::Result;
+use num::FromPrimitive;
+use snos::execution::helper::ExecutionHelperWrapper;
+use snos::io::input::StarknetOsInput;
+use snos::run_os;
+use starknet::providers::jsonrpc::JsonRpcRequest;
+use starknet_api::block::{BlockHash, BlockNumber, BlockTimestamp};
+use starknet_api::hash::StarkFelt;
+use starknet_core::types::FieldElement;
 use uuid::Uuid;
 
+use utils::time::get_current_timestamp_in_secs;
+
 use crate::config::Config;
+use crate::jobs::snos_job::dummy_state::DummyState;
 use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus};
 use crate::jobs::Job;
 
+use super::constants::JOB_METADATA_SNOS_BLOCK;
+
 pub struct SnosJob;
 
 #[async_trait]
@@ -29,28 +51,133 @@ impl Job for SnosJob {
         })
     }
 
-    async fn process_job(&self, _config: &Config, _job: &mut JobItem) -> Result<String> {
+    async fn process_job(&self, config: &Config, job: &mut JobItem) -> Result<String> {
+        // 0. Get block number from metadata
+        let block_number = self.get_block_number_from_metadata(job)?;
+
         // 1. Fetch SNOS input data from Madara
-        // 2. Import SNOS in Rust and execute it with the input data
+        let snos_input: StarknetOsInput = self.get_snos_input_from_madara(config, &block_number)?;
+
+        // 2. Build the required inputs for snos::run_os
+        // TODO: import BlockifierStateAdapter from Madara RPC and use it here
+        let mut state = DummyState {};
+
+        let block_number_and_hash = BlockNumberHashPair {
+            number: block_number,
+            hash: BlockHash(StarkFelt::from(
+                FieldElement::from_bytes_be(&snos_input.block_hash.clone().to_bytes_be())
+                    .expect("Could not convert Felt to FieldElement"),
+            )),
+        };
+
+        let block_info = BlockInfo {
+            block_number,
+            // TODO: Assert that we really want current_timestamp?
+            block_timestamp: BlockTimestamp(get_current_timestamp_in_secs()),
+            sequencer_address: snos_input.general_config.sequencer_address,
+            // TODO: retrieve prices from Madara?
+            gas_prices: GasPrices {
+                eth_l1_gas_price: NonZeroU128::new(0).unwrap(),
+                eth_l1_data_gas_price: NonZeroU128::new(0).unwrap(),
+                strk_l1_gas_price: NonZeroU128::new(0).unwrap(),
+                strk_l1_data_gas_price: NonZeroU128::new(0).unwrap(),
+            },
+            use_kzg_da: snos_input.general_config.use_kzg_da,
+        };
+
+        let chain_info = ChainInfo {
+            chain_id: snos_input.general_config.starknet_os_config.chain_id.clone(),
+            fee_token_addresses: FeeTokenAddresses {
+                eth_fee_token_address: snos_input.general_config.starknet_os_config.fee_token_address,
+                // TODO: assert that the STRK fee token address is deprecated_fee_token_address
+                strk_fee_token_address: snos_input.general_config.starknet_os_config.deprecated_fee_token_address,
+            },
+        };
+
+        let block_context = match pre_process_block(
+            &mut state,
+            Some(block_number_and_hash),
+            block_info,
+            chain_info,
+            VersionedConstants::latest_constants().clone(),
+        ) {
+            Ok(block_context) => block_context,
+            Err(e) => return Err(eyre!("pre_process_block failed for block #{}: {}", block_number, e)),
+        };
+
+        // TODO: contract_storage_map should be retrieved from where?
+        let contract_storage_map = HashMap::default();
+        let execution_helper = ExecutionHelperWrapper::new(
+            contract_storage_map,
+            vec![], // TODO: vec of TransactionExecutionInfo, how to get it?
+            &block_context,
+            (Felt252::from_u64(block_number.0).unwrap(), snos_input.block_hash),
+        );
+
+        // 3. Import SNOS in Rust and execute it with the input data
+        let (_cairo_pie, _snos_output) = match run_os(
+            // TODO: what is this path?
+            String::from("PATH/TO/THE/OS"),
+            // TODO: which layout should we choose?
+            LayoutName::plain,
+            snos_input,
+            block_context,
+            execution_helper,
+        ) {
+            Ok((cairo_pie, snos_output)) => (cairo_pie, snos_output),
+            Err(e) => return Err(eyre!("Could not run SNOS for block #{}: {}", block_number, e)),
+        };
+
         // 3. Store the received PIE in DB
+        // TODO: Store the PIE & the SnosOutput once S3 is implemented
         todo!()
     }
 
     async fn verify_job(&self, _config: &Config, _job: &mut JobItem) -> Result<JobVerificationStatus> {
         // No need for verification as of now. If we later on decide to outsource SNOS run
         // to another servicehow a, verify_job can be used to poll on the status of the job
-        todo!()
+        Ok(JobVerificationStatus::Verified)
     }
 
     fn max_process_attempts(&self) -> u64 {
-        todo!()
+        1
     }
 
     fn max_verification_attempts(&self) -> u64 {
-        todo!()
+        // TODO: isn't 10 a lot?
+        10
     }
 
     fn verification_polling_delay_seconds(&self) -> u64 {
-        todo!()
+        // TODO: what is an average run time for SNOS?
+        60
+    }
+}
+
+impl SnosJob {
+    /// Get the block number that needs to be run with SNOS for the current
+    /// job.
+    fn get_block_number_from_metadata(&self, job: &JobItem) -> Result<BlockNumber> {
+        let block_number: u64 = job
+            .metadata
+            .get(JOB_METADATA_SNOS_BLOCK)
+            .ok_or_else(|| eyre!("Block number to run with SNOS must be specified (snos job #{})", job.internal_id))?
+            .parse()?;
+        Ok(BlockNumber(block_number))
+    }
+
+    /// Retrieves the [StarknetOsInput] for the provided block number from Madara.
+    fn get_snos_input_from_madara(&self, _config: &Config, block_number: &BlockNumber) -> Result<StarknetOsInput> {
+        let raw_request = format!(
+            r#"{{
+            "id": 1,
+            "jsonrpc": "2.0",
+            "method": "madara_getSnosInput",
+            "params": [{{ "block_number": {} }}]
+        }}"#,
+            block_number
+        );
+        let _rpc_request = serde_json::from_str::<JsonRpcRequest>(&raw_request).expect("unable to parse request");
+        unimplemented!("Handler for madara_getSnosInput has not been implemented")
     }
 }

crates/orchestrator/src/tests/jobs/mod.rs

@@ -1,5 +1,8 @@
 use rstest::rstest;
 
+#[cfg(test)]
+pub mod snos_job;
+
 #[cfg(test)]
 pub mod da_job;