feat: artifacts (succinctlabs#1614)

scroll-tech · Oct 8, 2024 · 15e5db7 · 15e5db7
1 parent dcd99a3
commit 15e5db7
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 118 deletions.
diff --git a/.github/workflows/suite.yml b/.github/workflows/suite.yml
@@ -20,89 +20,89 @@ jobs:
   test-x86:
     name: ${{ matrix.workload.s3_folder }} (${{ matrix.workload.mode }})
     runs-on: ${{ matrix.workload.mode == 'cuda' && fromJSON('["runs-on", "family=g6.16xlarge", "ami=ami-0a63dc9cb9e934ba3", "spot=false", "run-id=${{ github.run_id }}"]') || fromJSON('["runs-on", "runner=64cpu-linux-x64", "spot=false", "run-id=${{ github.run_id }}"]') }}
-    # if: ${{ contains(github.event.head_commit.message, '(run-suite)') }}
+    if: ${{ contains(github.event.head_commit.message, '(run-suite)') }}
     strategy:
       matrix:
         workload:
           # Fibonacci 17k
-          - { s3_folder: fibonacci-17k, mode: cpu }
-          # - { s3_folder: fibonacci-17k, mode: cuda }
+          # - { s3_folder: fibonacci-17k, mode: cpu }
+          - { s3_folder: fibonacci-17k, mode: cuda }
           # - { s3_folder: fibonacci-17k, mode: network }
           # SSZ Withdrawals
-          - { s3_folder: ssz-withdrawals, mode: cpu }
-          # - { s3_folder: ssz-withdrawals, mode: cuda }
+          # - { s3_folder: ssz-withdrawals, mode: cpu }
+          - { s3_folder: ssz-withdrawals, mode: cuda }
           # - { s3_folder: ssz-withdrawals, mode: network }
           # Tendermint
-          - { s3_folder: tendermint, mode: cpu }
-          # - { s3_folder: tendermint, mode: cuda }
+          # - { s3_folder: tendermint, mode: cpu }
+          - { s3_folder: tendermint, mode: cuda }
           # - { s3_folder: tendermint, mode: network }
           # RSP 20526624
-          - { s3_folder: rsp-20526624, mode: cpu }
-          # - { s3_folder: rsp-20526624, mode: cuda }
+          # - { s3_folder: rsp-20526624, mode: cpu }
+          - { s3_folder: rsp-20526624, mode: cuda }
           # - { s3_folder: rsp-20526624, mode: network }
           # RSA
-          - { s3_folder: rsa, mode: cpu }
-          # - { s3_folder: rsa, mode: cuda }
+          # - { s3_folder: rsa, mode: cpu }
+          - { s3_folder: rsa, mode: cuda }
           # - { s3_folder: rsa, mode: network }
           # Regex
-          - { s3_folder: regex, mode: cpu }
-          # - { s3_folder: regex, mode: cuda }
+          # - { s3_folder: regex, mode: cpu }
+          - { s3_folder: regex, mode: cuda }
           # - { s3_folder: regex, mode: network }
           # Chess
-          - { s3_folder: chess, mode: cpu }
-          # - { s3_folder: chess, mode: cuda }
+          # - { s3_folder: chess, mode: cpu }
+          - { s3_folder: chess, mode: cuda }
           # - { s3_folder: chess, mode: network }
           # Tendermint
-          - { s3_folder: tendermint, mode: cpu }
-          # - { s3_folder: tendermint, mode: cuda }
+          # - { s3_folder: tendermint, mode: cpu }
+          - { s3_folder: tendermint, mode: cuda }
           # - { s3_folder: tendermint, mode: network }
           # RSP 20526624
-          - { s3_folder: rsp-20526624, mode: cpu }
-          # - { s3_folder: rsp-20526624, mode: cuda }
+          # - { s3_folder: rsp-20526624, mode: cpu }
+          - { s3_folder: rsp-20526624, mode: cuda }
           # - { s3_folder: rsp-20526624, mode: network }
           # RSA 
-          - { s3_folder: rsa, mode: cpu }
-          # - { s3_folder: rsa, mode: cuda }
+          # - { s3_folder: rsa, mode: cpu }
+          - { s3_folder: rsa, mode: cuda }
           # - { s3_folder: rsa, mode: network }
           # Regex
-          - { s3_folder: regex, mode: cpu }
-          # - { s3_folder: regex, mode: cuda }
+          # - { s3_folder: regex, mode: cpu }
+          - { s3_folder: regex, mode: cuda }
           # - { s3_folder: regex, mode: network }
           # JSON
-          - { s3_folder: json, mode: cpu }
-          # - { s3_folder: json, mode: cuda }
+          # - { s3_folder: json, mode: cpu }
+          - { s3_folder: json, mode: cuda }
           # - { s3_folder: json, mode: network }
           # Chess
-          - { s3_folder: chess, mode: cpu }
-          # - { s3_folder: chess, mode: cuda }
+          # - { s3_folder: chess, mode: cpu }
+          - { s3_folder: chess, mode: cuda }
           # - { s3_folder: chess, mode: network }
           # Blobstream 01j6z63fgafrc8jeh0k12gbtvw
-          - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cpu }
-          # - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cuda }
+          # - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cpu }
+          - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cuda }
           # # - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: network }
           # Blobstream 01j6z95bdme9svevmfyc974bja
-          - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cpu }
-          # - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cuda }
+          # - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cpu }
+          - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cuda }
           # # - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: network }
           # Blobstream 01j6z9ak0ke9srsppgywgke6fj
-          - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cpu }
-          # - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cuda }
+          # - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cpu }
+          - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cuda }
           # - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: network }
           # Vector 01j6xsv35re96tkgyda115320t
-          - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cpu }
-          # - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cuda }
+          # - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cpu }
+          - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cuda }
           # - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: network }
           # Vector 01j6xzy366ff5tbkzcrs8pma02
-          - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cpu }
-          # - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cuda }
+          # - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cpu }
+          - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cuda }
           # - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: network }
           # Vector 01j6y06de0fdaafemr8b1t69z3
-          - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cpu }
-          # - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cuda }
+          # - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cpu }
+          - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cuda }
           # - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: network }
           # Raiko A7 10
-          - { s3_folder: raiko-a7-10, mode: cpu }
-          # - { s3_folder: raiko-a7-10, mode: cuda }
+          # - { s3_folder: raiko-a7-10, mode: cpu }
+          - { s3_folder: raiko-a7-10, mode: cuda }
           # - { s3_folder: raiko-a7-10, mode: network }
           # OP Succinct
           # - { s3_folder: op-succinct-op-sepolia-range-17685896-17685897, mode: network }

diff --git a/crates/core/machine/src/lib.rs b/crates/core/machine/src/lib.rs
@@ -30,7 +30,7 @@ pub mod utils;
 /// This string should be updated whenever any step in verifying an SP1 proof changes, including
 /// core, recursion, and plonk-bn254. This string is used to download SP1 artifacts and the gnark
 /// docker image.
-pub const SP1_CIRCUIT_VERSION: &str = "v3.0.0-rc2-john";
+pub const SP1_CIRCUIT_VERSION: &str = "v3.0.0-alpha-test";
 
 // Re-export the `SP1ReduceProof` struct from sp1_core_machine.
 //

diff --git a/crates/cuda/src/lib.rs b/crates/cuda/src/lib.rs
@@ -157,7 +157,7 @@ impl SP1CudaProver {
         )
         .expect("failed to create client");
 
-        let timeout = Duration::from_secs(60);
+        let timeout = Duration::from_secs(300);
         let start_time = Instant::now();
 
         block_on(async {

diff --git a/crates/prover/dummy_vk_map.bin b/crates/prover/dummy_vk_map.bin
diff --git a/crates/prover/scripts/build_compress_vks.rs b/crates/prover/scripts/build_compress_vks.rs
@@ -37,25 +37,6 @@ fn main() {
     let range_start = args.start;
     let range_end = args.end;
 
-    // let mut allowd_vk_file = std::fs::File::open(build_dir.join("allowed_vk_map.bin")).unwrap();
-    // let allowed_vk_map: BTreeMap<[BabyBear; 8], usize> =
-    //     bincode::deserialize_from(&mut allowd_vk_file).unwrap();
-
-    // let deferered_data_file = std::fs::File::open(build_dir.join("deferred/vk_data.bin")).unwrap();
-    // let data: VkData = bincode::deserialize_from(deferered_data_file).unwrap();
-
-    // let new_set =
-    //     allowed_vk_map.keys().copied().chain(data.vk_map.keys().copied()).collect::<BTreeSet<_>>();
-
-    // let new_vk = VkData::new(new_set, 21);
-
-    // // overwrite the allowd_vk_map with the new one
-    // let mut allowd_vk_file =
-    //     std::fs::File::create(build_dir.join("allowed_vk_map_new.bin")).unwrap();
-    // bincode::serialize_into(&mut allowd_vk_file, &new_vk.vk_map).unwrap();
-
-    // // bincode::serialize_into(&mut allowd_vk_file, &new_vk.vk_map).unwrap();
-
     build_vk_map_to_file::<DefaultProverComponents>(
         build_dir,
         reduce_batch_size,
@@ -64,5 +45,6 @@ fn main() {
         num_setup_workers,
         range_start,
         range_end,
-    );
+    )
+    .unwrap();
 }
diff --git a/crates/prover/src/lib.rs b/crates/prover/src/lib.rs
@@ -106,6 +106,7 @@ const COMPRESS_CACHE_SIZE: usize = 3;
 pub const REDUCE_BATCH_SIZE: usize = 2;
 
 const VK_ALLOWED_VK_MAP_BYTES: &[u8] = include_bytes!("../allowed_vk_map.bin");
+const DUMMY_VK_ALLOWED_VK_MAP_BYTES: &[u8] = include_bytes!("../dummy_vk_map.bin");
 
 pub type CompressAir<F> = RecursionAir<F, COMPRESS_DEGREE>;
 pub type ShrinkAir<F> = RecursionAir<F, SHRINK_DEGREE>;
@@ -205,8 +206,11 @@ impl<C: SP1ProverComponents> SP1Prover<C> {
 
         tracing::info!("vk verification: {}", vk_verification);
 
-        let allowed_vk_map: BTreeMap<[BabyBear; DIGEST_SIZE], usize> =
-            bincode::deserialize(VK_ALLOWED_VK_MAP_BYTES).unwrap();
+        let allowed_vk_map: BTreeMap<[BabyBear; DIGEST_SIZE], usize> = if vk_verification {
+            bincode::deserialize(VK_ALLOWED_VK_MAP_BYTES).unwrap()
+        } else {
+            bincode::deserialize(DUMMY_VK_ALLOWED_VK_MAP_BYTES).unwrap()
+        };
 
         let (root, merkle_tree) = MerkleTree::commit(allowed_vk_map.keys().copied().collect());
 

diff --git a/crates/prover/src/shapes.rs b/crates/prover/src/shapes.rs
@@ -6,21 +6,20 @@ use std::{
     sync::{Arc, Mutex},
 };
 
+use thiserror::Error;
+
 use p3_baby_bear::BabyBear;
 use p3_field::AbstractField;
 use serde::{Deserialize, Serialize};
 use sp1_core_machine::riscv::CoreShapeConfig;
-use sp1_recursion_circuit::{
-    machine::{
-        SP1CompressWithVKeyWitnessValues, SP1CompressWithVkeyShape, SP1DeferredShape,
-        SP1DeferredWitnessValues, SP1RecursionShape, SP1RecursionWitnessValues,
-    },
-    merkle_tree::MerkleTree,
+use sp1_recursion_circuit::machine::{
+    SP1CompressWithVKeyWitnessValues, SP1CompressWithVkeyShape, SP1DeferredShape,
+    SP1DeferredWitnessValues, SP1RecursionShape, SP1RecursionWitnessValues,
 };
 use sp1_recursion_core::{shape::RecursionShapeConfig, RecursionProgram};
 use sp1_stark::{MachineProver, ProofShape, DIGEST_SIZE};
 
-use crate::{components::SP1ProverComponents, CompressAir, HashableKey, InnerSC, SP1Prover};
+use crate::{components::SP1ProverComponents, CompressAir, HashableKey, SP1Prover};
 
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub enum SP1ProofShape {
@@ -30,46 +29,6 @@ pub enum SP1ProofShape {
     Shrink(ProofShape),
 }
 
-#[derive(Clone, Serialize, Deserialize)]
-pub struct VkData {
-    pub vk_map: BTreeMap<[BabyBear; DIGEST_SIZE], usize>,
-    pub root: [BabyBear; DIGEST_SIZE],
-    pub merkle_tree: MerkleTree<BabyBear, InnerSC>,
-}
-
-impl VkData {
-    pub fn save(&self, build_dir: PathBuf, dummy: bool) -> Result<(), std::io::Error> {
-        let mut file = if dummy {
-            File::create(build_dir.join("dummy_vk_data.bin"))?
-        } else {
-            File::create(build_dir.join("vk_data.bin"))?
-        };
-        bincode::serialize_into(&mut file, self).unwrap();
-        Ok(())
-    }
-
-    pub fn load(build_dir: PathBuf) -> Result<Self, std::io::Error> {
-        let mut file = File::open(build_dir.join("vk_data.bin"))?;
-        let vk_data: Self = bincode::deserialize_from(&mut file).unwrap();
-        Ok(vk_data)
-    }
-
-    pub fn new(vk_set: BTreeSet<[BabyBear; DIGEST_SIZE]>, height: usize) -> Self {
-        let vk_map: BTreeMap<_, _> =
-            vk_set.into_iter().enumerate().map(|(i, vk_digest)| (vk_digest, i)).collect();
-
-        // Build a merkle tree from the vk map.
-        let mut vks_padded = vk_map.keys().cloned().collect::<Vec<_>>();
-        assert!(vks_padded.len() < 1 << height);
-        vks_padded.resize(1 << height, <[BabyBear; DIGEST_SIZE]>::default());
-        tracing::info!("building merkle tree");
-        let (root, merkle_tree) =
-            MerkleTree::<BabyBear, InnerSC>::commit(vk_map.keys().cloned().collect());
-
-        VkData { vk_map, root, merkle_tree }
-    }
-}
-
 #[derive(Debug, Clone)]
 pub enum SP1CompressProgramShape {
     Recursion(SP1RecursionShape),
@@ -78,6 +37,14 @@ pub enum SP1CompressProgramShape {
     Shrink(SP1CompressWithVkeyShape),
 }
 
+#[derive(Debug, Error)]
+pub enum VkBuildError {
+    #[error("IO error: {0}")]
+    IO(#[from] std::io::Error),
+    #[error("Serialization error: {0}")]
+    Bincode(#[from] bincode::Error),
+}
+
 pub fn build_vk_map<C: SP1ProverComponents>(
     reduce_batch_size: usize,
     dummy: bool,
@@ -225,23 +192,28 @@ pub fn build_vk_map_to_file<C: SP1ProverComponents>(
     num_setup_workers: usize,
     range_start: Option<usize>,
     range_end: Option<usize>,
-) {
-    std::fs::create_dir_all(&build_dir).expect("failed to create build directory");
+) -> Result<(), VkBuildError> {
+    std::fs::create_dir_all(&build_dir)?;
 
     tracing::info!("Building vk set");
 
-    let (vk_set, _, height) = build_vk_map::<C>(
+    let (vk_set, _, _) = build_vk_map::<C>(
         reduce_batch_size,
         dummy,
         num_compiler_workers,
         num_setup_workers,
         range_start.and_then(|start| range_end.map(|end| (start..end).collect())),
     );
 
-    tracing::info!("Creating vk data from vk set");
-    let vk_data = VkData::new(vk_set, height);
+    let vk_map = vk_set.into_iter().enumerate().map(|(i, vk)| (vk, i)).collect::<BTreeMap<_, _>>();
 
-    vk_data.save(build_dir, dummy).expect("failed to save vk data");
+    tracing::info!("Save the vk set to file");
+    let mut file = if dummy {
+        File::create(build_dir.join("dummy_vk_map.bin"))?
+    } else {
+        File::create(build_dir.join("vk_map.bin"))?
+    };
+    Ok(bincode::serialize_into(&mut file, &vk_map)?)
 }
 
 impl SP1ProofShape {
@@ -266,6 +238,7 @@ impl SP1ProofShape {
                     .get_all_shape_combinations(1)
                     .map(|mut x| Self::Shrink(x.pop().unwrap())),
             )
+            .take(10000)
     }
 
     pub fn dummy_vk_map<'a>(

diff --git a/crates/prover/wrap_vk.bin b/crates/prover/wrap_vk.bin
diff --git a/crates/prover/wraped_proof.bin b/crates/prover/wraped_proof.bin