Merge remote-tracking branch 'origin/tamir/v1.3.0-rc2' into scroll-de…

…v-0914
scroll-tech · Oct 8, 2024 · 7325250 · 7325250
2 parents 939c9d4 + 15e5db7
commit 7325250
Show file tree

Hide file tree

Showing 34 changed files with 593 additions and 183 deletions.
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -3,6 +3,14 @@ inputs:
   pull_token:
     description: "Token to use for private repo access"
     required: true
+  setup_gcc:
+    description: "Whether to setup GCC or not"
+    required: false
+    default: 'true'
+  setup_aws_cli:
+    description: "Whether to install AWS CLI or not"
+    required: false
+    default: 'true'
 runs:
   using: "composite"
   steps:
@@ -22,11 +30,41 @@ runs:
       shell: bash
       run: go version
 
+    - name: Check GCC version
+      id: check-gcc
+      shell: bash
+      run: |
+        if command -v gcc &> /dev/null; then
+          echo "gcc_exists=true" >> $GITHUB_OUTPUT
+          echo "gcc_version=$(gcc --version | head -n1 | awk '{print $NF}')" >> $GITHUB_OUTPUT
+        else
+          echo "gcc_exists=false" >> $GITHUB_OUTPUT
+        fi
+
     - name: Setup GCC
       uses: Dup4/actions-setup-gcc@v1
+      if: inputs.setup_gcc == 'true' && steps.check-gcc.outputs.gcc_exists != 'true'
       with:
         version: latest
 
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10' 
+
+    - name: Install AWS CLI v2
+      if: inputs.setup_aws_cli == 'true'
+      shell: bash
+      run: |
+        if ! command -v aws &> /dev/null; then
+          echo "AWS CLI not found. Installing..."
+          python3 -m pip install --user awscli
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+        else
+          echo "AWS CLI is already installed."
+        fi
+        export PATH="$HOME/.local/bin:$PATH"
+        aws --version
+
     - name: rust-cache
       uses: actions/cache@v3
       with:
@@ -45,3 +83,20 @@ runs:
       shell: bash
       run: |
         curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain 1.81.0 -y
+        . "$HOME/.cargo/env" 
+        echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+    # install pkg-config and openssl
+    - name: Install pkg-config and openssl
+      shell: bash
+      run: |
+        if ! dpkg -s pkg-config libssl-dev &> /dev/null; then
+          echo "pkg-config and/or libssl-dev not found. Installing..."
+          sudo apt-get update
+          sudo apt-get install -y pkg-config libssl-dev
+        else
+          echo "pkg-config and libssl-dev are already installed."
+        fi
+      
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v3
diff --git a/.github/runs-on.yml b/.github/runs-on.yml
@@ -0,0 +1,17 @@
+images:
+  nvidia-linux:
+    platform: "linux"
+    arch: "x64"
+    ami: "ami-0a63dc9cb9e934ba3"
+    owner: "421253708207"
+
+  dlami-x64:
+    platform: "linux"
+    arch: "x64"
+    owner: "898082745236" # AWS
+    name: "Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04)*"
+
+runners:
+  gpu-nvidia:
+    family: ["g6.4xlarge"]
+    image: dlami-x64
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -52,7 +52,7 @@ jobs:
         env:
           RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 -C target-cpu=native
           RUST_BACKTRACE: 1
-          FRI_QUERIES: 1
+          # FRI_QUERIES: 1
           SP1_DEV: 1
 
   test-arm:
@@ -89,7 +89,7 @@ jobs:
         env:
           RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 -C target-cpu=native
           RUST_BACKTRACE: 1
-          FRI_QUERIES: 1
+          # FRI_QUERIES: 1
           SP1_DEV: 1
 
   lint:
@@ -172,7 +172,7 @@ jobs:
 
       - name: Install SP1 toolchain
         run: |
-          cargo install --locked --path crates/cli
+          cargo install --locked --force --path crates/cli
           cargo prove install-toolchain
 
       - name: Run cargo fmt

diff --git a/.github/workflows/suite.yml b/.github/workflows/suite.yml
@@ -0,0 +1,139 @@
+name: Testing Suite
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - "**"
+    paths:
+      - "crates/**"
+      - "examples/**"
+      - "Cargo.toml"
+      - ".github/workflows/**"
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-x86:
+    name: ${{ matrix.workload.s3_folder }} (${{ matrix.workload.mode }})
+    runs-on: ${{ matrix.workload.mode == 'cuda' && fromJSON('["runs-on", "family=g6.16xlarge", "ami=ami-0a63dc9cb9e934ba3", "spot=false", "run-id=${{ github.run_id }}"]') || fromJSON('["runs-on", "runner=64cpu-linux-x64", "spot=false", "run-id=${{ github.run_id }}"]') }}
+    if: ${{ contains(github.event.head_commit.message, '(run-suite)') }}
+    strategy:
+      matrix:
+        workload:
+          # Fibonacci 17k
+          # - { s3_folder: fibonacci-17k, mode: cpu }
+          - { s3_folder: fibonacci-17k, mode: cuda }
+          # - { s3_folder: fibonacci-17k, mode: network }
+          # SSZ Withdrawals
+          # - { s3_folder: ssz-withdrawals, mode: cpu }
+          - { s3_folder: ssz-withdrawals, mode: cuda }
+          # - { s3_folder: ssz-withdrawals, mode: network }
+          # Tendermint
+          # - { s3_folder: tendermint, mode: cpu }
+          - { s3_folder: tendermint, mode: cuda }
+          # - { s3_folder: tendermint, mode: network }
+          # RSP 20526624
+          # - { s3_folder: rsp-20526624, mode: cpu }
+          - { s3_folder: rsp-20526624, mode: cuda }
+          # - { s3_folder: rsp-20526624, mode: network }
+          # RSA
+          # - { s3_folder: rsa, mode: cpu }
+          - { s3_folder: rsa, mode: cuda }
+          # - { s3_folder: rsa, mode: network }
+          # Regex
+          # - { s3_folder: regex, mode: cpu }
+          - { s3_folder: regex, mode: cuda }
+          # - { s3_folder: regex, mode: network }
+          # Chess
+          # - { s3_folder: chess, mode: cpu }
+          - { s3_folder: chess, mode: cuda }
+          # - { s3_folder: chess, mode: network }
+          # Tendermint
+          # - { s3_folder: tendermint, mode: cpu }
+          - { s3_folder: tendermint, mode: cuda }
+          # - { s3_folder: tendermint, mode: network }
+          # RSP 20526624
+          # - { s3_folder: rsp-20526624, mode: cpu }
+          - { s3_folder: rsp-20526624, mode: cuda }
+          # - { s3_folder: rsp-20526624, mode: network }
+          # RSA 
+          # - { s3_folder: rsa, mode: cpu }
+          - { s3_folder: rsa, mode: cuda }
+          # - { s3_folder: rsa, mode: network }
+          # Regex
+          # - { s3_folder: regex, mode: cpu }
+          - { s3_folder: regex, mode: cuda }
+          # - { s3_folder: regex, mode: network }
+          # JSON
+          # - { s3_folder: json, mode: cpu }
+          - { s3_folder: json, mode: cuda }
+          # - { s3_folder: json, mode: network }
+          # Chess
+          # - { s3_folder: chess, mode: cpu }
+          - { s3_folder: chess, mode: cuda }
+          # - { s3_folder: chess, mode: network }
+          # Blobstream 01j6z63fgafrc8jeh0k12gbtvw
+          # - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cpu }
+          - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: cuda }
+          # # - { s3_folder: blobstream-01j6z63fgafrc8jeh0k12gbtvw, mode: network }
+          # Blobstream 01j6z95bdme9svevmfyc974bja
+          # - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cpu }
+          - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: cuda }
+          # # - { s3_folder: blobstream-01j6z95bdme9svevmfyc974bja, mode: network }
+          # Blobstream 01j6z9ak0ke9srsppgywgke6fj
+          # - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cpu }
+          - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: cuda }
+          # - { s3_folder: blobstream-01j6z9ak0ke9srsppgywgke6fj, mode: network }
+          # Vector 01j6xsv35re96tkgyda115320t
+          # - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cpu }
+          - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: cuda }
+          # - { s3_folder: vector-01j6xsv35re96tkgyda115320t, mode: network }
+          # Vector 01j6xzy366ff5tbkzcrs8pma02
+          # - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cpu }
+          - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: cuda }
+          # - { s3_folder: vector-01j6xzy366ff5tbkzcrs8pma02, mode: network }
+          # Vector 01j6y06de0fdaafemr8b1t69z3
+          # - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cpu }
+          - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: cuda }
+          # - { s3_folder: vector-01j6y06de0fdaafemr8b1t69z3, mode: network }
+          # Raiko A7 10
+          # - { s3_folder: raiko-a7-10, mode: cpu }
+          - { s3_folder: raiko-a7-10, mode: cuda }
+          # - { s3_folder: raiko-a7-10, mode: network }
+          # OP Succinct
+          # - { s3_folder: op-succinct-op-sepolia-range-17685896-17685897, mode: network }
+          # - { s3_folder: op-succinct-op-sepolia-range-17985900-17985905, mode: network }
+          # - { s3_folder: op-succinct-op-sepolia-range-18129400-18129401, mode: network }
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+      - name: Setup CI
+        uses: ./.github/actions/setup
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_S3 }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3 }}
+          aws-region: us-west-2
+      - name: Copy files from S3
+        run: |
+          mkdir -p workdir
+          aws s3 cp s3://sp1-testing-suite/${{ matrix.workload.s3_folder }}/program.bin workdir/program.bin
+          aws s3 cp s3://sp1-testing-suite/${{ matrix.workload.s3_folder }}/stdin.bin workdir/stdin.bin
+      - name: Run sp1-perf
+        uses: actions-rs/cargo@v1
+        with:
+          command: run
+          toolchain: 1.81.0
+          args: --release -p sp1-perf -- --program workdir/program.bin --stdin workdir/stdin.bin --mode ${{ matrix.workload.mode }}
+        env:
+          RUST_LOG: debug
+          RUSTFLAGS: -Copt-level=3 -Ctarget-cpu=native
+          RUST_BACKTRACE: 1
+          SP1_PROVER: ${{ matrix.workload.mode == 'cpu' && 'local' || matrix.workload.mode == 'cuda' && 'cuda' || matrix.workload.mode == 'network' && 'network' }}
+          SP1_PRIVATE_KEY: ${{ secrets.SP1_PRIVATE_KEY }}
+          PROVER_NETWORK_RPC: https://43xadiwbrdkiquizwybgpzdhcu0pgvux.lambda-url.us-east-2.on.aws/
diff --git a/.gitignore b/.gitignore
@@ -14,6 +14,8 @@ pgo-data.profdata
 # Proofs
 **/proof-with-pis.bin
 **/proof-with-io.bin
+**/program.bin
+**/stdin.bin
 
 # Benchmark
 benchmark.csv

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -16,6 +16,7 @@ members = [
   "crates/derive",
   "crates/eval",
   "crates/helper",
+  "crates/perf",
   "crates/primitives",
   "crates/prover",
   "crates/recursion/circuit",

diff --git a/book/developers/usage-in-ci.md b/book/developers/usage-in-ci.md
@@ -2,7 +2,7 @@
 
 You may want to use SP1 in your [Github Actions](https://docs.github.com/en/actions) CI workflow.
 
-You first need have Rust installed, and you can use
+You first need to have Rust installed, and you can use
 [actions-rs/toolchain](https://github.com/actions-rs/toolchain) for this:
 
 ```yaml

diff --git a/book/generating-proofs/basics.md b/book/generating-proofs/basics.md
@@ -2,7 +2,7 @@
 
 All the methods you'll need for generating proofs are included in the `sp1_sdk` crate. Most importantly, you'll need to use the `ProverClient` to setup a proving key and verifying key for your program and then use the `execute`, `prove` and `verify` methods to execute your program, and generate and verify proofs.
 
-To make this more concrete, let's walk through a simple example of generating a proof for a Fiboancci program inside the zkVM.
+To make this more concrete, let's walk through a simple example of generating a proof for a Fibonacci program inside the zkVM.
 
 ## Example: Fibonacci
 

diff --git a/book/generating-proofs/prover-network/versions.md b/book/generating-proofs/prover-network/versions.md
@@ -13,7 +13,7 @@ If you submit a proof request to the prover network and you are not using a supp
 
 ## Changing versions
 
-You must switch to a supported version before submitting a proof. To do so, replace the `sp1-zkvm` version in your progam's `Cargo.toml`:
+You must switch to a supported version before submitting a proof. To do so, replace the `sp1-zkvm` version in your program's `Cargo.toml`:
 
 ```toml
 [dependencies]

diff --git a/book/introduction.md b/book/introduction.md
@@ -18,7 +18,7 @@ SP1 has undergone multiple audits from leading ZK security firms and is currentl
 
 Zero-knowledge proofs (ZKPs) are one of the most critical technologies to blockchain scaling, interoperability and privacy. But, historically building ZKP systems was extrememly complicated--requiring large teams with specialized cryptography expertise and taking years to go to production. 
 
-SP1 provides a performant, general-purpose zkVM that enables **any developer** to use ZKPs by writing normal code (in Rust), and get cheap and fast proofs. SP1 will enable ZKPs to become mainstream, introducing a a new era of verifiability for all of blockchain infrastructure and beyond.
+SP1 provides a performant, general-purpose zkVM that enables **any developer** to use ZKPs by writing normal code (in Rust), and get cheap and fast proofs. SP1 will enable ZKPs to become mainstream, introducing a new era of verifiability for all of blockchain infrastructure and beyond.
 
 
 ## SP1 enables a diversity of use-cases

diff --git a/book/why-use-sp1.md b/book/why-use-sp1.md
@@ -30,7 +30,7 @@ SP1 enables teams to use ZKPs in production with minimal overhead and fast timel
 
 **Maintainable:** With SP1, you can reuse existing Rust crates, like `revm`, `reth`, `tendermint-rs`, `serde` and more, to write your ZKP logic in maintainable, Rust code.
 
-**Go to market faster:** By reusing existng crates and expressing ZKP logic in regular code, SP1 significantly reduces audit surface area and complexity, enabling teams to go to market with ZKPs faster.
+**Go to market faster:** By reusing existing crates and expressing ZKP logic in regular code, SP1 significantly reduces audit surface area and complexity, enabling teams to go to market with ZKPs faster.
 
 ## Blazing Fast Performance
 

diff --git a/book/writing-programs/patched-crates.md b/book/writing-programs/patched-crates.md
@@ -41,7 +41,7 @@ ed25519-consensus = { git = "https://github.com/sp1-patches/ed25519-consensus",
 ecdsa-core = { git = "https://github.com/sp1-patches/signatures", package = "ecdsa", branch = "patch-ecdsa-v0.16.9" }
 secp256k1 = { git = "https://github.com/sp1-patches/rust-secp256k1", branch = "patch-secp256k1-v0.29.0" }
 substrate-bn = { git = "https://github.com/sp1-patches/bn", branch = "patch-v0.6.0" }
-bls12_381 = { git = "https://github.com/sp1-patches/bls12_381", branch = "patch-v0.8.8" }
+bls12_381 = { git = "https://github.com/sp1-patches/bls12_381", branch = "patch-v0.8.0" }
 
 ```
 

diff --git a/crates/core/machine/src/lib.rs b/crates/core/machine/src/lib.rs
@@ -30,7 +30,7 @@ pub mod utils;
 /// This string should be updated whenever any step in verifying an SP1 proof changes, including
 /// core, recursion, and plonk-bn254. This string is used to download SP1 artifacts and the gnark
 /// docker image.
-pub const SP1_CIRCUIT_VERSION: &str = "v3.0.0-rc2-scroll";
+pub const SP1_CIRCUIT_VERSION: &str = "v3.0.0-alpha-test-scroll";
 
 // Re-export the `SP1ReduceProof` struct from sp1_core_machine.
 //

diff --git a/crates/core/machine/src/riscv/mod.rs b/crates/core/machine/src/riscv/mod.rs
@@ -726,7 +726,7 @@ pub mod tests {
         let mut opts = SP1CoreOpts::default();
         opts.shard_size = 1024;
         opts.shard_batch_size = 2;
-        prove::<_, CpuProver<_, _>>(program, &stdin, BabyBearPoseidon2::new(), opts).unwrap();
+        prove::<_, CpuProver<_, _>>(program, &stdin, BabyBearPoseidon2::new(), opts, None).unwrap();
     }
 
     #[test]
@@ -739,6 +739,7 @@ pub mod tests {
             &stdin,
             BabyBearPoseidon2::new(),
             SP1CoreOpts::default(),
+            None,
         )
         .unwrap();
     }