Merge pull request #82 from ethereum-optimism/feature/mininny/binary-…

…serialization

Binary serialization of VMState and monorepo update to 117c9a427168

.circleci/config.yml

@@ -4,7 +4,7 @@ parameters:
   ci_builder_image:
     type: string
     # depends with rvsol/lib/optimism submodule version
-    default: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:v0.51.0
+    default: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:v0.53.0
 
 workflows:
   main:
@@ -27,9 +27,17 @@ workflows:
           requires:
             - devnet-allocs-including-asterisc
       - prestate-reproducibility:
-          matrix:
-            parameters:
-              version: ["1.0.0", "1.1.0", "1.2.0"]
+          version: "1.0.0"
+          asterisc-commit: "25feabf"
+      - prestate-reproducibility:
+          version: "1.1.0"
+          asterisc-commit: "25feabf"
+      - prestate-reproducibility:
+          version: "1.2.0"
+          asterisc-commit: "25feabf"
+      - prestate-reproducibility:
+          version: "1.3.1"
+          asterisc-commit: "25feabf"
 
 commands:
   checkout-with-monorepo:
@@ -84,7 +92,7 @@ jobs:
           working_directory: rvsol/lib/optimism/packages/contracts-bedrock
       - run:
           name: Generate default allocs
-          command: make devnet-allocs
+          command: make devnet-allocs-tests
           working_directory: rvsol/lib/optimism
       - save_cache:
           name: Save Go modules cache
@@ -104,7 +112,7 @@ jobs:
             - "packages/contracts-bedrock/forge-artifacts"
             - "packages/contracts-bedrock/deploy-config/devnetL1.json"
             - "packages/contracts-bedrock/deployments/devnetL1"
-            - ".devnet"
+            - ".devnet-standard"
 
   go-mod-download-monorepo:
     docker:
@@ -130,7 +138,7 @@ jobs:
           name: run Go linter
           command: |
             # Identify how many cores it defaults to
-            golangci-lint --help | grep concurrency
+            golangci-lint run --help | grep concurrency
             make lint-go
           working_directory: rvsol/lib/optimism
       - save_cache:
@@ -246,19 +254,19 @@ jobs:
       - run:
           name: Load devnet-allocs
           command: |
-            mkdir -p .devnet
-            cp /tmp/workspace/.devnet/allocs-l1.json .devnet/allocs-l1.json
-            cp /tmp/workspace/.devnet/allocs-l2-delta.json .devnet/allocs-l2-delta.json
-            cp /tmp/workspace/.devnet/allocs-l2-ecotone.json .devnet/allocs-l2-ecotone.json
-            cp /tmp/workspace/.devnet/allocs-l2-fjord.json .devnet/allocs-l2-fjord.json
-            cp /tmp/workspace/.devnet/allocs-l2-granite.json .devnet/allocs-l2-granite.json
-            cp /tmp/workspace/.devnet/addresses.json .devnet/addresses.json
+            mkdir -p .devnet-standard
+            cp /tmp/workspace/.devnet-standard/allocs-l1.json .devnet-standard/allocs-l1.json
+            cp /tmp/workspace/.devnet-standard/allocs-l2-delta.json .devnet-standard/allocs-l2-delta.json
+            cp /tmp/workspace/.devnet-standard/allocs-l2-ecotone.json .devnet-standard/allocs-l2-ecotone.json
+            cp /tmp/workspace/.devnet-standard/allocs-l2-fjord.json .devnet-standard/allocs-l2-fjord.json
+            cp /tmp/workspace/.devnet-standard/allocs-l2-granite.json .devnet-standard/allocs-l2-granite.json
+            cp /tmp/workspace/.devnet-standard/addresses.json .devnet-standard/addresses.json
             cp /tmp/workspace/packages/contracts-bedrock/deploy-config/devnetL1.json devnetL1.json
             cp -r /tmp/workspace/packages/contracts-bedrock/deployments/devnetL1 devnetL1
           working_directory: rvsol
       - run:
           name: Copy L1 Allocs
-          command: cp .devnet/allocs-l1.json allocs-l1.json
+          command: cp .devnet-standard/allocs-l1.json allocs-l1.json
           working_directory: rvsol
       - run:
           name: Generate allocs including asterisc
@@ -299,10 +307,10 @@ jobs:
       - run:
           name: Copy L2 Allocs
           command: |
-            cp .devnet/allocs-l2-delta.json .devnet-asterisc/allocs-l2-delta.json
-            cp .devnet/allocs-l2-ecotone.json .devnet-asterisc/allocs-l2-ecotone.json
-            cp .devnet/allocs-l2-fjord.json .devnet-asterisc/allocs-l2-fjord.json
-            cp .devnet/allocs-l2-granite.json .devnet-asterisc/allocs-l2-granite.json
+            cp .devnet-standard/allocs-l2-delta.json .devnet-asterisc/allocs-l2-delta.json
+            cp .devnet-standard/allocs-l2-ecotone.json .devnet-asterisc/allocs-l2-ecotone.json
+            cp .devnet-standard/allocs-l2-fjord.json .devnet-asterisc/allocs-l2-fjord.json
+            cp .devnet-standard/allocs-l2-granite.json .devnet-asterisc/allocs-l2-granite.json
           working_directory: rvsol
       - persist_to_workspace:
           root: rvsol
@@ -330,13 +338,13 @@ jobs:
       - run:
           name: Load devnet-allocs
           command: |
-            mkdir -p .devnet
-            cp /tmp/workspace/.devnet-asterisc/allocs-l1.json .devnet/allocs-l1.json
-            cp /tmp/workspace/.devnet-asterisc/allocs-l2-delta.json .devnet/allocs-l2-delta.json
-            cp /tmp/workspace/.devnet-asterisc/allocs-l2-ecotone.json .devnet/allocs-l2-ecotone.json
-            cp /tmp/workspace/.devnet-asterisc/allocs-l2-fjord.json .devnet/allocs-l2-fjord.json
-            cp /tmp/workspace/.devnet-asterisc/allocs-l2-granite.json .devnet/allocs-l2-granite.json
-            cp /tmp/workspace/.devnet-asterisc/addresses.json .devnet/addresses.json
+            mkdir -p .devnet-standard
+            cp /tmp/workspace/.devnet-asterisc/allocs-l1.json .devnet-standard/allocs-l1.json
+            cp /tmp/workspace/.devnet-asterisc/allocs-l2-delta.json .devnet-standard/allocs-l2-delta.json
+            cp /tmp/workspace/.devnet-asterisc/allocs-l2-ecotone.json .devnet-standard/allocs-l2-ecotone.json
+            cp /tmp/workspace/.devnet-asterisc/allocs-l2-fjord.json .devnet-standard/allocs-l2-fjord.json
+            cp /tmp/workspace/.devnet-asterisc/allocs-l2-granite.json .devnet-standard/allocs-l2-granite.json
+            cp /tmp/workspace/.devnet-asterisc/addresses.json .devnet-standard/addresses.json
             mkdir -p packages/contracts-bedrock
             cp -r /tmp/workspace/packages/contracts-bedrock/deploy-config packages/contracts-bedrock
             mkdir -p packages/contracts-bedrock/deployments/devnetL1
@@ -369,8 +377,17 @@ jobs:
     parameters:
       version:
         type: string
+      asterisc-commit:
+        type: string
     steps:
       - checkout
+      - run:
+          name: Checkout asterisc to correct commit
+          command: |
+            # we need to use proper asterisc version that works with specific op-program version 
+            cp prestates.json temp.json
+            git checkout "<<parameters.asterisc-commit>>"
+            cp -f temp.json prestates.json
       - setup_remote_docker
       - run:
           name: Fetch submodules for asterisc

.github/workflows/ci.yaml

@@ -13,7 +13,7 @@ jobs:
       - name: Install Golang
         uses: actions/setup-go@v4
         with:
-          go-version: '1.21.x'
+          go-version: '1.22.x'
       - name: Build rvsol
         run: forge build
         working-directory: rvsol
@@ -38,7 +38,7 @@ jobs:
       - name: Install Golang
         uses: actions/setup-go@v5
         with:
-          go-version: '1.21.x'
+          go-version: '1.22.x'
       - name: Build FFI
         run: go build
         working-directory: rvgo/scripts/go-ffi
@@ -66,7 +66,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.21.x'
+          go-version: '1.22.x'
           cache: false
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v4
@@ -83,7 +83,7 @@ jobs:
       - name: Install Golang
         uses: actions/setup-go@v5
         with:
-          go-version: '1.21.x'
+          go-version: '1.22.x'
       - name: Check if optimism submodule version matches go.mod version
         run: ./.github/scripts/check_versions.sh
 
@@ -96,7 +96,7 @@ jobs:
         run: git submodule update --init
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.21.x'
+          go-version: '1.22.x'
       - name: Build asterisc
         run: make build-rvgo && cp rvgo/bin/asterisc tests/op-program-test/
       - name: Build op-program

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM golang:1.21.1-alpine3.18 AS builder
+FROM golang:1.22.7-alpine3.20 AS builder
 WORKDIR /build
 
 # Copy the context into the container
@@ -9,7 +9,7 @@ RUN apk add --no-cache git make bash
 
 RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build make
 
-FROM alpine:3.18 AS export
+FROM alpine:3.20 AS export
 
 RUN apk add --no-cache bash
 COPY --from=builder /build/rvgo/bin/asterisc /usr/local/bin/asterisc

Dockerfile.repro

@@ -1,4 +1,4 @@
-FROM golang:1.21.3-alpine3.18 as builder
+FROM golang:1.22.7-alpine3.20 as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers git jq bash
 

Makefile

@@ -68,8 +68,8 @@ fuzz-mac:
 OP_PROGRAM_PATH ?= $(MONOREPO_ROOT)/op-program/bin-riscv/op-program-client-riscv.elf
 
 prestate: build-rvgo op-program-riscv
-	./rvgo/bin/asterisc load-elf --path $(OP_PROGRAM_PATH) --out ./rvgo/bin/prestate.json --meta ./rvgo/bin/meta.json
-	./rvgo/bin/asterisc run --proof-at '=0' --stop-at '=1' --input ./rvgo/bin/prestate.json --meta ./rvgo/bin/meta.json --proof-fmt './rvgo/bin/%d.json' --output ""
+	./rvgo/bin/asterisc load-elf --path $(OP_PROGRAM_PATH) --out ./rvgo/bin/prestate.bin.gz --meta ./rvgo/bin/meta.json
+	./rvgo/bin/asterisc run --proof-at '=0' --stop-at '=1' --input ./rvgo/bin/prestate.bin.gz --meta ./rvgo/bin/meta.json --proof-fmt './rvgo/bin/%d.json' --output ""
 	mv ./rvgo/bin/0.json ./rvgo/bin/prestate-proof.json
 .PHONY: prestate
 
@@ -96,10 +96,16 @@ devnet-allocs: devnet-allocs-monorepo prestate
 	./rvsol/scripts/devnet_allocs.sh
 .PHONY: devnet-allocs
 
-devnet-clean:
+devnet-clean-monorepo:
+	make -C $(MONOREPO_ROOT) devnet-clean
+.PHONY: devnet-clean-monorepo
+
+devnet-clean: devnet-clean-monorepo
 	rm -rf .devnet
-	rm -rf packages/contracts-bedrock/deployments
-	rm -rf packages/contracts-bedrock/deploy-config
+	rm -rf .devnet-standard
+	rm -rf ./rvsol/devnetL1
+	rm -rf ./rvsol/deployments
+	rm -f ./rvsol/devnetL1.json
 .PHONY: devnet-clean
 
 reproducible-prestate:

go.mod

@@ -1,14 +1,14 @@
 module github.com/ethereum-optimism/asterisc
 
-go 1.21
+go 1.22
 
-toolchain go1.21.1
+toolchain go1.22.7
 
-replace github.com/ethereum/go-ethereum v1.14.8 => github.com/ethereum-optimism/op-geth v1.101408.0-rc.4.0.20240827042333-110c433a2469
+replace github.com/ethereum/go-ethereum v1.14.11 => github.com/ethereum-optimism/op-geth v1.101408.1-0.20241002211323-d5a96613c22b
 
 require (
-	github.com/ethereum-optimism/optimism v1.9.2-0.20240926003233-874c0e9878da
-	github.com/ethereum/go-ethereum v1.14.8
+	github.com/ethereum-optimism/optimism v1.9.2-0.20241008153126-117c9a427168
+	github.com/ethereum/go-ethereum v1.14.11
 	github.com/holiman/uint256 v1.3.1
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.7.0
@@ -25,7 +25,7 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.10.0 // indirect
+	github.com/bits-and-blooms/bitset v1.13.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/btcsuite/btcd v0.24.2 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
@@ -59,7 +59,7 @@ require (
 	github.com/ethereum-optimism/go-ethereum-hdwallet v0.1.3 // indirect
 	github.com/ethereum-optimism/superchain-registry/superchain v0.0.0-20240910145426-b3905c89e8ac // indirect
 	github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
-	github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 // indirect
+	github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/ferranbt/fastssz v0.1.2 // indirect
@@ -76,7 +76,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -139,7 +138,7 @@ require (
 	github.com/multiformats/go-base32 v0.1.0 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr v0.13.0 // indirect
-	github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
+	github.com/multiformats/go-multiaddr-dns v0.4.0 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.2.0 // indirect
 	github.com/multiformats/go-multicodec v0.9.0 // indirect
@@ -149,7 +148,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/onsi/ginkgo/v2 v2.20.0 // indirect
-	github.com/onsi/gomega v1.34.1 // indirect
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
@@ -188,7 +186,7 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/status-im/keycard-go v0.2.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/supranational/blst v0.3.11 // indirect
+	github.com/supranational/blst v0.3.13 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
@@ -202,20 +200,21 @@ require (
 	go.uber.org/mock v0.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
-	golang.org/x/crypto v0.27.0 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
 	golang.org/x/mod v0.20.0 // indirect
 	golang.org/x/net v0.28.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/term v0.24.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
-	golang.org/x/time v0.6.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
 	golang.org/x/tools v0.24.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect
+	lukechampine.com/uint128 v1.3.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )