diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml new file mode 100644 index 000000000..dc57c0626 --- /dev/null +++ b/.github/actions/bootstrap/action.yaml @@ -0,0 +1,20 @@ +--- +name: "Test local bootstrap" + +runs: + using: composite + steps: + - name: Setup + uses: actions/setup-go@v3 + with: + go-version: 1.20.x + + - name: Build + shell: bash + run: | + make geth + go run build/ci.go install -static ./cmd/bootnode + + - name: Test + shell: bash + run: ./.github/scripts/bootstrap_test.sh diff --git a/.github/actions/cache/golang/action.yml b/.github/actions/cache/golang/action.yml new file mode 100644 index 000000000..e97a7fee2 --- /dev/null +++ b/.github/actions/cache/golang/action.yml @@ -0,0 +1,58 @@ +--- +name: "Restore cache Go build and mod files" +description: "Restore cache Go build and mod files" + +inputs: + cache-key-suffix: + description: Suffix to append to the cache key + required: false + default: ${{ github.sha }} + refresh-go-cache: + description: Flag to control cache refresh + default: false +runs: + using: "composite" + steps: + - name: Restore Go mod (pkg) + if: ${{ inputs.refresh-go-cache != 'true' }} + uses: actions/cache/restore@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }} + restore-keys: | + ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }} + ${{ runner.os }}-gomod- + + - name: Restore Go build (test) + if: ${{ inputs.refresh-go-cache != 'true' }} + uses: actions/cache/restore@v3 + with: + path: | + ~/.cache/go-build + key: ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }} + restore-keys: | + ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }} + ${{ runner.os }}-gobuild- + + - name: Cache Go mod (pkg) + if: ${{ inputs.refresh-go-cache == 'true' }} + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }} + restore-keys: | + ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }} + ${{ runner.os }}-gomod- + + - name: Cache Go build (test) + if: ${{ inputs.refresh-go-cache == 'true' }} + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + key: ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }} + restore-keys: | + ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }} + ${{ runner.os }}-gobuild- diff --git a/.github/actions/cache/golangci-lint/action.yml b/.github/actions/cache/golangci-lint/action.yml new file mode 100644 index 000000000..4e741afee --- /dev/null +++ b/.github/actions/cache/golangci-lint/action.yml @@ -0,0 +1,52 @@ +--- +name: "Cache golangci-lint" +description: "Cache golangci-lint and analysis cache" + +inputs: + cache-key-suffix: + description: Suffix to append to the cache key + required: false + default: ${{ github.sha }} + golangci-lint-version: + description: Golangci-lint version to use + required: false + default: v1.51.1 + refresh-analysis-cache: + description: Flag to control if golangci-lint analysis cache needs to be refreshed + default: false +runs: + using: "composite" + steps: + - name: Cache golangci-lint + uses: actions/cache@v3 + id: cache-golangci-lint + with: + path: ~/go/bin/golangci-lint + key: ${{ runner.os }}-golangci-lint@${{ inputs.golangci-lint-version }} + + - name: Install golangci-lint + if: steps.cache-golangci-lint.outputs.cache-hit != 'true' + shell: bash + run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@${{ inputs.golangci-lint-version }} + + - name: Restore golangci-lint analysis cache + if: ${{ inputs.refresh-analysis-cache != 'true' }} + uses: actions/cache/restore@v3 + with: + path: ~/.cache/golangci-lint + # This technique will make the cache key unique to the commit SHA, + # so that we can still hit cache using the restore-key and renew the cache using the key. + key: ${{ runner.os }}-golangci-lint-analysis-cache-${{ inputs.cache-key-suffix }} + restore-keys: | + ${{ runner.os }}-golangci-lint-analysis-cache- + + - name: Cache golangci-lint analysis cache + if: ${{ inputs.refresh-analysis-cache == 'true' }} + uses: actions/cache@v3 + with: + path: ~/.cache/golangci-lint + # This technique will make the cache key unique to the commit SHA, + # so that we can still hit cache using the restore-key and renew the cache using the key. + key: ${{ runner.os }}-golangci-lint-analysis-cache-${{ inputs.cache-key-suffix }} + restore-keys: | + ${{ runner.os }}-golangci-lint-analysis-cache- diff --git a/.github/actions/rewind/action.yaml b/.github/actions/rewind/action.yaml new file mode 100644 index 000000000..45d5a198c --- /dev/null +++ b/.github/actions/rewind/action.yaml @@ -0,0 +1,19 @@ +--- +name: "Test chain rewind" + +runs: + using: composite + steps: + - name: Setup + uses: actions/setup-go@v3 + with: + go-version: 1.20.x + + - name: Build + shell: bash + run: | + make geth + + - name: Test + shell: bash + run: ./.github/scripts/rewind_test.sh diff --git a/.github/scripts/bootstrap_snap_sync.sh b/.github/scripts/bootstrap_snap_sync.sh new file mode 100755 index 000000000..8cb7c4ef7 --- /dev/null +++ b/.github/scripts/bootstrap_snap_sync.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# This script can be used to start a local geth node from genesis against a specified +# network. For example, `./.github/scripts/dev.toml` is a config for connecting to devnet +# if you have port-forwarded to the p2p partner pod via `kubectl port-forward pod/zkevm-geth-partner-0 30300:30300`. + +set -e +set -o pipefail + +# Clean up subprocesses on exit +_exit() { + pkill geth || true + # sleep 5s to let it kill + sleep 5 +} +trap _exit EXIT + +# Set the env var to enable long range sync +export GETH_FLAG_IMMUTABLE_LONG_RANGE_SYNC="1" + +./build/bin/geth immutable bootstrap local \ +--env devnet \ +--syncmode snap \ +--gcmode full \ +--config ./.github/scripts/dev.toml \ +--boots "0" \ +--rpcs "1" \ +--validators "0" diff --git a/.github/scripts/bootstrap_test.sh b/.github/scripts/bootstrap_test.sh new file mode 100755 index 000000000..a91cf96ec --- /dev/null +++ b/.github/scripts/bootstrap_test.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# These tests bootstrap a local network and run tests through the eth JSON RPC. +# The network will undergo a number of hard forks between various test fixtures. +# The Prevrandao fork is intended to be <= Shanghai fork. + +# TODO(serge): move these tests to a proper test suite implemented in Go +# when we know what the final test suite will look like. +# Features for test suite: +# * Set of fixtures that create transactions affected by all EVM-related hard forks +# * Fixtures can be run with expectation of success or failure depending on fork state + +set -e +set -o pipefail + +# necho prints a line without a newline +function necho() { + echo -n "> $*: " +} + +# Clean up subprocesses on exit +_exit() { + pkill geth || true + # sleep 5s to let it kill + sleep 5 +} +trap _exit EXIT + +# Assign in case PWD changes +dir="$PWD" +log="/tmp/bootstrapout" +boots=2 +validators=1 +rpcs=2 +# Set prevrandao override to be before shanghai +now=$(date +%s) +prevrandao_timestamp=$now +# Set shanghai override to be after genesis block and pre-shanghai tests +shanghai_timestamp=$((now+30)) +# Set cancun override a few blocks after shanghai +cancun_timestamp=$((shanghai_timestamp+4)) + +export GETH_FLAG_NET_RESTRICT="0.0.0.0/0" +export GETH_FLAG_P2P_SUBNET="127.0.0.1/32" + +function start_geth() { + necho "Starting geth" + + # Bootstrap and run local network + ./build/bin/geth immutable bootstrap local \ + --boots "$boots" \ + --rpcs "$rpcs" \ + --validators "$validators" \ + --blocklistfilepath "$dir/cmd/geth/testdata/acl_list.txt" \ + --override.prevrandao="$prevrandao_timestamp" \ + --override.shanghai="$shanghai_timestamp" \ + --override.cancun="$cancun_timestamp" > "$log" 2>&1 & + + # Wait for geth processes to start + while [ "$(pgrep geth | wc -l)" -lt "$((boots+rpcs+1))" ]; do + sleep 1 + echo -n "." + done + echo "" + necho "To view geth logs run" + echo "tail -f $log" + sleep 2 +} + +function wait_shanghai() { + # Wait for shanghai timestamp + now=$(date +%s) + if [ "$now" -lt "$shanghai_timestamp" ]; then + diff=$((shanghai_timestamp-now+5)) + necho "Waiting $diff seconds for shanghai timestamp" + sleep "$diff" + fi + echo "" +} + +# Run geth +start_geth + +# Need to use 0x44 opcode for Prevrandao fork. +echo "> Running post-prevrandao, pre-shanghai tests" +go test -count=1 -v ./tests/immutable \ +-privkey="$dir/cmd/geth/testdata/key.prv" \ +-rpc=http://localhost:8546 \ +-validatoradmin=http://localhost:8545 \ +-skipvoting=true \ +-run='.*Randao.*' + +# Wait for shanghai override +wait_shanghai + +# Run tests +# Need to use contracts compiled with solc relevant to Shanghai fork. +echo "> Running post-fork tests" +go test -count=1 -v ./tests/immutable \ +-privkey="$dir/cmd/geth/testdata/key.prv" \ +-blockedprivkey="$dir/cmd/geth/testdata/blockedkey.prv" \ +-rpc=http://localhost:8546 \ +-validatoradmin=http://localhost:8545 \ +-skipvoting=true + +echo "> Running vote test" # Tests after this will fail due to stalled block production +./build/bin/geth immutable vote add --voters http://localhost:8545 --validator 0x7442eD1e3c9FD421F47d12A2742AfF5DaFBf43f8 +echo "Tests finished successfully" diff --git a/.github/scripts/dev.toml b/.github/scripts/dev.toml new file mode 100644 index 000000000..b2802ec1e --- /dev/null +++ b/.github/scripts/dev.toml @@ -0,0 +1,83 @@ +[Eth] +NetworkId = 15003 +SyncMode = "full" +EthDiscoveryURLs = [] +SnapDiscoveryURLs = [] +NoPruning = true +NoPrefetch = false +TxLookupLimit = 2350000 +TransactionHistory = 2350000 +StateScheme = "hash" +DatabaseCache = 512 +DatabaseFreezer = "" +TrieCleanCache = 256 +TrieDirtyCache = 256 +TrieTimeout = 3600000000000 +SnapshotCache = 64 +Preimages = false +FilterLogCacheSize = 32 +EnablePreimageRecording = false +RPCGasCap = 50000000 +RPCEVMTimeout = 5000000000 +RPCTxFeeCap = 0e+00 + +[Eth.Miner] +GasFloor = 0 +GasCeil = 30000000 +GasPrice = 10000000000 +Recommit = 999999999000000000 +NewPayloadTimeout = 2000000000 + +[Eth.TxPool] +Locals = [] +NoLocals = true +Journal = "transactions.rlp" +Rejournal = 3600000000000 +PriceLimit = 10000000000 +PriceBump = 10 +AccountSlots = 16 +GlobalSlots = 5120 +AccountQueue = 64 +GlobalQueue = 1024 +Lifetime = 3600000000000 + +[Eth.BlobPool] +Datadir = "blobpool" +Datacap = 1 +PriceBump = 100 + +[Eth.GPO] +Blocks = 120 +Percentile = 60 +MaxHeaderHistory = 1024 +MaxBlockHistory = 1024 +MaxPrice = 500000000000 +IgnorePrice = 2 + +[Node] +DataDir = "" +IPCPath = "geth.ipc" +HTTPHost = "" +HTTPModules = [] +WSHost = "" +WSModules = [] +AllowUnprotectedTxs = true + +[Node.P2P] +MaxPeers = 100 +NoDiscovery = true +StaticNodes = [ + "enode://56724a169266dedfbb41dc50a1f237f4b55569fadccce347c2bb25526c1357b6ff9450ce91c0f21a13abb02fa348ad9e928e8a4fee969a1f14a943c8f6409fb1@127.0.0.1:30300", +] +TrustedNodes = [ + "enode://56724a169266dedfbb41dc50a1f237f4b55569fadccce347c2bb25526c1357b6ff9450ce91c0f21a13abb02fa348ad9e928e8a4fee969a1f14a943c8f6409fb1@127.0.0.1:30300", +] +ListenAddr = "" +DiscAddr = "" +EnableMsgEvents = false + +[Node.HTTPTimeouts] +ReadTimeout = 0 +ReadHeaderTimeout = 0 +WriteTimeout = 0 +IdleTimeout = 0 diff --git a/.github/scripts/rewind_test.sh b/.github/scripts/rewind_test.sh new file mode 100755 index 000000000..8ca468234 --- /dev/null +++ b/.github/scripts/rewind_test.sh @@ -0,0 +1,167 @@ +#!/bin/bash + +# These tests bootstrap a local network and run tests through the eth JSON RPC. + +set -e +set -o pipefail + +# necho prints a line without a newline +function necho() { + echo -n "> $*: " +} + +# Clean up subprocesses on exit +function stop_geth() { + pkill geth || true + # Wait for kill + sleep 1 + while [ "$(pgrep geth | wc -l)" -gt 1 ]; do + sleep 1 + echo -n "." + done +} + +function _exit() { + stop_geth +} +trap _exit EXIT + +# Assign in case PWD changes +dir="$PWD" +log="/tmp/rewindout" +boots=0 +validators=1 +rpcs=0 +now=$(date +%s) +prevrandao_timestamp=$now +shanghai_timestamp=$now +cancun_timestamp=$now +root_dir="$(mktemp -d)" + +function start_geth() { + necho "Starting geth" + dir="$root_dir/devnet/chain-15003/validator-0" + export GETH_FLAG_PASSWORD_FILEPATH="$dir/password" + + ./build/bin/geth \ + --datadir "$dir" --log.debug --networkid 15003 --metrics --metrics.addr 127.0.0.1 \ + --metrics.port 6060 --authrpc.port 8550 --verbosity 4 --port 30300 \ + --rpc.debugdisable --rpc.txpooldisable \ + --config "$root_dir/devnet/chain-15003/config.toml" --pprof --pprof.port 7070 \ + --miner.etherbase "$(cat "$dir/address")" \ + --mine --http --http.port 8545 --cache 128 --cache.database 35 \ + --cache.trie 35 --cache.gc 10 --cache.snapshot 20 --gcmode archive --syncmode full \ + --override.prevrandao="$prevrandao_timestamp" \ + --override.shanghai="$shanghai_timestamp" \ + --override.cancun="$cancun_timestamp" >> "$log" 2>&1 & + + # Wait for geth processes to start + while [ "$(pgrep geth | wc -l)" -lt "$((boots+rpcs+1))" ]; do + sleep 1 + echo -n "." + done + echo "" + necho "To view geth logs run" + echo "tail -f $log" +} + +function bootstrap_geth() { + necho "Bootstrapping geth" + + # Bootstrap and run local network + ./build/bin/geth immutable bootstrap local \ + --datadir "$root_dir" \ + --boots "$boots" \ + --rpcs "$rpcs" \ + --validators "$validators" \ + --blocklistfilepath "$dir/cmd/geth/testdata/acl_list.txt" \ + --override.prevrandao="$prevrandao_timestamp" \ + --override.shanghai="$shanghai_timestamp" \ + --override.cancun="$cancun_timestamp" >> "$log" 2>&1 & + + # Wait for geth processes to start + while [ "$(pgrep geth | wc -l)" -lt "$((boots+rpcs+1))" ]; do + sleep 1 + echo -n "." + done + echo "" + necho "To view geth logs run" + echo "tail -f $log" +} + +function get_head() { + resp=$(curl -Ss --location 'http://127.0.0.1:8545' \ + --header 'Content-Type: application/json' \ + --data '{ + "jsonrpc":"2.0", + "method":"eth_getBlockByNumber", + "params":[ + "latest", + true + ], + "id":1 + }') + block=$(echo "$resp" | jq -r '.result.number') + block=$((block)) + if [ "$block" == 0 ]; then + echo "Chain not progressing, exiting" + exit 1 + fi +} + +# Run geth +rm "$log" || true +bootstrap_geth + +# Check block after a few should have been sealed +sleep 20 +get_head +pre_rewind_block="$block" +echo "Block before rewind: $pre_rewind_block" + +# Kill the geth process and rewind +echo "Killing geth" +stop_geth + +echo "Rewinding chain" +rewind_block=0 +./build/bin/geth immutable rewind \ +--datadir "$root_dir/devnet/chain-15003/validator-0" \ +"$rewind_block" >> "$log" 2>&1 +cat "$root_dir/devnet/chain-15003/validator-0/rewind_history.yaml" + +# Restart geth and wait for smaller period of time than pre rewind +start_geth +sleep 8 + +# Check latest header and compare against pre-rewind block +get_head +echo "Block after rewind: $block" +if [ "$block" -ge "$pre_rewind_block" ]; then + echo "Latest ($block) is not less than pre-rewind block ($pre_rewind_block)" + exit 1 +fi + +# Kill the geth process and rewind to same block again, expecting no change +sleep 10 # Wait for chain to progress further beyond `block` +echo "Killing geth" +stop_geth + +echo "Rewinding chain again" +pre_rewind_block="$block" +rewind_block=0 +./build/bin/geth immutable rewind \ +--datadir "$root_dir/devnet/chain-15003/validator-0" \ +"$rewind_block" >> "$log" 2>&1 +cat "$root_dir/devnet/chain-15003/validator-0/rewind_history.yaml" + +start_geth +sleep 8 +get_head +echo "Block after second rewind: $block" +if [ "$block" -le "$pre_rewind_block" ]; then + echo "Latest ($block) should be greater than pre rewind block ($pre_rewind_block)" + exit 1 +fi + +echo "Tests finished successfully" diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml new file mode 100644 index 000000000..02994c378 --- /dev/null +++ b/.github/workflows/pr.yaml @@ -0,0 +1,76 @@ +name: PR + +on: + pull_request: + push: + branches: master + +concurrency: + group: PR-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + bootstrap: + name: Local E2E Tests + runs-on: ubuntu-20-04-4-cores + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Cache modules + uses: ./.github/actions/cache/golang + - uses: ./.github/actions/bootstrap + + rewind: + name: Rewind Chain Tests + runs-on: ubuntu-20-04-4-cores + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Cache modules + uses: ./.github/actions/cache/golang + - uses: ./.github/actions/rewind + + lint: + name: Lint + runs-on: ubuntu-20-04-4-cores + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # We need the full history to get the base commit in order to compute the diff in golanci-lint + - uses: actions/setup-go@v3 + with: + go-version: 1.20.x + - name: Cache modules + uses: ./.github/actions/cache/golang + - name: Cache linter + uses: ./.github/actions/cache/golangci-lint + - name: Debug golangci-lint cache + run: | + golangci-lint cache status + - name: Lint + shell: bash + run: | + golangci-lint run \ + --new-from-rev=${{ github.event.pull_request.base.sha || 'HEAD~'}} \ + --concurrency=4 \ + --out-format=github-actions \ + --config .golangci.yml \ + ./... + + tests: + name: Unit Tests + runs-on: ubuntu-20-04-4-cores + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v3 + with: + go-version: 1.20.x + - name: Cache modules + uses: ./.github/actions/cache/golang + - name: All Tests + shell: bash + run: | + go run build/ci.go install + go test $(go list ./... | grep -v 'go-ethereum/tests/immutable') -p 1 --parallel 4 -timeout=30m \ No newline at end of file