From 7ed522b67ef714ce2dea9490e1a88322e770f030 Mon Sep 17 00:00:00 2001
From: Zhenyang Shi <wcgcyx@gmail.com>
Date: Thu, 12 Dec 2024 11:05:48 +1000
Subject: [PATCH] Add PR workflow (#15)

* Create pr.yaml

* Add scripts

* Add actions

* Remove unnecessary files
---
 .github/actions/bootstrap/action.yaml         |  20 +++
 .github/actions/cache/golang/action.yml       |  58 ++++++
 .../actions/cache/golangci-lint/action.yml    |  52 ++++++
 .github/actions/rewind/action.yaml            |  19 ++
 .github/scripts/bootstrap_snap_sync.sh        |  28 +++
 .github/scripts/bootstrap_test.sh             | 108 +++++++++++
 .github/scripts/dev.toml                      |  83 +++++++++
 .github/scripts/rewind_test.sh                | 167 ++++++++++++++++++
 .github/workflows/pr.yaml                     |  76 ++++++++
 9 files changed, 611 insertions(+)
 create mode 100644 .github/actions/bootstrap/action.yaml
 create mode 100644 .github/actions/cache/golang/action.yml
 create mode 100644 .github/actions/cache/golangci-lint/action.yml
 create mode 100644 .github/actions/rewind/action.yaml
 create mode 100755 .github/scripts/bootstrap_snap_sync.sh
 create mode 100755 .github/scripts/bootstrap_test.sh
 create mode 100644 .github/scripts/dev.toml
 create mode 100755 .github/scripts/rewind_test.sh
 create mode 100644 .github/workflows/pr.yaml

diff --git a/.github/actions/bootstrap/action.yaml b/.github/actions/bootstrap/action.yaml
new file mode 100644
index 000000000..dc57c0626
--- /dev/null
+++ b/.github/actions/bootstrap/action.yaml
@@ -0,0 +1,20 @@
+---
+name: "Test local bootstrap"
+
+runs:
+  using: composite
+  steps:
+    - name: Setup
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.20.x
+
+    - name: Build
+      shell: bash
+      run: |
+        make geth
+        go run build/ci.go install -static ./cmd/bootnode
+
+    - name: Test
+      shell: bash
+      run: ./.github/scripts/bootstrap_test.sh
diff --git a/.github/actions/cache/golang/action.yml b/.github/actions/cache/golang/action.yml
new file mode 100644
index 000000000..e97a7fee2
--- /dev/null
+++ b/.github/actions/cache/golang/action.yml
@@ -0,0 +1,58 @@
+---
+name: "Restore cache Go build and mod files"
+description: "Restore cache Go build and mod files"
+
+inputs:
+  cache-key-suffix:
+    description: Suffix to append to the cache key
+    required: false
+    default: ${{ github.sha }}
+  refresh-go-cache:
+    description: Flag to control cache refresh
+    default: false
+runs:
+  using: "composite"
+  steps:
+    - name: Restore Go mod (pkg)
+      if: ${{ inputs.refresh-go-cache != 'true' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: |          
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }}
+        restore-keys: |
+          ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }}
+          ${{ runner.os }}-gomod-
+
+    - name: Restore Go build (test)
+      if: ${{ inputs.refresh-go-cache != 'true' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: |
+          ~/.cache/go-build
+        key: ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }}
+        restore-keys: |
+          ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }}
+          ${{ runner.os }}-gobuild-
+
+    - name: Cache Go mod (pkg)
+      if: ${{ inputs.refresh-go-cache == 'true' }}
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }}
+        restore-keys: |
+          ${{ runner.os }}-gomod-${{ hashFiles('**/go.sum') }}
+          ${{ runner.os }}-gomod-
+
+    - name: Cache Go build (test)
+      if: ${{ inputs.refresh-go-cache == 'true' }}
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.cache/go-build
+        key: ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }}-${{ inputs.cache-key-suffix }}
+        restore-keys: |
+          ${{ runner.os }}-gobuild-${{ hashFiles('**/go.sum') }}
+          ${{ runner.os }}-gobuild-
diff --git a/.github/actions/cache/golangci-lint/action.yml b/.github/actions/cache/golangci-lint/action.yml
new file mode 100644
index 000000000..4e741afee
--- /dev/null
+++ b/.github/actions/cache/golangci-lint/action.yml
@@ -0,0 +1,52 @@
+---
+name: "Cache golangci-lint"
+description: "Cache golangci-lint and analysis cache"
+
+inputs:
+  cache-key-suffix:
+    description: Suffix to append to the cache key
+    required: false
+    default: ${{ github.sha }}
+  golangci-lint-version:
+    description: Golangci-lint version to use
+    required: false
+    default: v1.51.1
+  refresh-analysis-cache:
+    description: Flag to control if golangci-lint analysis cache needs to be refreshed
+    default: false
+runs:
+  using: "composite"
+  steps:
+    - name: Cache golangci-lint
+      uses: actions/cache@v3
+      id: cache-golangci-lint
+      with:
+        path: ~/go/bin/golangci-lint
+        key: ${{ runner.os }}-golangci-lint@${{ inputs.golangci-lint-version }}
+
+    - name: Install golangci-lint
+      if: steps.cache-golangci-lint.outputs.cache-hit != 'true'
+      shell: bash
+      run: go install github.com/golangci/golangci-lint/cmd/golangci-lint@${{ inputs.golangci-lint-version }}
+
+    - name: Restore golangci-lint analysis cache
+      if: ${{ inputs.refresh-analysis-cache != 'true' }}
+      uses: actions/cache/restore@v3
+      with:
+        path: ~/.cache/golangci-lint
+        # This technique will make the cache key unique to the commit SHA,
+        # so that we can still hit cache using the restore-key and renew the cache using the key.
+        key: ${{ runner.os }}-golangci-lint-analysis-cache-${{ inputs.cache-key-suffix }}
+        restore-keys: |
+          ${{ runner.os }}-golangci-lint-analysis-cache-
+
+    - name: Cache golangci-lint analysis cache
+      if: ${{ inputs.refresh-analysis-cache == 'true' }}
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/golangci-lint
+        # This technique will make the cache key unique to the commit SHA,
+        # so that we can still hit cache using the restore-key and renew the cache using the key.
+        key: ${{ runner.os }}-golangci-lint-analysis-cache-${{ inputs.cache-key-suffix }}
+        restore-keys: |
+          ${{ runner.os }}-golangci-lint-analysis-cache-
diff --git a/.github/actions/rewind/action.yaml b/.github/actions/rewind/action.yaml
new file mode 100644
index 000000000..45d5a198c
--- /dev/null
+++ b/.github/actions/rewind/action.yaml
@@ -0,0 +1,19 @@
+---
+name: "Test chain rewind"
+
+runs:
+  using: composite
+  steps:
+    - name: Setup
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.20.x
+
+    - name: Build
+      shell: bash
+      run: |
+        make geth
+
+    - name: Test
+      shell: bash
+      run: ./.github/scripts/rewind_test.sh
diff --git a/.github/scripts/bootstrap_snap_sync.sh b/.github/scripts/bootstrap_snap_sync.sh
new file mode 100755
index 000000000..8cb7c4ef7
--- /dev/null
+++ b/.github/scripts/bootstrap_snap_sync.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# This script can be used to start a local geth node from genesis against a specified
+# network. For example, `./.github/scripts/dev.toml` is a config for connecting to devnet
+# if you have port-forwarded to the p2p partner pod via `kubectl port-forward pod/zkevm-geth-partner-0 30300:30300`.
+
+set -e
+set -o pipefail
+
+# Clean up subprocesses on exit
+_exit() {
+  pkill geth || true
+  # sleep 5s to let it kill
+  sleep 5
+}
+trap _exit EXIT
+
+# Set the env var to enable long range sync
+export GETH_FLAG_IMMUTABLE_LONG_RANGE_SYNC="1"
+
+./build/bin/geth immutable bootstrap local \
+--env devnet \
+--syncmode snap \
+--gcmode full \
+--config ./.github/scripts/dev.toml \
+--boots "0" \
+--rpcs "1" \
+--validators "0"
diff --git a/.github/scripts/bootstrap_test.sh b/.github/scripts/bootstrap_test.sh
new file mode 100755
index 000000000..a91cf96ec
--- /dev/null
+++ b/.github/scripts/bootstrap_test.sh
@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# These tests bootstrap a local network and run tests through the eth JSON RPC.
+# The network will undergo a number of hard forks between various test fixtures.
+# The Prevrandao fork is intended to be <= Shanghai fork.
+
+# TODO(serge): move these tests to a proper test suite implemented in Go
+# when we know what the final test suite will look like.
+# Features for test suite:
+# * Set of fixtures that create transactions affected by all EVM-related hard forks
+# * Fixtures can be run with expectation of success or failure depending on fork state
+
+set -e
+set -o pipefail
+
+# necho prints a line without a newline
+function necho() {
+  echo -n "> $*: "
+}
+
+# Clean up subprocesses on exit
+_exit() {
+  pkill geth || true
+  # sleep 5s to let it kill
+  sleep 5
+}
+trap _exit EXIT
+
+# Assign in case PWD changes
+dir="$PWD"
+log="/tmp/bootstrapout"
+boots=2
+validators=1
+rpcs=2
+# Set prevrandao override to be before shanghai
+now=$(date +%s)
+prevrandao_timestamp=$now
+# Set shanghai override to be after genesis block and pre-shanghai tests
+shanghai_timestamp=$((now+30))
+# Set cancun override a few blocks after shanghai
+cancun_timestamp=$((shanghai_timestamp+4))
+
+export GETH_FLAG_NET_RESTRICT="0.0.0.0/0"
+export GETH_FLAG_P2P_SUBNET="127.0.0.1/32"
+
+function start_geth() {
+  necho "Starting geth"
+
+  # Bootstrap and run local network
+  ./build/bin/geth immutable bootstrap local \
+  --boots "$boots" \
+  --rpcs "$rpcs" \
+  --validators "$validators" \
+  --blocklistfilepath "$dir/cmd/geth/testdata/acl_list.txt" \
+  --override.prevrandao="$prevrandao_timestamp" \
+  --override.shanghai="$shanghai_timestamp" \
+  --override.cancun="$cancun_timestamp" > "$log" 2>&1 &
+
+  # Wait for geth processes to start
+  while [ "$(pgrep geth | wc -l)" -lt "$((boots+rpcs+1))" ]; do
+    sleep 1
+    echo -n "."
+  done
+  echo ""
+  necho "To view geth logs run"
+  echo "tail -f $log"
+  sleep 2
+}
+
+function wait_shanghai() {
+  # Wait for shanghai timestamp
+  now=$(date +%s)
+  if [ "$now" -lt "$shanghai_timestamp" ]; then
+    diff=$((shanghai_timestamp-now+5))
+    necho "Waiting $diff seconds for shanghai timestamp"
+    sleep "$diff"
+  fi
+  echo ""
+}
+
+# Run geth
+start_geth
+
+# Need to use 0x44 opcode for Prevrandao fork.
+echo "> Running post-prevrandao, pre-shanghai tests"
+go test -count=1 -v ./tests/immutable \
+-privkey="$dir/cmd/geth/testdata/key.prv" \
+-rpc=http://localhost:8546 \
+-validatoradmin=http://localhost:8545 \
+-skipvoting=true \
+-run='.*Randao.*'
+
+# Wait for shanghai override
+wait_shanghai
+
+# Run tests
+# Need to use contracts compiled with solc relevant to Shanghai fork.
+echo "> Running post-fork tests"
+go test -count=1 -v ./tests/immutable \
+-privkey="$dir/cmd/geth/testdata/key.prv" \
+-blockedprivkey="$dir/cmd/geth/testdata/blockedkey.prv" \
+-rpc=http://localhost:8546 \
+-validatoradmin=http://localhost:8545 \
+-skipvoting=true
+
+echo "> Running vote test" # Tests after this will fail due to stalled block production
+./build/bin/geth immutable vote add --voters http://localhost:8545 --validator 0x7442eD1e3c9FD421F47d12A2742AfF5DaFBf43f8
+echo "Tests finished successfully"
diff --git a/.github/scripts/dev.toml b/.github/scripts/dev.toml
new file mode 100644
index 000000000..b2802ec1e
--- /dev/null
+++ b/.github/scripts/dev.toml
@@ -0,0 +1,83 @@
+[Eth]
+NetworkId = 15003
+SyncMode = "full"
+EthDiscoveryURLs = []
+SnapDiscoveryURLs = []
+NoPruning = true
+NoPrefetch = false
+TxLookupLimit = 2350000
+TransactionHistory = 2350000
+StateScheme = "hash"
+DatabaseCache = 512
+DatabaseFreezer = ""
+TrieCleanCache = 256
+TrieDirtyCache = 256
+TrieTimeout = 3600000000000
+SnapshotCache = 64
+Preimages = false
+FilterLogCacheSize = 32
+EnablePreimageRecording = false
+RPCGasCap = 50000000
+RPCEVMTimeout = 5000000000
+RPCTxFeeCap = 0e+00
+
+[Eth.Miner]
+GasFloor = 0
+GasCeil = 30000000
+GasPrice = 10000000000
+Recommit = 999999999000000000
+NewPayloadTimeout = 2000000000
+
+[Eth.TxPool]
+Locals = []
+NoLocals = true
+Journal = "transactions.rlp"
+Rejournal = 3600000000000
+PriceLimit = 10000000000
+PriceBump = 10
+AccountSlots = 16
+GlobalSlots = 5120
+AccountQueue = 64
+GlobalQueue = 1024
+Lifetime = 3600000000000
+
+[Eth.BlobPool]
+Datadir = "blobpool"
+Datacap = 1
+PriceBump = 100
+
+[Eth.GPO]
+Blocks = 120
+Percentile = 60
+MaxHeaderHistory = 1024
+MaxBlockHistory = 1024
+MaxPrice = 500000000000
+IgnorePrice = 2
+
+[Node]
+DataDir = ""
+IPCPath = "geth.ipc"
+HTTPHost = ""
+HTTPModules = []
+WSHost = ""
+WSModules = []
+AllowUnprotectedTxs = true
+
+[Node.P2P]
+MaxPeers = 100
+NoDiscovery = true
+StaticNodes = [
+	"enode://56724a169266dedfbb41dc50a1f237f4b55569fadccce347c2bb25526c1357b6ff9450ce91c0f21a13abb02fa348ad9e928e8a4fee969a1f14a943c8f6409fb1@127.0.0.1:30300",
+]
+TrustedNodes = [
+	"enode://56724a169266dedfbb41dc50a1f237f4b55569fadccce347c2bb25526c1357b6ff9450ce91c0f21a13abb02fa348ad9e928e8a4fee969a1f14a943c8f6409fb1@127.0.0.1:30300",
+]
+ListenAddr = ""
+DiscAddr = ""
+EnableMsgEvents = false
+
+[Node.HTTPTimeouts]
+ReadTimeout = 0
+ReadHeaderTimeout = 0
+WriteTimeout = 0
+IdleTimeout = 0
diff --git a/.github/scripts/rewind_test.sh b/.github/scripts/rewind_test.sh
new file mode 100755
index 000000000..8ca468234
--- /dev/null
+++ b/.github/scripts/rewind_test.sh
@@ -0,0 +1,167 @@
+#!/bin/bash
+
+# These tests bootstrap a local network and run tests through the eth JSON RPC.
+
+set -e
+set -o pipefail
+
+# necho prints a line without a newline
+function necho() {
+  echo -n "> $*: "
+}
+
+# Clean up subprocesses on exit
+function stop_geth() {
+  pkill geth || true
+  # Wait for kill
+  sleep 1
+  while [ "$(pgrep geth | wc -l)" -gt 1 ]; do
+    sleep 1
+    echo -n "."
+  done
+}
+
+function _exit() {
+  stop_geth
+}
+trap _exit EXIT
+
+# Assign in case PWD changes
+dir="$PWD"
+log="/tmp/rewindout"
+boots=0
+validators=1
+rpcs=0
+now=$(date +%s)
+prevrandao_timestamp=$now
+shanghai_timestamp=$now
+cancun_timestamp=$now
+root_dir="$(mktemp -d)"
+
+function start_geth() {
+  necho "Starting geth"
+  dir="$root_dir/devnet/chain-15003/validator-0"
+  export GETH_FLAG_PASSWORD_FILEPATH="$dir/password"
+
+  ./build/bin/geth \
+  --datadir "$dir" --log.debug --networkid 15003 --metrics --metrics.addr 127.0.0.1 \
+  --metrics.port 6060 --authrpc.port 8550 --verbosity 4 --port 30300 \
+  --rpc.debugdisable --rpc.txpooldisable \
+  --config "$root_dir/devnet/chain-15003/config.toml" --pprof --pprof.port 7070 \
+  --miner.etherbase "$(cat "$dir/address")" \
+  --mine --http --http.port 8545 --cache 128 --cache.database 35 \
+  --cache.trie 35 --cache.gc 10 --cache.snapshot 20 --gcmode archive --syncmode full \
+  --override.prevrandao="$prevrandao_timestamp" \
+  --override.shanghai="$shanghai_timestamp" \
+  --override.cancun="$cancun_timestamp" >> "$log" 2>&1 &
+
+  # Wait for geth processes to start
+  while [ "$(pgrep geth | wc -l)" -lt "$((boots+rpcs+1))" ]; do
+    sleep 1
+    echo -n "."
+  done
+  echo ""
+  necho "To view geth logs run"
+  echo "tail -f $log"
+}
+
+function bootstrap_geth() {
+  necho "Bootstrapping geth"
+
+  # Bootstrap and run local network
+  ./build/bin/geth immutable bootstrap local \
+  --datadir "$root_dir" \
+  --boots "$boots" \
+  --rpcs "$rpcs" \
+  --validators "$validators" \
+  --blocklistfilepath "$dir/cmd/geth/testdata/acl_list.txt" \
+  --override.prevrandao="$prevrandao_timestamp" \
+  --override.shanghai="$shanghai_timestamp" \
+  --override.cancun="$cancun_timestamp" >> "$log" 2>&1 &
+
+  # Wait for geth processes to start
+  while [ "$(pgrep geth | wc -l)" -lt "$((boots+rpcs+1))" ]; do
+    sleep 1
+    echo -n "."
+  done
+  echo ""
+  necho "To view geth logs run"
+  echo "tail -f $log"
+}
+
+function get_head() {
+  resp=$(curl -Ss --location 'http://127.0.0.1:8545' \
+  --header 'Content-Type: application/json' \
+  --data '{
+  	"jsonrpc":"2.0",
+  	"method":"eth_getBlockByNumber",
+  	"params":[
+  		"latest", 
+  		true
+  	],
+  	"id":1
+  }')
+  block=$(echo "$resp" | jq -r '.result.number')
+  block=$((block))
+  if [ "$block" == 0 ]; then
+    echo "Chain not progressing, exiting"
+    exit 1
+  fi
+}
+
+# Run geth
+rm "$log" || true
+bootstrap_geth
+
+# Check block after a few should have been sealed
+sleep 20
+get_head
+pre_rewind_block="$block"
+echo "Block before rewind: $pre_rewind_block"
+
+# Kill the geth process and rewind
+echo "Killing geth"
+stop_geth
+
+echo "Rewinding chain"
+rewind_block=0
+./build/bin/geth immutable rewind \
+--datadir "$root_dir/devnet/chain-15003/validator-0" \
+"$rewind_block" >> "$log" 2>&1
+cat "$root_dir/devnet/chain-15003/validator-0/rewind_history.yaml"
+
+# Restart geth and wait for smaller period of time than pre rewind
+start_geth
+sleep 8
+
+# Check latest header and compare against pre-rewind block
+get_head
+echo "Block after rewind: $block"
+if [ "$block" -ge "$pre_rewind_block" ]; then
+  echo "Latest ($block) is not less than pre-rewind block ($pre_rewind_block)"
+  exit 1
+fi
+
+# Kill the geth process and rewind to same block again, expecting no change
+sleep 10 # Wait for chain to progress further beyond `block`
+echo "Killing geth"
+stop_geth
+
+echo "Rewinding chain again"
+pre_rewind_block="$block"
+rewind_block=0
+./build/bin/geth immutable rewind \
+--datadir "$root_dir/devnet/chain-15003/validator-0" \
+"$rewind_block" >> "$log" 2>&1
+cat "$root_dir/devnet/chain-15003/validator-0/rewind_history.yaml"
+
+start_geth
+sleep 8
+get_head
+echo "Block after second rewind: $block"
+if [ "$block" -le "$pre_rewind_block" ]; then
+  echo "Latest ($block) should be greater than pre rewind block ($pre_rewind_block)"
+  exit 1
+fi
+
+echo "Tests finished successfully"
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
new file mode 100644
index 000000000..02994c378
--- /dev/null
+++ b/.github/workflows/pr.yaml
@@ -0,0 +1,76 @@
+name: PR
+
+on:
+  pull_request:
+  push:
+    branches: master
+
+concurrency:
+  group: PR-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  bootstrap:
+    name: Local E2E Tests
+    runs-on: ubuntu-20-04-4-cores
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache modules
+        uses: ./.github/actions/cache/golang
+      - uses: ./.github/actions/bootstrap
+
+  rewind:
+    name: Rewind Chain Tests
+    runs-on: ubuntu-20-04-4-cores
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - name: Cache modules
+        uses: ./.github/actions/cache/golang
+      - uses: ./.github/actions/rewind
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-20-04-4-cores
+    timeout-minutes: 35
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # We need the full history to get the base commit in order to compute the diff in golanci-lint
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+      - name: Cache modules
+        uses: ./.github/actions/cache/golang
+      - name: Cache linter
+        uses: ./.github/actions/cache/golangci-lint
+      - name: Debug golangci-lint cache
+        run: |
+          golangci-lint cache status
+      - name: Lint
+        shell: bash
+        run: |
+          golangci-lint run \
+            --new-from-rev=${{ github.event.pull_request.base.sha || 'HEAD~'}} \
+            --concurrency=4 \
+            --out-format=github-actions \
+            --config .golangci.yml \
+            ./...
+
+  tests:
+    name: Unit Tests
+    runs-on: ubuntu-20-04-4-cores
+    timeout-minutes: 35
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+      - name: Cache modules
+        uses: ./.github/actions/cache/golang
+      - name: All Tests
+        shell: bash
+        run: |
+          go run build/ci.go install
+          go test $(go list ./... | grep -v 'go-ethereum/tests/immutable') -p 1 --parallel 4 -timeout=30m
\ No newline at end of file