diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6cdd0bd4a8b9..00f51054ed15 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -6,9 +6,10 @@ jobs: linter-check: runs-on: ubuntu-latest steps: - - uses: actions/setup-go@v2 + - uses: actions/setup-go@v3 with: - go-version: '1.16.x' + go-version: ">=1.18.0" + check-latest: true - uses: actions/checkout@v2 - name: Run linter - run: go run build/ci.go lint \ No newline at end of file + run: go run build/ci.go lint diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index af20fb410880..2b05dcda5f8f 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -3,64 +3,5 @@ name: Build and test on: [pull_request] jobs: - build: - name: Run docker build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Run docker build - run: docker build -t vulcanize/go-ethereum . - - geth-unit-test: - name: Run geth unit test - strategy: - matrix: - go-version: [ 1.16.x] - platform: [ubuntu-latest] - runs-on: ${{ matrix.platform }} - env: - GO111MODULE: on - GOPATH: /tmp/go - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Run unit tests - run: | - make test - - statediff-unit-test: - name: Run state diff unit test - env: - GOPATH: /tmp/go - strategy: - matrix: - go-version: [ 1.16.x] - platform: [ubuntu-latest] - runs-on: ${{ matrix.platform }} - steps: - - name: Create GOPATH - run: mkdir -p /tmp/go - - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - - name: Checkout code - uses: actions/checkout@v2 - - - name: Start database - run: docker-compose -f docker-compose.yml up -d ipld-eth-db - - - name: Run unit tests - run: - make statedifftest \ No newline at end of file + run-tests: + uses: ./.github/workflows/tests.yml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 5b8453e8fc92..610f8afbff82 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -3,8 +3,11 @@ on: release: types: [published] jobs: + run-tests: + uses: ./.github/workflows/tests.yml build: name: Run docker build and publish + needs: run-tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000000..26aa10770c75 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,128 @@ +name: Tests for Geth that are used in multiple jobs. + +on: + workflow_call: + +env: + stack-orchestrator-ref: ${{ github.event.inputs.stack-orchestrator-ref || 'main'}} + ipld-eth-db-ref: ${{ github.event.inputs.ipld-ethcl-db-ref || 'main' }} + GOPATH: /tmp/go + +jobs: + build: + name: Run docker build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Run docker build + run: docker build -t vulcanize/go-ethereum . + + geth-unit-test: + name: Run geth unit test + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.18.0" + check-latest: true + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Run unit tests + run: | + make test + + statediff-unit-test: + name: Run state diff unit test + runs-on: ubuntu-latest + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.18.0" + check-latest: true + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Start database + run: docker-compose -f docker-compose.yml up -d ipld-eth-db + + - name: Run unit tests + run: make statedifftest + + private-network-test: + name: Start Geth in a private network. + runs-on: ubuntu-latest + steps: + - name: Create GOPATH + run: mkdir -p /tmp/go + + - uses: actions/setup-go@v3 + with: + go-version: ">=1.18.0" + check-latest: true + + - name: Checkout code + uses: actions/checkout@v3 + with: + path: "./go-ethereum" + + - uses: actions/checkout@v3 + with: + ref: ${{ env.stack-orchestrator-ref }} + path: "./stack-orchestrator/" + repository: vulcanize/stack-orchestrator + fetch-depth: 0 + + - uses: actions/checkout@v3 + with: + ref: ${{ env.ipld-eth-db-ref }} + repository: vulcanize/ipld-eth-db + path: "./ipld-eth-db/" + fetch-depth: 0 + + - name: Create config file + run: | + echo vulcanize_ipld_eth_db=$GITHUB_WORKSPACE/ipld-eth-db/ > $GITHUB_WORKSPACE/config.sh + echo vulcanize_go_ethereum=$GITHUB_WORKSPACE/go-ethereum/ >> $GITHUB_WORKSPACE/config.sh + echo db_write=true >> $GITHUB_WORKSPACE/config.sh + cat $GITHUB_WORKSPACE/config.sh + + - name: Compile Geth + run: | + cd $GITHUB_WORKSPACE/stack-orchestrator/helper-scripts + ./compile-geth.sh -e docker -p $GITHUB_WORKSPACE/config.sh + cd - + + - name: Run docker compose + run: | + docker-compose \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-db.yml" \ + -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" \ + --env-file $GITHUB_WORKSPACE/config.sh \ + up -d --build + + - name: Make sure the /root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS exists within a certain time frame. + shell: bash + run: | + COUNT=0 + ATTEMPTS=10 + until $(docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" cp go-ethereum:/root/transaction_info/STATEFUL_TEST_DEPLOYED_ADDRESS ./STATEFUL_TEST_DEPLOYED_ADDRESS) || [[ $COUNT -eq $ATTEMPTS ]]; do echo -e "$(( COUNT++ ))... \c"; sleep 10; done + [[ $COUNT -eq $ATTEMPTS ]] && echo "Could not find the successful contract deployment" && (exit 1) + cat ./STATEFUL_TEST_DEPLOYED_ADDRESS + sleep 15; + + - name: Create a new transaction. + shell: bash + run: | + docker compose -f "$GITHUB_WORKSPACE/stack-orchestrator/docker/local/docker-compose-go-ethereum.yml" exec go-ethereum /bin/bash /root/transaction_info/NEW_TRANSACTION + echo $? diff --git a/README.md b/README.md index 0987200d3b9a..e3c1b1550975 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,7 @@ Official Golang implementation of the Ethereum protocol. -[![API Reference]( -https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 -)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) +[![API Reference](https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667)](https://pkg.go.dev/github.com/ethereum/go-ethereum?tab=doc) [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum) [![Travis](https://travis-ci.com/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.com/ethereum/go-ethereum) [![Discord](https://img.shields.io/badge/discord-join%20chat-blue.svg)](https://discord.gg/nthXNEv) @@ -12,6 +10,21 @@ https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/6874 Automated builds are available for stable releases and the unstable master branch. Binary archives are published at https://geth.ethereum.org/downloads/. +## Vulcanize Specific + +This section captures components specific to vulcanize. + +### Branching Structure + +We currently follow the following branching structure. + +1. Create a branch: `v1.10.18-statediff-vX` --> feature/some-feature` +2. Create a PR upstream: `feature/some-feature` --> `v1.10.18-statediff-vX` +3. When a release is ready, create a release branch: `v1.10.18-statediff-vX` --> `v1.10.18-statediff-X.Y.Z` +4. When `v1.10.18-statediff-vX` is stable, merge it to `statediff`. + +This process is subject to change. + ## Building the source For prerequisites and detailed build instructions please read the [Installation Instructions](https://geth.ethereum.org/docs/install-and-build/installing-geth). @@ -34,16 +47,16 @@ make all The go-ethereum project comes with several wrappers/executables found in the `cmd` directory. -| Command | Description | -| :-----------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | -| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | -| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | -| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | -| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | -| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | -| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | -| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | +| Command | Description | +| :--------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/interface/command-line-options) for command line options. | +| `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | +| `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | +| `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/dapp/native-bindings) page for details. | +| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | +| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | +| `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://eth.wiki/en/fundamentals/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | +| `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | ## Running `geth` @@ -56,17 +69,17 @@ on how you can run your own `geth` instance. Minimum: -* CPU with 2+ cores -* 4GB RAM -* 1TB free storage space to sync the Mainnet -* 8 MBit/sec download Internet service +- CPU with 2+ cores +- 4GB RAM +- 1TB free storage space to sync the Mainnet +- 8 MBit/sec download Internet service Recommended: -* Fast CPU with 4+ cores -* 16GB+ RAM -* High Performance SSD with at least 1TB free space -* 25+ MBit/sec download Internet service +- Fast CPU with 4+ cores +- 16GB+ RAM +- High Performance SSD with at least 1TB free space +- 25+ MBit/sec download Internet service ### Full node on the main Ethereum network @@ -80,15 +93,16 @@ $ geth console ``` This command will: - * Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), - causing it to download more data in exchange for avoiding processing the entire history - of the Ethereum network, which is very CPU intensive. - * Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), - (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) - (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), - as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). - This tool is optional and if you leave it out you can always attach to an already running - `geth` instance with `geth attach`. + +- Start `geth` in snap sync mode (default, can be changed with the `--syncmode` flag), + causing it to download more data in exchange for avoiding processing the entire history + of the Ethereum network, which is very CPU intensive. +- Start up `geth`'s built-in interactive [JavaScript console](https://geth.ethereum.org/docs/interface/javascript-console), + (via the trailing `console` subcommand) through which you can interact using [`web3` methods](https://github.com/ChainSafe/web3.js/blob/0.20.7/DOCUMENTATION.md) + (note: the `web3` version bundled within `geth` is very old, and not up to date with official docs), + as well as `geth`'s own [management APIs](https://geth.ethereum.org/docs/rpc/server). + This tool is optional and if you leave it out you can always attach to an already running + `geth` instance with `geth attach`. ### A Full node on the Görli test network @@ -107,27 +121,27 @@ useful on the testnet too. Please, see above for their explanations if you've sk Specifying the `--goerli` flag, however, will reconfigure your `geth` instance a bit: - * Instead of connecting the main Ethereum network, the client will connect to the Görli - test network, which uses different P2P bootnodes, different network IDs and genesis - states. - * Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` - will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on - Linux). Note, on OSX and Linux this also means that attaching to a running testnet node - requires the use of a custom endpoint since `geth attach` will try to attach to a - production node endpoint by default, e.g., - `geth attach /goerli/geth.ipc`. Windows users are not affected by - this. - -*Note: Although there are some internal protective measures to prevent transactions from +- Instead of connecting the main Ethereum network, the client will connect to the Görli + test network, which uses different P2P bootnodes, different network IDs and genesis + states. +- Instead of using the default data directory (`~/.ethereum` on Linux for example), `geth` + will nest itself one level deeper into a `goerli` subfolder (`~/.ethereum/goerli` on + Linux). Note, on OSX and Linux this also means that attaching to a running testnet node + requires the use of a custom endpoint since `geth attach` will try to attach to a + production node endpoint by default, e.g., + `geth attach /goerli/geth.ipc`. Windows users are not affected by + this. + +_Note: Although there are some internal protective measures to prevent transactions from crossing over between the main network and test network, you should make sure to always use separate accounts for play-money and real-money. Unless you manually move accounts, `geth` will by default correctly separate the two networks and will not make any -accounts available between them.* +accounts available between them._ ### Full node on the Rinkeby test network Go Ethereum also supports connecting to the older proof-of-authority based test network -called [*Rinkeby*](https://www.rinkeby.io) which is operated by members of the community. +called [_Rinkeby_](https://www.rinkeby.io) which is operated by members of the community. ```shell $ geth --rinkeby console @@ -144,7 +158,7 @@ network's low difficulty/security. $ geth --ropsten console ``` -*Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory.* +_Note: Older Geth configurations store the Ropsten database in the `testnet` subdirectory._ ### Configuration @@ -162,7 +176,7 @@ export your existing configuration: $ geth --your-favourite-flags dumpconfig ``` -*Note: This works only with `geth` v1.6.0 and above.* +_Note: This works only with `geth` v1.6.0 and above._ #### Docker quick start @@ -176,7 +190,7 @@ docker run -d --name ethereum-node -v /Users/alice/ethereum:/root \ ``` This will start `geth` in snap-sync mode with a DB memory allowance of 1GB just as the -above command does. It will also create a persistent volume in your home directory for +above command does. It will also create a persistent volume in your home directory for saving your blockchain as well as map the default ports. There is also an `alpine` tag available for a slim version of the image. @@ -200,19 +214,19 @@ you'd expect. HTTP based JSON-RPC API options: - * `--http` Enable the HTTP-RPC server - * `--http.addr` HTTP-RPC server listening interface (default: `localhost`) - * `--http.port` HTTP-RPC server listening port (default: `8545`) - * `--http.api` API's offered over the HTTP-RPC interface (default: `eth,net,web3`) - * `--http.corsdomain` Comma separated list of domains from which to accept cross origin requests (browser enforced) - * `--ws` Enable the WS-RPC server - * `--ws.addr` WS-RPC server listening interface (default: `localhost`) - * `--ws.port` WS-RPC server listening port (default: `8546`) - * `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) - * `--ws.origins` Origins from which to accept websockets requests - * `--ipcdisable` Disable the IPC-RPC server - * `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) - * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) +- `--http` Enable the HTTP-RPC server +- `--http.addr` HTTP-RPC server listening interface (default: `localhost`) +- `--http.port` HTTP-RPC server listening port (default: `8545`) +- `--http.api` API's offered over the HTTP-RPC interface (default: `eth,net,web3`) +- `--http.corsdomain` Comma separated list of domains from which to accept cross origin requests (browser enforced) +- `--ws` Enable the WS-RPC server +- `--ws.addr` WS-RPC server listening interface (default: `localhost`) +- `--ws.port` WS-RPC server listening port (default: `8546`) +- `--ws.api` API's offered over the WS-RPC interface (default: `eth,net,web3`) +- `--ws.origins` Origins from which to accept websockets requests +- `--ipcdisable` Disable the IPC-RPC server +- `--ipcapi` API's offered over the IPC-RPC interface (default: `admin,debug,eth,miner,net,personal,shh,txpool,web3`) +- `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect via HTTP, WS or IPC to a `geth` node configured with the above flags and you'll @@ -302,8 +316,8 @@ that other nodes can use to connect to it and exchange peer information. Make su replace the displayed IP address information (most probably `[::]`) with your externally accessible IP to get the actual `enode` URL. -*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less -recommended way.* +_Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less +recommended way._ #### Starting up your member nodes @@ -317,8 +331,8 @@ do also specify a custom `--datadir` flag. $ geth --datadir=path/to/custom/data/folder --bootnodes= ``` -*Note: Since your network will be completely cut off from the main and test networks, you'll -also need to configure a miner to process transactions and create new blocks for you.* +_Note: Since your network will be completely cut off from the main and test networks, you'll +also need to configure a miner to process transactions and create new blocks for you._ #### Running a private miner @@ -356,13 +370,13 @@ and merge procedures quick and simple. Please make sure your contributions adhere to our coding guidelines: - * Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) - guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). - * Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) - guidelines. - * Pull requests need to be based on and opened against the `master` branch. - * Commit messages should be prefixed with the package(s) they modify. - * E.g. "eth, rpc: make trace configs optional" +- Code must adhere to the official Go [formatting](https://golang.org/doc/effective_go.html#formatting) + guidelines (i.e. uses [gofmt](https://golang.org/cmd/gofmt/)). +- Code must be documented adhering to the official Go [commentary](https://golang.org/doc/effective_go.html#commentary) + guidelines. +- Pull requests need to be based on and opened against the `master` branch. +- Commit messages should be prefixed with the package(s) they modify. + - E.g. "eth, rpc: make trace configs optional" Please see the [Developers' Guide](https://geth.ethereum.org/docs/developers/devguide) for more details on configuring your environment, managing project dependencies, and diff --git a/statediff/known_gaps.go b/statediff/known_gaps.go index 682903ed437e..df6860c83f93 100644 --- a/statediff/known_gaps.go +++ b/statediff/known_gaps.go @@ -100,7 +100,7 @@ func minMax(array []*big.Int) (*big.Int, *big.Int) { // 3. Write to prometheus directly. // 4. Logs and error. func (kg *KnownGapsState) pushKnownGaps(startingBlockNumber *big.Int, endingBlockNumber *big.Int, checkedOut bool, processingKey int64) error { - if startingBlockNumber.Cmp(endingBlockNumber) != -1 { + if startingBlockNumber.Cmp(endingBlockNumber) == 1 { return fmt.Errorf("Starting Block %d, is greater than ending block %d", startingBlockNumber, endingBlockNumber) } knownGap := models.KnownGapsModel{