Merge remote-tracking branch 'eth/master'

.circleci/config.yml

@@ -1,79 +1,14 @@
 version: 2.1
-orbs:
-  go: circleci/[email protected]
-
-jobs:
-  # This job builds the hive executable and stores it in the workspace.
-  build:
-    docker:
-      - image: cimg/go:1.21
-    steps:
-      # Build it.
-      - checkout
-      - go/load-cache
-      - go/mod-download
-      - go/save-cache
-      - run: {command: 'go build -ldflags="-s -extldflags=-static" -tags "osusergo netgo static_build" .'}
-      # Store the executable.
-      - persist_to_workspace:
-          root: .
-          paths: ["hive"]
 
-  # This job runs the smoke test simulations. This requires a virtual
-  # machine instead of the container-based build environment because
-  # hive needs to be able to talk to the docker containers it creates.
-  smoke-tests:
-    machine:
-      image: ubuntu-2004:202201-02
-    steps:
-      - checkout
-      - attach_workspace: {at: "/tmp/build"}
-      - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum"
-      - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum"
+setup: true
 
-  # This job also runs the smoke test simulations, but against a remote dockerd.
-  smoke-tests-remote-docker:
-    docker:
-      - image: cimg/base:2022.04
-    steps:
-      - checkout
-      - attach_workspace: {at: "/tmp/build"}
-      - setup_remote_docker: {version: 20.10.14}
-      - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum --loglevel 5"
-      - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum --loglevel 5"
-
-  # This job runs the go unit tests.
-  go-test:
-    docker:
-      - image: cimg/go:1.21
-    steps:
-      # Get the source.
-      - checkout
-      - go/load-cache
-      - go/mod-download
-      - go/save-cache
-      # Run the tests.
-      - run:
-          name: "hive module tests"
-          command: "go test -cover ./..."
-      - run:
-          name: "hiveproxy module tests"
-          command: "go test -cover ./..."
-          working_directory: "./hiveproxy"
-      - run:
-          name: "Compile Go simulators"
-          command: ".circleci/compile-simulators.sh"
+orbs:
+  path-filtering: circleci/[email protected]
 
 workflows:
-  main:
+  setup-workflow:
     jobs:
-      - go-test
-      - build
-      - smoke-tests:
-          requires: ["build"]
-      - smoke-tests-remote-docker:
-          requires: ["build"]
+      - path-filtering/filter:
+          mapping: |
+            simulators/portal/.* rust-ci true
+          base-revision: origin/master

.circleci/continue_config.yml

@@ -0,0 +1,106 @@
+version: 2.1
+orbs:
+  go: circleci/[email protected]
+
+parameters:
+  rust-ci:
+    type: boolean
+    default: false  
+
+jobs:
+  # This job builds the hive executable and stores it in the workspace.
+  build:
+    docker:
+      - image: cimg/go:1.21
+    steps:
+      # Build it.
+      - checkout
+      - go/load-cache
+      - go/mod-download
+      - go/save-cache
+      - run: {command: 'go build -ldflags="-s -extldflags=-static" -tags "osusergo netgo static_build" .'}
+      # Store the executable.
+      - persist_to_workspace:
+          root: .
+          paths: ["hive"]
+
+  # This job runs the smoke test simulations. This requires a virtual
+  # machine instead of the container-based build environment because
+  # hive needs to be able to talk to the docker containers it creates.
+  smoke-tests:
+    machine:
+      image: ubuntu-2004:202201-02
+    steps:
+      - checkout
+      - attach_workspace: {at: "/tmp/build"}
+      - run:
+          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum"
+      - run:
+          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum"
+
+  # This job also runs the smoke test simulations, but against a remote dockerd.
+  smoke-tests-remote-docker:
+    docker:
+      - image: cimg/base:2022.04
+    steps:
+      - checkout
+      - attach_workspace: {at: "/tmp/build"}
+      - setup_remote_docker: {version: 20.10.14}
+      - run:
+          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum --loglevel 5"
+      - run:
+          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum --loglevel 5"
+
+  # This job runs the go unit tests.
+  go-test:
+    docker:
+      - image: cimg/go:1.21
+    steps:
+      # Get the source.
+      - checkout
+      - go/load-cache
+      - go/mod-download
+      - go/save-cache
+      # Run the tests.
+      - run:
+          name: "hive module tests"
+          command: "go test -cover ./..."
+      - run:
+          name: "hiveproxy module tests"
+          command: "go test -cover ./..."
+          working_directory: "./hiveproxy"
+      - run:
+          name: "Compile Go simulators"
+          command: ".circleci/compile-simulators.sh"
+  # this makes sure the rust code is good
+  rust-simulators:
+    docker:
+      - image: cimg/rust:1.75.0
+    steps:
+      - checkout
+      - run:
+          name: Install rustfmt
+          command: rustup component add rustfmt
+      - run:
+          name: Install Clippy
+          command: rustup component add clippy
+      - run:
+          name: Install Clang
+          command: sudo apt update && sudo apt-get install clang -y
+      - run:
+          name: "Lint, build, test Rust simulators"
+          command: ".circleci/rust-simulators.sh"          
+
+workflows:
+  main:
+    jobs:
+      - go-test
+      - build
+      - smoke-tests:
+          requires: ["build"]
+      - smoke-tests-remote-docker:
+          requires: ["build"]
+  rust-jobs:
+    when: << pipeline.parameters.rust-ci >>
+    jobs:
+      - rust-simulators

.circleci/rust-simulators.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# This causes the bash script to exit immediately if any commands errors out
+set -e
+
+failed=""
+sims=$(find simulators -name Cargo.toml)
+for d in $sims; do
+    d="$(dirname "$d")"
+    echo "Lint, build, test $d"
+    ( cd "$d" || exit 1;
+    cargo fmt --all -- --check;
+    cargo clippy --all --all-targets --all-features --no-deps -- --deny warnings;
+    cargo test --workspace -- --nocapture;
+    )
+done

.gitignore

@@ -3,5 +3,9 @@ workspace
 .idea/
 # build output
 /hive
-
-hiveview
+
+hiveview
+
+# build output for rust simulators files
+simulators/**/Cargo.lock
+simulators/**/target

clients/besu/Dockerfile.git

@@ -8,7 +8,7 @@ ARG github=hyperledger/besu
 
 RUN echo "installing java on ubuntu base image" \
     && apt-get update && apt-get install -y git libsodium-dev libnss3-dev \
-    && apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909 \
+    && apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909* \
     && apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.* \
     && echo "Cloning: $github - $tag" \
     && git clone --depth 1 --branch $tag https://github.com/$github \

clients/besu/besu.sh

@@ -156,7 +156,7 @@ RPCFLAGS="$RPCFLAGS --rpc-ws-enabled --rpc-ws-api=DEBUG,ETH,NET,WEB3,ADMIN --rpc
 # Enable merge support if needed
 if [ "$HIVE_TERMINAL_TOTAL_DIFFICULTY" != "" ]; then
     echo "0x7365637265747365637265747365637265747365637265747365637265747365" > /jwtsecret
-    RPCFLAGS="$RPCFLAGS --engine-host-allowlist=* --engine-jwt-enabled --engine-jwt-secret /jwtsecret"
+    RPCFLAGS="$RPCFLAGS --engine-host-allowlist=* --engine-jwt-secret /jwtsecret"
 fi
 
 # Start Besu.

clients/erigon/Dockerfile.git

@@ -10,7 +10,7 @@ RUN echo "Cloning: $github - $tag" \
     && apk add bash build-base ca-certificates git jq \
     && git clone --depth 1 --branch $tag https://github.com/$github \
     && cd erigon \
-    && make erigon \
+    && make BUILD_TAGS=nosqlite,noboltdb,nosilkworm erigon \
     && cp build/bin/erigon /usr/local/bin/erigon
 
 ## Final stage: Sets up the environment for running erigon

clients/erigon/erigon.sh

@@ -139,6 +139,9 @@ if [ "$HIVE_TERMINAL_TOTAL_DIFFICULTY" != "" ]; then
     FLAGS="$FLAGS --authrpc.addr=0.0.0.0 --authrpc.jwtsecret=/jwt.secret"
 fi
 
+# Configure snapshots.
+FLAGS="$FLAGS --snapshots=false"
+
 # Launch the main client.
 FLAGS="$FLAGS --nat=none"
 echo "Running erigon with flags $FLAGS"

clients/erigon/mapper.jq

@@ -54,7 +54,7 @@ def to_bool:
     "grayGlacierBlock": env.HIVE_FORK_GRAY_GLACIER|to_int,
     "mergeNetsplitBlock": env.HIVE_MERGE_BLOCK_ID|to_int,
     "terminalTotalDifficulty": env.HIVE_TERMINAL_TOTAL_DIFFICULTY|to_int,
-    "terminalTotalDifficultyPassed": (if env.HIVE_TERMINAL_TOTAL_DIFFICULTY_PASSED == null then 1 else env.HIVE_TERMINAL_TOTAL_DIFFICULTY_PASSED|to_bool end),
+    "terminalTotalDifficultyPassed": (if env.HIVE_TERMINAL_TOTAL_DIFFICULTY_PASSED == null then true else env.HIVE_TERMINAL_TOTAL_DIFFICULTY_PASSED|to_bool end),
     "shanghaiTime": env.HIVE_SHANGHAI_TIMESTAMP|to_int,
     "cancunTime": env.HIVE_CANCUN_TIMESTAMP|to_int,
   }|remove_empty

clients/fluffy/Dockerfile

@@ -0,0 +1,12 @@
+ARG branch=amd64-master-latest
+
+FROM statusim/nimbus-fluffy:$branch
+
+ADD fluffy.sh /fluffy.sh
+RUN chmod +x /fluffy.sh
+
+RUN echo "latest" > /version.txt
+
+EXPOSE 8545 9009/udp
+
+ENTRYPOINT ["/fluffy.sh"]

clients/fluffy/fluffy.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Immediately abort the script on any error encountered
+set -e
+
+IP_ADDR=$(hostname -i | awk '{print $1}')
+FLAGS=""
+
+if [ "$HIVE_PORTAL_NETWORKS_SELECTED" != "" ]; then
+    if [[ $HIVE_PORTAL_NETWORKS_SELECTED =~ "beacon" ]]; then
+        # Providing atrusted block root is required currently to enable the beacon network.
+        # It can be a made up value for now as tests are not doing any sync.
+        FLAGS="$FLAGS --trusted-block-root:0x0000000000000000000000000000000000000000000000000000000000000000"
+    fi
+fi
+
+
+if [ "$HIVE_CLIENT_PRIVATE_KEY" != "" ]; then
+    FLAGS="$FLAGS --netkey-unsafe=0x$HIVE_CLIENT_PRIVATE_KEY"
+fi
+
+# Fluffy runs all networks by default, so we can not configure to run networks individually
+fluffy --rpc --rpc-address="0.0.0.0" --nat:extip:"$IP_ADDR" --network=none --log-level="debug" $FLAGS

clients/fluffy/hive.yaml

@@ -0,0 +1,2 @@
+roles:
+  - portal

clients/go-ethereum/geth.sh

@@ -140,7 +140,6 @@ fi
 if [ "$HIVE_MINER_EXTRA" != "" ]; then
     FLAGS="$FLAGS --miner.extradata $HIVE_MINER_EXTRA"
 fi
-FLAGS="$FLAGS --miner.gasprice 16000000000"
 
 # Configure LES.
 if [ "$HIVE_LES_SERVER" == "1" ]; then

clients/lighthouse-bn/lighthouse_bn.sh

@@ -51,6 +51,10 @@ if [[ "$HIVE_ETH2_BEACON_NODE_INDEX" != "" ]]; then
     fi
 fi
 
+if [[ "$HIVE_ETH2_TRUSTED_PEER_IDS" != "" ]]; then
+    trustedpeers="$trustedpeers,$HIVE_ETH2_TRUSTED_PEER_IDS"
+fi
+
 LOG=info
 case "$HIVE_LOGLEVEL" in
     0|1) LOG=error ;;

clients/lodestar-bn/lodestar_bn.sh

@@ -38,6 +38,7 @@ bootnodes_option=$([[ "$HIVE_ETH2_BOOTNODE_ENRS" == "" ]] && echo "" || echo "--
 metrics_option=$([[ "$HIVE_ETH2_METRICS_PORT" == "" ]] && echo "" || echo "--metrics --metrics.address=$CONTAINER_IP --metrics.port=$HIVE_ETH2_METRICS_PORT")
 builder_option=$([[ "$HIVE_ETH2_BUILDER_ENDPOINT" == "" ]] && echo "" || echo "--builder --builder.urls $HIVE_ETH2_BUILDER_ENDPOINT")
 echo BUILDER=$builder_option
+peer_score_option=$([[ "$HIVE_ETH2_DISABLE_PEER_SCORING" == "" ]] && echo "" || echo "--disablePeerScoring")
 
 echo "bootnodes option : ${bootnodes_option}"
 
@@ -64,6 +65,7 @@ node /usr/app/node_modules/.bin/lodestar \
     $metrics_option \
     $bootnodes_option \
     $builder_option \
+    $peer_score_option \
     --enr.ip="${CONTAINER_IP}" \
     --enr.tcp="${HIVE_ETH2_P2P_TCP_PORT:-9000}" \
     --enr.udp="${HIVE_ETH2_P2P_UDP_PORT:-9000}" \

clients/nethermind/Dockerfile.git

@@ -1,7 +1,7 @@
 ### Build Nethermind From Git:
 
 ## Builder stage: Compiles nethermind from a git repository
-FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
+FROM mcr.microsoft.com/dotnet/sdk:8.0-jammy AS build
 
 ARG github=nethermindeth/nethermind
 ARG tag=master

clients/nimbus-el/nimbus.sh

@@ -115,14 +115,17 @@ fi
 
 set -e
 
-# Configure RPC.
+# Configure RPC
+FLAGS="$FLAGS --http-address:0.0.0.0 --http-port:8545"
+FLAGS="$FLAGS --rpc --rpc-api:eth,debug"
+FLAGS="$FLAGS --ws --ws-api:eth,debug"
+
+# Configure graphql
 if [ "$HIVE_GRAPHQL_ENABLED" != "" ]; then
-  FLAGS="$FLAGS --graphql --graphql-address:0.0.0.0 --graphql-port:8545"
-else
-  FLAGS="$FLAGS --rpc --rpc-api:eth,debug --rpc-address:0.0.0.0 --rpc-port:8545"
-  FLAGS="$FLAGS --ws --ws-api:eth,debug --ws-address:0.0.0.0 --ws-port:8546"
+  FLAGS="$FLAGS --graphql"
 fi
 
+# Configure engine api
 if [ "$HIVE_TERMINAL_TOTAL_DIFFICULTY" != "" ]; then
   echo "0x7365637265747365637265747365637265747365637265747365637265747365" > /jwtsecret
   FLAGS="$FLAGS --engine-api:true --engine-api-address:0.0.0.0 --engine-api-port:8551 --jwt-secret:/jwtsecret"

clients/prysm-vc/prysm_vc.sh

@@ -50,11 +50,10 @@ echo Starting Prysm Validator Client
     --verbosity="$LOG" \
     --accept-terms-of-use=true \
     --prater \
-    --beacon-rpc-provider="$HIVE_ETH2_BN_API_IP:${HIVE_ETH2_BN_GRPC_PORT:-3500}" \
-    --beacon-rpc-gateway-provider="$HIVE_ETH2_BN_API_IP:${HIVE_ETH2_BN_API_PORT:-4000}" \
+    --enable-beacon-rest-api=true \
+    --beacon-rest-api-provider="http://$HIVE_ETH2_BN_API_IP:${HIVE_ETH2_BN_API_PORT:-4000}" \
     --datadir="/data/vc" \
     --wallet-dir="/data/validators" \
     --wallet-password-file="/wallet.pass" \
     --chain-config-file="/hive/input/config.yaml" \
-    $builder_option
-# NOTE: gRPC/RPC ports are inverted to allow the simulator to access the REST API
+    $builder_option