Nightly -- Full Network Tests #45
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly -- Full Network Tests | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| env: | |
| CARGO_INCREMENTAL: 0 # bookkeeping for incremental builds has overhead, not useful in CI. | |
| WORKFLOW_URL: https://github.com/maidsafe/stableset_net/actions/runs | |
| jobs: | |
| e2e: | |
| name: E2E tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| continue-on-error: true | |
| - name: Build node and client | |
| run: cargo build --release --bin safenode --bin safe --bin faucet | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| interval: 2000 | |
| node-path: target/release/safenode | |
| faucet-path: target/release/faucet | |
| platform: ${{ matrix.os }} | |
| - name: Check contact peer | |
| shell: bash | |
| run: echo "Peer is $SAFE_PEERS" | |
| - name: Create and fund a wallet to pay for files storage | |
| run: | | |
| cargo run --bin faucet --release -- --log-output-dest=data-dir send 1000000 $(cargo run --bin safe --release -- --log-output-dest=data-dir wallet address | tail -n 1) | tail -n 1 > transfer_hex | |
| cargo run --bin safe --release -- --log-output-dest=data-dir wallet receive --file transfer_hex | |
| env: | |
| SN_LOG: "all" | |
| timeout-minutes: 2 | |
| - name: Start a client to pay for files storage | |
| run: cargo run --bin safe --release -- --log-output-dest=data-dir wallet pay "./resources" | |
| env: | |
| SN_LOG: "all" | |
| timeout-minutes: 2 | |
| - name: Start a client to carry out chunk actions | |
| run: cargo run --bin safe --release -- --log-output-dest=data-dir files upload "./resources" | |
| env: | |
| SN_LOG: "all" | |
| timeout-minutes: 2 | |
| - name: Start a client to create a register | |
| run: cargo run --bin safe --release -- --log-output-dest=data-dir register create baobao | |
| env: | |
| SN_LOG: "all" | |
| timeout-minutes: 2 | |
| - name: Start a client to get a register | |
| run: cargo run --bin safe --release -- --log-output-dest=data-dir register get -n baobao | |
| env: | |
| SN_LOG: "all" | |
| timeout-minutes: 2 | |
| - name: Start a client to edit a register | |
| run: cargo run --bin safe --release -- --log-output-dest=data-dir register edit -n baobao wood | |
| env: | |
| SN_LOG: "all" | |
| timeout-minutes: 2 | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_e2e | |
| platform: ${{ matrix.os }} | |
| - name: post notification to slack on failure | |
| if: ${{ failure() }} | |
| uses: bryannice/[email protected] | |
| env: | |
| SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} | |
| SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" | |
| SLACK_TITLE: "Nightly E2E Test Run Failed" | |
| full_unit: | |
| name: Full Unit Tests (including proptests) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| continue-on-error: true | |
| - name: Build unit tests before running | |
| run: cargo test --release --lib --bins --no-run | |
| timeout-minutes: 30 | |
| - name: Run testnet tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_testnet | |
| - name: Run network tests | |
| timeout-minutes: 25 | |
| run: cargo test --release -p sn_networking | |
| - name: Run protocol tests | |
| timeout-minutes: 25 | |
| run: cargo test --release -p sn_protocol | |
| - name: Run transfers tests | |
| timeout-minutes: 25 | |
| run: cargo test --release --package sn_transfers | |
| - name: Run register tests | |
| shell: bash | |
| timeout-minutes: 50 | |
| env: | |
| PROPTEST_CASES: 512 | |
| run: cargo test --release -p sn_registers | |
| - name: post notification to slack on failure | |
| if: ${{ failure() }} | |
| uses: bryannice/[email protected] | |
| env: | |
| SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} | |
| SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" | |
| SLACK_TITLE: "Nightly Unit Test Run Failed" | |
| gossipsub: | |
| if: "!startsWith(github.event.head_commit.message, 'chore(release):')" | |
| name: Gossipsub E2E tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build node | |
| run: cargo build --release --bin safenode | |
| timeout-minutes: 30 | |
| - name: Build gossipsub testing executable | |
| run: cargo test --release -p sn_node --features=local-discovery --test msgs_over_gossipsub --no-run | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| interval: 2000 | |
| node-path: target/release/safenode | |
| faucet-path: target/release/faucet | |
| platform: ${{ matrix.os }} | |
| - name: Gossipsub - nodes to subscribe to topics, and publish messages | |
| run: cargo test --release -p sn_node --features local-discovery msgs_over_gossipsub -- --nocapture | |
| timeout-minutes: 20 | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_e2e | |
| platform: ${{ matrix.os }} | |
| spend_test: | |
| name: spend tests against network | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| continue-on-error: true | |
| - name: Build safe bins | |
| run: cargo build --release --features local-discovery --bin safenode --bin faucet | |
| timeout-minutes: 30 | |
| - name: Build testing executable | |
| run: cargo test --release -p sn_node --features=local-discovery --test sequential_transfers --test storage_payments --test nodes_rewards --no-run | |
| timeout-minutes: 30 | |
| env: | |
| CARGO_TARGET_DIR: "./transfer-target" | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| interval: 2000 | |
| node-path: target/release/safenode | |
| faucet-path: target/release/faucet | |
| platform: ${{ matrix.os }} | |
| # This should be first to avoid slow reward acceptance etc | |
| - name: execute the nodes rewards tests | |
| run: cargo test --release -p sn_node --features="local-discovery" --test nodes_rewards -- --nocapture --test-threads=1 | |
| env: | |
| CARGO_TARGET_DIR: "./transfer-target" | |
| SN_LOG: "all" | |
| timeout-minutes: 25 | |
| - name: execute the spend test | |
| run: cargo test --release -p sn_node --features="local-discovery" --test sequential_transfers -- --nocapture | |
| env: | |
| CARGO_TARGET_DIR: "./transfer-target" | |
| SN_LOG: "all" | |
| timeout-minutes: 10 | |
| - name: execute the storage payment tests | |
| run: cargo test --release -p sn_node --features="local-discovery" --test storage_payments -- --nocapture --test-threads=1 | |
| env: | |
| CARGO_TARGET_DIR: "./transfer-target" | |
| SN_LOG: "all" | |
| timeout-minutes: 10 | |
| - name: Small wait to allow reward receipt | |
| run: sleep 30 | |
| timeout-minutes: 1 | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_spend | |
| platform: ${{ matrix.os }} | |
| - name: post notification to slack on failure | |
| if: ${{ failure() }} | |
| uses: bryannice/[email protected] | |
| env: | |
| SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} | |
| SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" | |
| SLACK_TITLE: "Nightly Spend Test Run Failed" | |
| churn: | |
| name: Network churning tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| node_data_path: /home/runner/.local/share/safe/node | |
| - os: windows-latest | |
| node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node | |
| - os: macos-latest | |
| node_data_path: /Users/runner/Library/Application Support/safe/node | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| continue-on-error: true | |
| - name: Build safe bins | |
| run: cargo build --release --features local-discovery --bin safenode --bin faucet | |
| timeout-minutes: 30 | |
| - name: Build churn tests | |
| run: cargo test --release -p sn_node --features=local-discovery --test data_with_churn --no-run | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| interval: 2000 | |
| node-path: target/release/safenode | |
| faucet-path: target/release/faucet | |
| platform: ${{ matrix.os }} | |
| - name: Chunks data integrity during nodes churn (during 10min) (in theory) | |
| run: cargo test --release -p sn_node --features="local-discovery" --test data_with_churn -- --nocapture | |
| env: | |
| TEST_DURATION_MINS: 60 | |
| TEST_CHURN_CYCLES: 6 | |
| SN_LOG: "all" | |
| timeout-minutes: 90 | |
| - name: Verify restart of nodes using rg | |
| shell: bash | |
| timeout-minutes: 1 | |
| # get the counts, then the specific line, and then the digit count only | |
| # then check we have an expected level of restarts | |
| # TODO: make this use an env var, or relate to testnet size | |
| run : | | |
| restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "Restart $restart_count nodes" | |
| peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "PeerRemovedFromRoutingTable $peer_removed times" | |
| if [ $peer_removed -lt $restart_count ]; then | |
| echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count" | |
| exit 1 | |
| fi | |
| node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
| echo "Node dir count is $node_count" | |
| # TODO: reenable this once the testnet dir creation is tidied up to avoid a large count here | |
| # if [ $restart_count -lt $node_count ]; then | |
| # echo "Restart count of: $restart_count is less than the node count of: $node_count" | |
| # exit 1 | |
| # fi | |
| - name: Verify data replication using rg | |
| shell: bash | |
| timeout-minutes: 1 | |
| # get the counts, then the specific line, and then the digit count only | |
| # then check we have an expected level of replication | |
| # TODO: make this use an env var, or relate to testnet size | |
| run : | | |
| fetching_attempt_count=$(rg "FetchingKeysForReplication" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "Carried out $fetching_attempt_count fetching attempts" | |
| node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
| if [ $fetching_attempt_count -lt $node_count ]; then | |
| echo "Replication fetching attempts of: $fetching_attempt_count is less than the node count of: $node_count" | |
| exit 1 | |
| fi | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_churn | |
| platform: ${{ matrix.os }} | |
| - name: post notification to slack on failure | |
| if: ${{ failure() }} | |
| uses: bryannice/[email protected] | |
| env: | |
| SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} | |
| SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" | |
| SLACK_TITLE: "Nightly Churn Test Run Failed" | |
| verify_data_location: | |
| name: Verify data location | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| include: | |
| - os: ubuntu-latest | |
| node_data_path: /home/runner/.local/share/safe/node | |
| - os: windows-latest | |
| node_data_path: C:\\Users\\runneradmin\\AppData\\Roaming\\safe\\node | |
| - os: macos-latest | |
| node_data_path: /Users/runner/Library/Application Support/safe/node | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| - uses: Swatinem/rust-cache@v2 | |
| continue-on-error: true | |
| - name: Build safe bins | |
| run: cargo build --release --features local-discovery --bin safenode --bin faucet | |
| timeout-minutes: 30 | |
| - name: Build data location test | |
| run: cargo test --release -p sn_node --features=local-discovery --test verify_data_location --no-run | |
| timeout-minutes: 30 | |
| - name: Start a local network | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: start | |
| interval: 2000 | |
| node-path: target/release/safenode | |
| faucet-path: target/release/faucet | |
| platform: ${{ matrix.os }} | |
| - name: Verify the location of the data on the network (approx 12 * 5 mins) | |
| run: cargo test --release -p sn_node --features="local-discovery" --test verify_data_location -- --nocapture | |
| env: | |
| CHURN_COUNT: 12 | |
| SN_LOG: "all" | |
| timeout-minutes: 90 | |
| - name: Verify restart of nodes using rg | |
| shell: bash | |
| timeout-minutes: 1 | |
| # get the counts, then the specific line, and then the digit count only | |
| # then check we have an expected level of restarts | |
| # TODO: make this use an env var, or relate to testnet size | |
| run : | | |
| restart_count=$(rg "Node is restarting in" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "Restart $restart_count nodes" | |
| peer_removed=$(rg "PeerRemovedFromRoutingTable" "${{ matrix.node_data_path }}" -c --stats | \ | |
| rg "(\d+) matches" | rg "\d+" -o) | |
| echo "PeerRemovedFromRoutingTable $peer_removed times" | |
| if [ $peer_removed -lt $restart_count ]; then | |
| echo "PeerRemovedFromRoutingTable times of: $peer_removed is less than the restart count of: $restart_count" | |
| exit 1 | |
| fi | |
| node_count=$(ls "${{ matrix.node_data_path }}" | wc -l) | |
| echo "Node dir count is $node_count" | |
| - name: Stop the local network and upload logs | |
| if: always() | |
| uses: maidsafe/sn-local-testnet-action@main | |
| with: | |
| action: stop | |
| log_file_prefix: safe_test_logs_data_location | |
| platform: ${{ matrix.os }} | |
| - name: post notification to slack on failure | |
| if: ${{ failure() }} | |
| uses: bryannice/[email protected] | |
| env: | |
| SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} | |
| SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" | |
| SLACK_TITLE: "Nightly Data Location Test Run Failed" |