[gha] Refactor Forge Stable to run individual tests

.github/workflows/forge-stable.yaml

@@ -24,6 +24,30 @@ on:
         required: false
         type: string
         description: The git SHA1 to checkout. This affects the Forge test runner that is used. If not specified, the latest main will be used
+      JOB_NAME:
+        required: false
+        type: choice
+        description: The job to run. If all, all jobs will be run
+        default: all
+        options:
+          - all
+          - forge-framework-upgrade-test
+          - forge-realistic-env-load-sweep
+          - forge-realistic-env-workload-sweep
+          - forge-realistic-env-graceful-overload
+          - forge-realistic-env-graceful-workload-sweep
+          - forge-realistic-env-fairness-workload-sweep
+          - forge-realistic-network-tuned-for-throughput
+          - forge-consensus-stress-test
+          - forge-workload-mix-test
+          - forge-single-vfn-perf
+          - forge-fullnode-reboot-stress-test
+          - forge-compat
+          - forge-changing-working-quorum-test
+          - forge-changing-working-quorum-test-high-load
+          - forge-pfn-const-tps-realistic-env
+          - forge-realistic-env-max-load-long
+
   # NOTE: to support testing different branches on different schedules, you need to specify the cron schedule in the 'determine-test-branch' step as well below
   # Reference: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
   schedule:
@@ -122,212 +146,73 @@ jobs:
           echo "IMAGE_TAG: [${IMAGE_TAG}](https://github.com/${{ github.repository }}/commit/${IMAGE_TAG})" >> $GITHUB_STEP_SUMMARY
           echo "To cancel this job, do `pnpm infra ci cancel-workflow ${{ github.run_id }}` from internal-ops" >> $GITHUB_STEP_SUMMARY
 
-  ### Real-world-network tests.
-  # Run forge framework upgradability test. This is a PR required job.
-  run-forge-framework-upgrade-test:
-    if: ${{ github.event_name != 'pull_request' }}
-    needs:
-      - determine-test-metadata
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG_FOR_COMPAT_TEST }}
-      FORGE_NAMESPACE: forge-framework-upgrade-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
-      FORGE_TEST_SUITE: framework_upgrade
-      POST_TO_SLACK: true
-
-  run-forge-realistic-env-load-sweep:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-framework-upgrade-test] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-realistic-env-load-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes (6 tests, each for 300 seconds)
-      FORGE_TEST_SUITE: realistic_env_load_sweep
-      POST_TO_SLACK: true
-
-  run-forge-realistic-env-workload-sweep:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-realistic-env-load-sweep] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-realistic-env-workload-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 2000 # Run for 33 minutes (5 tests, each for 400 seconds)
-      FORGE_TEST_SUITE: realistic_env_workload_sweep
-      POST_TO_SLACK: true
-
-  run-forge-realistic-env-graceful-overload:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-realistic-env-workload-sweep] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-realistic-env-graceful-overload-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
-      FORGE_TEST_SUITE: realistic_env_graceful_overload
-      POST_TO_SLACK: true
-
-  run-forge-realistic-env-graceful-workload-sweep:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-realistic-env-graceful-overload] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-realistic-env-graceful-workload-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 2100 # Run for 5 minutes per test, 7 tests.
-      FORGE_TEST_SUITE: realistic_env_graceful_workload_sweep
-      POST_TO_SLACK: true
-
-  run-forge-realistic-env-fairness-workload-sweep:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-realistic-env-graceful-workload-sweep] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-realistic-env-fairness-workload-sweep-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 900 # Run for 5 minutes per test, 3 tests.
-      FORGE_TEST_SUITE: realistic_env_fairness_workload_sweep
-      POST_TO_SLACK: true
-
-  run-forge-realistic-network-tuned-for-throughput:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [ determine-test-metadata, run-forge-realistic-env-fairness-workload-sweep ] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-realistic-network-tuned-for-throughput-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
-      FORGE_TEST_SUITE: realistic_network_tuned_for_throughput
-      FORGE_ENABLE_PERFORMANCE: true
-      POST_TO_SLACK: true
-
-  ### Forge Correctness/Componenet/Stress tests
-
-  run-forge-consensus-stress-test:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-realistic-network-tuned-for-throughput] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-consensus-stress-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 2400 # Run for 40 minutes
-      FORGE_TEST_SUITE: consensus_stress_test
-      POST_TO_SLACK: true
-
-  run-forge-workload-mix-test:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-consensus-stress-test] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-workload-mix-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
-      FORGE_TEST_SUITE: workload_mix
-      POST_TO_SLACK: true
-
-  run-forge-single-vfn-perf:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-workload-mix-test] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-continuous-e2e-single-vfn-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 480 # Run for 8 minutes
-      FORGE_TEST_SUITE: single_vfn_perf
-      POST_TO_SLACK: true
-
-  run-forge-fullnode-reboot-stress-test:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-single-vfn-perf] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-fullnode-reboot-stress-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 1800 # Run for 30 minutes
-      FORGE_TEST_SUITE: fullnode_reboot_stress_test
-      POST_TO_SLACK: true
-
-  ### Compatibility Forge tests
-
-  run-forge-compat:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-fullnode-reboot-stress-test] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      FORGE_NAMESPACE: forge-compat-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 300 # Run for 5 minutes
-      # This will upgrade from testnet branch to the latest main
-      FORGE_TEST_SUITE: compat
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG_FOR_COMPAT_TEST }}
-      GIT_SHA: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }} # this is the git ref to checkout
-      POST_TO_SLACK: true
-
-  ### Changing working quorum Forge tests
-
-  run-forge-changing-working-quorum-test:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-compat] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-changing-working-quorum-test-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 1200 # Run for 20 minutes
-      FORGE_TEST_SUITE: changing_working_quorum_test
-      POST_TO_SLACK: true
-      FORGE_ENABLE_FAILPOINTS: true
-
-  run-forge-changing-working-quorum-test-high-load:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-changing-working-quorum-test] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-changing-working-quorum-test-high-load-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
-      FORGE_TEST_SUITE: changing_working_quorum_test_high_load
-      POST_TO_SLACK: true
-      FORGE_ENABLE_FAILPOINTS: true
-
-  # Measures PFN latencies with a constant TPS (with a realistic environment)
-  run-forge-pfn-const-tps-realistic-env:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-changing-working-quorum-test-high-load] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-pfn-const-tps-with-realistic-env-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 900 # Run for 15 minutes
-      FORGE_TEST_SUITE: pfn_const_tps_with_realistic_env
-      POST_TO_SLACK: true
-
-
-  # longest test for last, to get useful signal from short tests first
-
-  run-forge-realistic-env-max-load-long:
-    if: ${{ github.event_name != 'pull_request' && always() }}
-    needs: [determine-test-metadata, run-forge-pfn-const-tps-realistic-env] # Only run after the previous job completes
-    uses: aptos-labs/aptos-core/.github/workflows/workflow-run-forge.yaml@main
-    secrets: inherit
-    with:
-      IMAGE_TAG: ${{ needs.determine-test-metadata.outputs.IMAGE_TAG }}
-      FORGE_NAMESPACE: forge-realistic-env-max-load-long-${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
-      FORGE_RUNNER_DURATION_SECS: 7200 # Run for 2 hours
-      FORGE_TEST_SUITE: realistic_env_max_load_large
-      POST_TO_SLACK: true
+  run-forge-test:
+    needs: [determine-test-metadata]
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        tests:
+          - TEST_NAME: forge-framework-upgrade-test
+            FORGE_RUNNER_DURATION_SECS: 7200
+            FORGE_TEST_SUITE: framework_upgrade
+          - TEST_NAME: forge-realistic-env-load-sweep
+            FORGE_RUNNER_DURATION_SECS: 1800
+            FORGE_TEST_SUITE: realistic_env_load_sweep
+          - TEST_NAME: forge-realistic-env-workload-sweep
+            FORGE_RUNNER_DURATION_SECS: 2000
+            FORGE_TEST_SUITE: realistic_env_workload_sweep
+          - TEST_NAME: forge-realistic-env-graceful-overload
+            FORGE_RUNNER_DURATION_SECS: 1200
+            FORGE_TEST_SUITE: realistic_env_graceful_overload
+          - TEST_NAME: forge-realistic-env-graceful-workload-sweep
+            FORGE_RUNNER_DURATION_SECS: 2100
+            FORGE_TEST_SUITE: realistic_env_graceful_workload_sweep
+          - TEST_NAME: forge-realistic-env-fairness-workload-sweep
+            FORGE_RUNNER_DURATION_SECS: 900
+            FORGE_TEST_SUITE: realistic_env_fairness_workload_sweep
+          - TEST_NAME: forge-realistic-network-tuned-for-throughput
+            FORGE_RUNNER_DURATION_SECS: 900
+            FORGE_TEST_SUITE: realistic_network_tuned_for_throughput
+            FORGE_ENABLE_PERFORMANCE: true
+          - TEST_NAME: forge-consensus-stress-test
+            FORGE_RUNNER_DURATION_SECS: 2400
+            FORGE_TEST_SUITE: consensus_stress_test
+          - TEST_NAME: forge-workload-mix-test
+            FORGE_RUNNER_DURATION_SECS: 900
+            FORGE_TEST_SUITE: workload_mix
+          - TEST_NAME: forge-single-vfn-perf
+            FORGE_RUNNER_DURATION_SECS: 480
+            FORGE_TEST_SUITE: single_vfn_perf
+          - TEST_NAME: forge-fullnode-reboot-stress-test
+            FORGE_RUNNER_DURATION_SECS: 1800
+            FORGE_TEST_SUITE: fullnode_reboot_stress_test
+          - TEST_NAME: forge-compat
+            FORGE_RUNNER_DURATION_SECS: 300
+            FORGE_TEST_SUITE: compat
+          - TEST_NAME: forge-changing-working-quorum-test
+            FORGE_RUNNER_DURATION_SECS: 1200
+            FORGE_TEST_SUITE: changing_working_quorum_test
+            FORGE_ENABLE_FAILPOINTS: true
+          - TEST_NAME: forge-changing-working-quorum-test-high-load
+            FORGE_RUNNER_DURATION_SECS: 900
+            FORGE_TEST_SUITE: changing_working_quorum_test_high_load
+            FORGE_ENABLE_FAILPOINTS: true
+          - TEST_NAME: forge-pfn-const-tps-realistic-env
+            FORGE_RUNNER_DURATION_SECS: 900
+            FORGE_TEST_SUITE: pfn_const_tps_with_realistic_env
+          - TEST_NAME: forge-realistic-env-max-load-long
+            FORGE_RUNNER_DURATION_SECS: 7200
+            FORGE_TEST_SUITE: realistic_env_max_load_large
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run Forge test
+        if: ${{ github.event.inputs.JOB_NAME == 'all' || matrix.job == github.event.inputs.JOB_NAME }}
+        uses: aptos-labs/aptos-core/.github/actions/run-forge-test@main
+        with:
+          IMAGE_TAG: ${{ ((matrix.FORGE_TEST_SUITE == 'compat' && needs.determine-test-metadata.outputs.IMAGE_TAG_FOR_COMPAT_TEST) || needs.determine-test-metadata.outputs.IMAGE_TAG) }}
+          BRANCH_HASH: ${{ needs.determine-test-metadata.outputs.BRANCH_HASH }}
+          FORGE_RUNNER_DURATION_SECS: ${{ matrix.FORGE_RUNNER_DURATION_SECS }}
+          FORGE_TEST_SUITE: ${{ matrix.FORGE_TEST_SUITE }}
+          FORGE_ENABLE_PERFORMANCE: ${{ matrix.FORGE_ENABLE_PERFORMANCE || false }}
+          FORGE_ENABLE_FAILPOINTS: ${{ matrix.FORGE_ENABLE_FAILPOINTS || false }}