Test: Make chopsticks tests more resilient

.github/workflows/xcm-assethub-tests.yml

@@ -0,0 +1,80 @@
+---
+name: AssetHub XCM tests
+on:
+  schedule:
+    - cron: '38 */2 * * *' # Run every 2 hours
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  chopsticks_kusama_assethub_test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Launch chopsticks
+        timeout-minutes: 3
+        run: |
+          npx --yes @acala-network/[email protected] \
+            xcm \
+            -r scripts/configs/kusama.yml \
+            -p scripts/configs/kintsugi.yml \
+            -p scripts/configs/statemine.yml \
+            &> log.txt &
+          echo "Waiting for log to show chopsticks is ready..."
+          tail -f log.txt | grep -q "Connected parachains"
+          echo "... detected chopsticks is ready."
+
+      - name: Run Kusama AssetHub tests
+        run: |
+          yarn install --frozen-lockfile
+          npx ts-node scripts/kusama-chopsticks-assethub-test.ts
+      - name: Show error log
+        if: failure()
+        run: |
+          tail -n 100 log.txt
+      - name: Report status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: failure()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}
+
+  chopsticks_polkadot_assethub_test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Launch chopsticks
+        timeout-minutes: 3
+        run: |
+          npx --yes @acala-network/[email protected] \
+            xcm \
+            -r scripts/configs/polkadot.yml \
+            -p scripts/configs/interlay.yml \
+            -p scripts/configs/statemint.yml \
+            &> log.txt &
+          echo "Waiting for log to show chopsticks is ready..."
+          tail -f log.txt | grep -q "Connected parachains"
+          echo "... detected chopsticks is ready."
+
+      - name: Run Polkadot AssetHub tests
+        run: |
+          yarn install --frozen-lockfile
+          npx ts-node scripts/polkadot-chopsticks-assethub-test.ts
+      - name: Show error log
+        if: failure()
+        run: |
+          tail -n 100 log.txt
+      - name: Report status to Discord
+        uses: sarisia/actions-status-discord@v1
+        if: failure()
+        with:
+          webhook: ${{ secrets.DISCORD_WEBHOOK }}

.github/workflows/xcm-tests.yml

@@ -1,31 +1,30 @@
 ---
-name: Daily XCM tests
+name: XCM tests
 on:
   schedule:
-    - cron: '5 * * * *' # Run hourly
+    - cron: '12 */2 * * *' # Run every 2 hours
   push:
     branches:
       - master
   pull_request:
     branches:
       - master
 
-
 jobs:
   chopsticks_kintsugi_test:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 15
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Launch chopsticks
+        timeout-minutes: 3
         run: |
           npx --yes @acala-network/[email protected] \
             xcm \
             -r scripts/configs/kusama.yml \
             -p scripts/configs/kintsugi.yml \
-            -p scripts/configs/statemine.yml \
             -p scripts/configs/karura.yml \
             -p scripts/configs/parallel-heiko.yml \
             -p scripts/configs/bifrost.yml \
@@ -57,22 +56,21 @@ jobs:
 
   chopsticks_interlay_test:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 15
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Launch chopsticks
+        timeout-minutes: 3
         run: |
           npx --yes @acala-network/[email protected] \
             xcm \
             -r scripts/configs/polkadot.yml \
             -p scripts/configs/interlay.yml \
-            -p scripts/configs/statemint.yml \
             -p scripts/configs/hydradx.yml \
             -p scripts/configs/acala.yml \
             -p scripts/configs/astar.yml \
-            -p scripts/configs/parallel.yml \
             -p scripts/configs/bifrost-polkadot.yml \
             &> log.txt &
           echo "Waiting for log to show chopsticks is ready..."

package.json

@@ -33,7 +33,9 @@
     "test:ci": "jest --forceExit",
     "lint": "polkadot-dev-run-lint",
     "chopsticks-test:kintsugi": "ts-node scripts/kintsugi-chopsticks-test",
-    "chopsticks-test:interlay": "ts-node scripts/interlay-chopsticks-test"
+    "chopsticks-test:interlay": "ts-node scripts/interlay-chopsticks-test",
+    "chopsticks-test:kusama-assethub": "ts-node scripts/kusama-chopsticks-assethub-test",
+    "chopsticks-test:polkadot-assethub": "ts-node scripts/polkadot-chopsticks-assethub-test"
   },
   "peerDependencies": {
     "@polkadot/api": "^10"

scripts/chopsticks-test.ts

@@ -191,11 +191,13 @@ async function retryCheckTransfer(
  * Can skip specific test cases provided by the skipCases filter, and will also skip routes if no matching adapter has been provided.
  * 
  * @param adapterEndpoints Records containing ChainName as key, an instantiated adapter and a list of ws(s) links as endpoints for each.
+ * @param includeAssetHubSpecialCases Boolean value whether to add relay chain to/from assethub test cases (true) or not (false).
  * @param skipCases An array of xcm test cases to skip.
  */
 export async function runTestCasesAndExit(
     // record key is chainname
     adapterEndpoints: Record<ChainName, { adapter: BaseCrossChainAdapter, endpoints: Array<string> }>,
+    includeAssetHubSpecialCases: boolean = false,
     // skip cases: array of to, from and/or token to skip tests for
     skipCases: Partial<RouterTestCase>[] = []
 ): Promise<void> {
@@ -231,21 +233,23 @@ export async function runTestCasesAndExit(
     }));
 
     // add in special cases: polkadot/kusama <=> asset hub
-    const relayId = chains.includes("polkadot") ? "polkadot" : "kusama";
-    const assetHubId = relayId === "polkadot" ? "statemint" : "statemine";
-    const token = relayId === "polkadot" ? "DOT" : "KSM";
-    testCases.push(
-        {
-            to: assetHubId as ChainName,
-            from: relayId as ChainName,
-            token
-        },
-        {
-            to: relayId as ChainName,
-            from: assetHubId as ChainName,
-            token
-        },
-    );
+    if (includeAssetHubSpecialCases) {
+        const relayId = chains.includes("polkadot") ? "polkadot" : "kusama";
+        const assetHubId = relayId === "polkadot" ? "statemint" : "statemine";
+        const token = relayId === "polkadot" ? "DOT" : "KSM";
+        testCases.push(
+            {
+                to: assetHubId as ChainName,
+                from: relayId as ChainName,
+                token
+            },
+            {
+                to: relayId as ChainName,
+                from: assetHubId as ChainName,
+                token
+            },
+        );
+    }
 
     const isSkipCase = (testCase: {to: ChainName, from: ChainName, token: string}): boolean => {
         return skipCases.some((skipCase) =>

scripts/interlay-chopsticks-test.ts

@@ -3,14 +3,13 @@
 /* tslint:disable:no-unused-variable */
 import { PolkadotAdapter } from "../src/adapters/polkadot";
 import { InterlayAdapter } from "../src/adapters/interlay";
-import { StatemintAdapter } from "../src/adapters/statemint";
 import { HydraAdapter } from "../src/adapters/hydradx";
 import { AcalaAdapter } from "../src/adapters/acala";
 import { AstarAdapter } from "../src/adapters/astar";
-import { ParallelAdapter } from "../src/adapters/parallel";
+// import { ParallelAdapter } from "../src/adapters/parallel";
 import { BifrostPolkadotAdapter } from "../src/adapters/bifrost";
 import { BaseCrossChainAdapter } from "../src/base-chain-adapter";
-import { RouterTestCase, runTestCasesAndExit } from "./chopsticks-test";
+import { runTestCasesAndExit } from "./chopsticks-test";
 
 main().catch((err) => {
     console.log("Error thrown by script:");
@@ -25,17 +24,13 @@ async function main(): Promise<void> {
         // reminder: parachains get ports in oder of arguments, starting with 8000 and incremented for each following one; 
         //           relaychain gets its port last after all parachains.
         interlay: { adapter: new InterlayAdapter(), endpoints: ['ws://127.0.0.1:8000'] },
-        statemint: { adapter: new StatemintAdapter(), endpoints: ['ws://127.0.0.1:8001'] },
-        hydra: { adapter: new HydraAdapter(), endpoints: ['ws://127.0.0.1:8002'] },
-        acala: { adapter: new AcalaAdapter(), endpoints: ['ws://127.0.0.1:8003'] },
-        astar: { adapter: new AstarAdapter(), endpoints: ['ws://127.0.0.1:8004'] },
-        parallel: { adapter: new ParallelAdapter(), endpoints: ['ws://127.0.0.1:8005'] },
-        bifrost_polkadot: { adapter: new BifrostPolkadotAdapter(), endpoints: ['ws://127.0.0.1:8006']},
-        polkadot: { adapter: new PolkadotAdapter(), endpoints: ['ws://127.0.0.1:8007'] },
+        hydra: { adapter: new HydraAdapter(), endpoints: ['ws://127.0.0.1:8001'] },
+        acala: { adapter: new AcalaAdapter(), endpoints: ['ws://127.0.0.1:8002'] },
+        astar: { adapter: new AstarAdapter(), endpoints: ['ws://127.0.0.1:8003'] },
+        // parallel: { adapter: new ParallelAdapter(), endpoints: ['ws://127.0.0.1:8004'] },
+        bifrost_polkadot: { adapter: new BifrostPolkadotAdapter(), endpoints: ['ws://127.0.0.1:8004']},
+        polkadot: { adapter: new PolkadotAdapter(), endpoints: ['ws://127.0.0.1:8005'] },
     };
 
-    const filterCases: Partial<RouterTestCase>[] = [
-    ];
-
-    await runTestCasesAndExit(adaptersEndpoints, filterCases);
+    await runTestCasesAndExit(adaptersEndpoints);
 }

scripts/kintsugi-chopsticks-test.ts

@@ -6,7 +6,6 @@ import { BifrostKusamaAdapter } from "../src/adapters/bifrost";
 import { KintsugiAdapter } from "../src/adapters/interlay";
 import { HeikoAdapter } from "../src/adapters/parallel";
 import { KusamaAdapter } from "../src/adapters/polkadot";
-import { StatemineAdapter } from "../src/adapters/statemint";
 import { BaseCrossChainAdapter } from "../src/base-chain-adapter";
 import { runTestCasesAndExit } from "./chopsticks-test";
 
@@ -23,11 +22,10 @@ async function main(): Promise<void> {
         // reminder: parachains get ports in oder of arguments, starting with 8000 and incremented for each following one; 
         //           relaychain gets its port last after all parachains.
         kintsugi:   { adapter: new KintsugiAdapter(),   endpoints: ['ws://127.0.0.1:8000'] },
-        statemine:  { adapter: new StatemineAdapter(),  endpoints: ['ws://127.0.0.1:8001'] },
-        karura:     { adapter: new KaruraAdapter(),     endpoints: ['ws://127.0.0.1:8002'] },
-        heiko:      { adapter: new HeikoAdapter(),      endpoints: ['ws://127.0.0.1:8003'] },
-        bifrost:    { adapter: new BifrostKusamaAdapter(),    endpoints: ['ws://127.0.0.1:8004'] },
-        kusama:     { adapter: new KusamaAdapter(),     endpoints: ['ws://127.0.0.1:8005'] },
+        karura:     { adapter: new KaruraAdapter(),     endpoints: ['ws://127.0.0.1:8001'] },
+        heiko:      { adapter: new HeikoAdapter(),      endpoints: ['ws://127.0.0.1:8002'] },
+        bifrost:    { adapter: new BifrostKusamaAdapter(),    endpoints: ['ws://127.0.0.1:8003'] },
+        kusama:     { adapter: new KusamaAdapter(),     endpoints: ['ws://127.0.0.1:8004'] },
     };
 
     await runTestCasesAndExit(adaptersEndpoints);

scripts/kusama-chopsticks-assethub-test.ts

@@ -0,0 +1,40 @@
+/* eslint @typescript-eslint/no-var-requires: "off" */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* tslint:disable:no-unused-variable */
+import { KintsugiAdapter } from "../src/adapters/interlay";
+import { StatemineAdapter } from "../src/adapters/statemint";
+import { KusamaAdapter } from "../src/adapters/polkadot";
+import { BaseCrossChainAdapter } from "../src/base-chain-adapter";
+import { RouterTestCase, runTestCasesAndExit } from "./chopsticks-test";
+
+main().catch((err) => {
+    console.log("Error thrown by script:");
+    console.log(err);
+    process.exit(-1);
+});
+
+async function main(): Promise<void> {
+    const adaptersEndpoints : Record<string, { adapter: BaseCrossChainAdapter, endpoints: Array<string> }> = {
+        // make sure endpoints are aligned with the ports spun up by chopsticks config in
+        // .github/workflows/xcm-tests.yml
+        // reminder: parachains get ports in oder of arguments, starting with 8000 and incremented for each following one; 
+        //           relaychain gets its port last after all parachains.
+        kintsugi:   { adapter: new KintsugiAdapter(),   endpoints: ['ws://127.0.0.1:8000'] },
+        statemine: { adapter: new StatemineAdapter(), endpoints: ['ws://127.0.0.1:8001'] },
+        kusama:     { adapter: new KusamaAdapter(),     endpoints: ['ws://127.0.0.1:8002'] },
+    };
+
+    // already tested in kintsugi-chopsticks-test
+    const skipCases: Partial<RouterTestCase>[] = [
+        {
+            from: "kintsugi",
+            to: "kusama",
+        },
+        {
+            from: "kusama",
+            to: "kintsugi",
+        },
+    ];
+
+    await runTestCasesAndExit(adaptersEndpoints, true, skipCases);
+}

scripts/polkadot-chopsticks-assethub-test.ts

@@ -0,0 +1,40 @@
+/* eslint @typescript-eslint/no-var-requires: "off" */
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* tslint:disable:no-unused-variable */
+import { PolkadotAdapter } from "../src/adapters/polkadot";
+import { InterlayAdapter } from "../src/adapters/interlay";
+import { StatemintAdapter } from "../src/adapters/statemint";
+import { BaseCrossChainAdapter } from "../src/base-chain-adapter";
+import { RouterTestCase, runTestCasesAndExit } from "./chopsticks-test";
+
+main().catch((err) => {
+    console.log("Error thrown by script:");
+    console.log(err);
+    process.exit(-1);
+});
+
+async function main(): Promise<void> {
+    const adaptersEndpoints : Record<string, { adapter: BaseCrossChainAdapter, endpoints: Array<string> }> = {
+        // make sure endpoints are aligned with the ports spun up by chopsticks config in
+        // .github/workflows/xcm-tests.yml
+        // reminder: parachains get ports in oder of arguments, starting with 8000 and incremented for each following one; 
+        //           relaychain gets its port last after all parachains.
+        interlay: { adapter: new InterlayAdapter(), endpoints: ['ws://127.0.0.1:8000'] },
+        statemint: { adapter: new StatemintAdapter(), endpoints: ['ws://127.0.0.1:8001'] },
+        polkadot: { adapter: new PolkadotAdapter(), endpoints: ['ws://127.0.0.1:8002'] },
+    };
+
+    // already tested in interlay-chopsticks-test
+    const skipCases: Partial<RouterTestCase>[] = [
+        {
+            from: "interlay",
+            to: "polkadot",
+        },
+        {
+            from: "polkadot",
+            to: "interlay",
+        },
+    ];
+
+    await runTestCasesAndExit(adaptersEndpoints, true, skipCases);
+}