feat: test implementations with reusable jsons

.github/workflows/test-implementations-e2e.yml

@@ -0,0 +1,89 @@
+name: Generate Implementations Dashboard
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+  schedule:
+    - cron: "* */1 * * *" # every one hour
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pull-outputs:
+    runs-on: "ubuntu-latest"
+    strategy:
+      matrix:
+        target: ["ipfs/kubo", "ipfs/boxo", "ipfs/bifrost-gateway"]
+      fail-fast: false
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: get repo details
+        id: get-details
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OWNER_AND_REPO: ${{ matrix.target }}
+        run: |
+          DETAILS=$(gh api repos/${OWNER_AND_REPO})
+
+          DEFAULT_BRANCH=$(echo $DETAILS | jq -r '.default_branch')
+          echo "default-branch=${DEFAULT_BRANCH}" >> $GITHUB_OUTPUT
+
+          NAME=$(echo $DETAILS | jq -r '.name')
+          echo "name=${NAME}" >> $GITHUB_OUTPUT
+      - name: Download json output
+        id: download-artifact
+        uses: dawidd6/action-download-artifact@v2 # TODO: pin
+        with:
+          workflow: gateway-conformance.yml
+          workflow_conclusion: "completed" # TODO: ideally we could request success|failure (https://github.com/dawidd6/action-download-artifact#usage)
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ steps.get-details.outputs.default-branch }}
+          name: gateway-conformance.json
+          repo: ${{ matrix.target }}
+          if_no_artifact_found: fail
+      - name: Upload JSON output
+        if: (failure() || success())
+        uses: actions/upload-artifact@v3
+        with:
+          name: conformance-${{ steps.get-details.outputs.name }}.json
+          path: ./output.json
+  aggregate:
+    runs-on: "ubuntu-latest"
+    needs: [pull-outputs]
+    # the tests might have failed
+    if: always()
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: "gateway-conformance"
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - name: Aggregate results
+        working-directory: ./artifacts
+        run: |
+          mkdir ./aggregates
+
+          # download-artifact downloads artifacts in a directory named after the artifact
+          # details: https://github.com/actions/download-artifact#download-all-artifacts
+          for folder in ./conformance-*.json; do
+            file="${folder}/output.json"
+            new_file="aggregates/${folder#.\/conformance-}" # drop the ./conformance- prefix
+            jq -ns 'inputs' "$file" | node ../gateway-conformance/aggregate.js 1 > "${new_file}"
+          done
+
+          node ../gateway-conformance/aggregate-into-table.js ./aggregates/*.json > ./table.md
+      - name: Set summary
+        if: (failure() || success())
+        run: cat ./artifacts/table.md >> $GITHUB_STEP_SUMMARY