From ce7adf7aa9ad0855204486a58019cb5de1bdf09e Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 21 Aug 2023 10:33:27 +0200 Subject: [PATCH 1/6] ci: introduce test-implementations-e2e --- .../workflows/test-implementations-e2e.yml | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 .github/workflows/test-implementations-e2e.yml diff --git a/.github/workflows/test-implementations-e2e.yml b/.github/workflows/test-implementations-e2e.yml new file mode 100644 index 000000000..aa32b8fe4 --- /dev/null +++ b/.github/workflows/test-implementations-e2e.yml @@ -0,0 +1,81 @@ +name: Generate Implementations Dashboard + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: true + +jobs: + pull-outputs: + runs-on: "ubuntu-latest" + strategy: + matrix: + target: ["singulargarden/kubo", "singulargarden/bifrost-gateway2"] + fail-fast: false + defaults: + run: + shell: bash + steps: + - name: get default branch + id: get-default-branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OWNER_AND_REPO: ${{ matrix.target }} + run: | + DEFAULT_BRANCH=$(gh api repos/${OWNER_AND_REPO} --jq '.default_branch') + echo "default-branch=${DEFAULT_BRANCH}" >> $GITHUB_OUTPUT + - name: Download json output + id: download-artifact + uses: dawidd6/action-download-artifact@v2 # TODO: pin + with: + workflow: gateway-conformance.yml + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: ${{ steps.get-default-branch.outputs.default-branch }} + name: gateway-conformance.json + repo: ${{ matrix.target }} + if_no_artifact_found: fail + - name: Upload JSON output + if: (failure() || success()) + uses: actions/upload-artifact@v3 + with: + name: conformance-${{ matrix.target }}.json + path: ./output.json + aggregate: + runs-on: "ubuntu-latest" + needs: [pull-outputs] + # the tests might have failed + if: always() + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v3 + with: + path: "gateway-conformance" + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + - name: Aggregate results + working-directory: ./artifacts + run: | + mkdir ./aggregates + + # download-artifact downloads artifacts in a directory named after the artifact + # details: https://github.com/actions/download-artifact#download-all-artifacts + for folder in ./conformance-*.json; do + file="${folder}/output.json" + new_file="aggregates/${folder#conformance-}" + jq -ns 'inputs' "$file" | node ../gateway-conformance/aggregate.js 1 > "${new_file}" + done + + node ../gateway-conformance/aggregate-into-table.js ./aggregates/*.json > ./table.md + - name: Set summary + if: (failure() || success()) + run: cat ./artifacts/table.md >> $GITHUB_STEP_SUMMARY From 4046079f320b736c6bdbde24975112ad6ee38a00 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 21 Aug 2023 10:43:34 +0200 Subject: [PATCH 2/6] ci: extract implementation name --- .github/workflows/test-implementations-e2e.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-implementations-e2e.yml b/.github/workflows/test-implementations-e2e.yml index aa32b8fe4..602707166 100644 --- a/.github/workflows/test-implementations-e2e.yml +++ b/.github/workflows/test-implementations-e2e.yml @@ -22,21 +22,26 @@ jobs: run: shell: bash steps: - - name: get default branch - id: get-default-branch + - name: get repo details + id: get-details env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER_AND_REPO: ${{ matrix.target }} run: | - DEFAULT_BRANCH=$(gh api repos/${OWNER_AND_REPO} --jq '.default_branch') + DETAILS=$(gh api repos/${OWNER_AND_REPO}) + + DEFAULT_BRANCH=$(echo $DETAILS | jq -r '.default_branch') echo "default-branch=${DEFAULT_BRANCH}" >> $GITHUB_OUTPUT + + NAME=$(echo $DETAILS | jq -r '.name') + echo "name=${NAME}" >> $GITHUB_OUTPUT - name: Download json output id: download-artifact uses: dawidd6/action-download-artifact@v2 # TODO: pin with: workflow: gateway-conformance.yml github_token: ${{ secrets.GITHUB_TOKEN }} - branch: ${{ steps.get-default-branch.outputs.default-branch }} + branch: ${{ steps.get-details.outputs.default-branch }} name: gateway-conformance.json repo: ${{ matrix.target }} if_no_artifact_found: fail @@ -44,7 +49,7 @@ jobs: if: (failure() || success()) uses: actions/upload-artifact@v3 with: - name: conformance-${{ matrix.target }}.json + name: conformance-${{ steps.get-details.outputs.name }}.json path: ./output.json aggregate: runs-on: "ubuntu-latest" From ea5a4638f5f263ff08ffd07888e9a334f604278a Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 21 Aug 2023 11:05:00 +0200 Subject: [PATCH 3/6] ci: allow failed jobs inputs --- .github/workflows/test-implementations-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-implementations-e2e.yml b/.github/workflows/test-implementations-e2e.yml index 602707166..7b858d686 100644 --- a/.github/workflows/test-implementations-e2e.yml +++ b/.github/workflows/test-implementations-e2e.yml @@ -40,6 +40,7 @@ jobs: uses: dawidd6/action-download-artifact@v2 # TODO: pin with: workflow: gateway-conformance.yml + workflow_conclusion: "completed" # TODO: ideally we could request success|failure (https://github.com/dawidd6/action-download-artifact#usage) github_token: ${{ secrets.GITHUB_TOKEN }} branch: ${{ steps.get-details.outputs.default-branch }} name: gateway-conformance.json From d41d664fe0517ee60f8934a9bda5b4d42c3d0abc Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 21 Aug 2023 11:05:54 +0200 Subject: [PATCH 4/6] ci: dashboard every one hour --- .github/workflows/test-implementations-e2e.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-implementations-e2e.yml b/.github/workflows/test-implementations-e2e.yml index 7b858d686..ea614fb2a 100644 --- a/.github/workflows/test-implementations-e2e.yml +++ b/.github/workflows/test-implementations-e2e.yml @@ -6,6 +6,8 @@ on: branches: - main pull_request: + schedule: + - cron: "* */1 * * *" # every one hour concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} From 98b1606dde3099c1bd1dd29b4943e3622c8bb09d Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 21 Aug 2023 11:21:35 +0200 Subject: [PATCH 5/6] ci: fix implementation names --- .github/workflows/test-implementations-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-implementations-e2e.yml b/.github/workflows/test-implementations-e2e.yml index ea614fb2a..a94b5c22d 100644 --- a/.github/workflows/test-implementations-e2e.yml +++ b/.github/workflows/test-implementations-e2e.yml @@ -79,7 +79,7 @@ jobs: # details: https://github.com/actions/download-artifact#download-all-artifacts for folder in ./conformance-*.json; do file="${folder}/output.json" - new_file="aggregates/${folder#conformance-}" + new_file="aggregates/${folder#.\/conformance-}" # drop the ./conformance- prefix jq -ns 'inputs' "$file" | node ../gateway-conformance/aggregate.js 1 > "${new_file}" done From 2336e607a8ecf77a00ae0283ccf67a81cac22d74 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 21 Aug 2023 11:22:28 +0200 Subject: [PATCH 6/6] ci: current list of gateways --- .github/workflows/test-implementations-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-implementations-e2e.yml b/.github/workflows/test-implementations-e2e.yml index a94b5c22d..bb6eb1f1f 100644 --- a/.github/workflows/test-implementations-e2e.yml +++ b/.github/workflows/test-implementations-e2e.yml @@ -18,7 +18,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - target: ["singulargarden/kubo", "singulargarden/bifrost-gateway2"] + target: ["ipfs/kubo", "ipfs/boxo", "ipfs/bifrost-gateway"] fail-fast: false defaults: run: