From 192f2543ba463323da9fcd3fd18310e35125142a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 19 Sep 2024 18:55:47 +0200 Subject: [PATCH] refactor,ci(ct): finishing touches for #10478 Re-enable and change everything necessary to reference the upstream IQSS context as of now. --- .github/workflows/container_app_push.yml | 10 ++- .github/workflows/container_base_push.yml | 67 ++++++++++----------- .github/workflows/container_maintenance.yml | 12 ++-- .github/workflows/maven_unit_test.yml | 4 ++ 4 files changed, 44 insertions(+), 49 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index e3dbcd4d2c6..86b45203bb2 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -35,13 +35,11 @@ jobs: packages: write pull-requests: write # Only run in upstream repo - avoid unnecessary runs in forks - # TODO: re-enable for final PR - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven with: pom-paths: | pom.xml @@ -51,6 +49,7 @@ jobs: # TODO: Add a filter step here, that avoids building the image if this is a PR and there are other files touched than declared above. # Use https://github.com/dorny/paths-filter to solve this. This will ensure we do not run this twice if this workflow # will be triggered by the other workflows already (base image or java changes) + # To become a part of #10618. - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) run: > @@ -117,8 +116,7 @@ jobs: ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven with: pom-paths: | pom.xml diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 24b1102d3a7..55eabc42c74 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -7,8 +7,6 @@ on: - 'v[6-9].**' branches: - 'develop' - # TODO: delete for final PR - - '10478-version-base-img' # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore paths: - 'modules/container-base/**' @@ -17,13 +15,13 @@ on: - 'modules/dataverse-parent/pom.xml' - '.github/workflows/container_base_push.yml' + # These TODOs are left for #10618 # TODO: we are missing a workflow_call option here, so we can trigger this flow from pr comments and maven tests (keep the secrets availability in mind!) # TODO: we are missing a pull_request option here (filter for stuff that would trigger the maven runs!) so we can trigger preview builds for them when coming from the main repo (keep the secrets availability in mind!) env: PLATFORMS: linux/amd64,linux/arm64 - # TODO: set back to develop for final PR - DEVELOPMENT_BRANCH: 10478-version-base-img + DEVELOPMENT_BRANCH: develop jobs: build: @@ -32,16 +30,14 @@ jobs: permissions: contents: read packages: read - # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} outputs: base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven with: pom-paths: modules/container-base/pom.xml @@ -55,34 +51,33 @@ jobs: # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - # TODO: re-enable for final PR - #- name: Setup SSH agent - # if: ${{ github.event_name != 'schedule' }} - # uses: webfactory/ssh-agent@v0.9.0 - # with: - # ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - #- name: Provide the known hosts key and the builder config - # if: ${{ github.event_name != 'schedule' }} - # run: | - # echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts - # mkdir -p modules/container-base/target/buildx-state/buildx/instances - # cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF - # { "Name": "maven", - # "Driver": "docker-container", - # "Dynamic": false, - # "Nodes": [{"Name": "maven0", - # "Endpoint": "unix:///var/run/docker.sock", - # "Platforms": [{"os": "linux", "architecture": "amd64"}], - # "DriverOpts": null, - # "Flags": ["--allow-insecure-entitlement=network.host"], - # "Files": null}, - # {"Name": "maven1", - # "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", - # "Platforms": [{"os": "linux", "architecture": "arm64"}], - # "DriverOpts": null, - # "Flags": ["--allow-insecure-entitlement=network.host"], - # "Files": null}]} - # EOF + - name: Setup SSH agent + if: ${{ github.event_name != 'schedule' }} + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} + - name: Provide the known hosts key and the builder config + if: ${{ github.event_name != 'schedule' }} + run: | + echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts + mkdir -p modules/container-base/target/buildx-state/buildx/instances + cat > modules/container-base/target/buildx-state/buildx/instances/maven << EOF + { "Name": "maven", + "Driver": "docker-container", + "Dynamic": false, + "Nodes": [{"Name": "maven0", + "Endpoint": "unix:///var/run/docker.sock", + "Platforms": [{"os": "linux", "architecture": "amd64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}, + {"Name": "maven1", + "Endpoint": "ssh://${{ secrets.BUILDER_ARM64_SSH_CONNECTION }}", + "Platforms": [{"os": "linux", "architecture": "arm64"}], + "DriverOpts": null, + "Flags": ["--allow-insecure-entitlement=network.host"], + "Files": null}]} + EOF # Determine the base image name we are going to use from here on - name: Determine base image name diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 2b1deaded5f..6a7eaa03710 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -26,18 +26,15 @@ jobs: permissions: contents: read packages: read - # TODO: re-enable for final PR # Only run in upstream repo - avoid unnecessary runs in forks - #if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} outputs: supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} steps: - name: Checkout and Setup Maven - # TODO: change to upstream location in final PR - # Necessary as the checked out release branch might not contain the action as files - uses: gdcc/wip-dataverse-base-image/.github/actions/setup-maven@10478-version-base-img + uses: IQSS/dataverse/.github/actions/setup-maven with: pom-paths: modules/container-base/pom.xml @@ -68,7 +65,9 @@ jobs: run: | .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} - # TODO: This job should become part of the matrix as an action, so we don't need to fiddle with matrix outputs hacks + # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. + # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string + # Will be part of working on #10618, app image versioned tags. #push-app-img: # name: "Rebase & Publish App Image" # permissions: @@ -77,7 +76,6 @@ jobs: # pull-requests: write # secrets: inherit # needs: - # - discover # - build # strategy: # fail-fast: false diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 1c9c6c7d5e3..a94b17a67ba 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -30,6 +30,7 @@ jobs: continue-on-error: ${{ matrix.experimental }} runs-on: ubuntu-latest steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -95,6 +96,7 @@ jobs: # status: "Experimental" continue-on-error: ${{ matrix.experimental }} steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - name: Set up JDK ${{ matrix.jdk }} @@ -128,6 +130,7 @@ jobs: needs: integration-test name: Coverage Report Submission steps: + # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - uses: actions/checkout@v3 - uses: actions/setup-java@v3 @@ -159,6 +162,7 @@ jobs: # TODO: Add a filter step here, that avoids calling the app image release workflow if there are changes to the base image. # Use https://github.com/dorny/paths-filter to solve this. Will require and additional job or adding to integration-test job. # This way we ensure that we're not running the app image flow with a non-matching base image. + # To become a part of #10618. push-app-img: name: Publish App Image