Merge pull request #422 from LF-Decentralized-Trust-labs/ci-chart-rel…

…ease

Simplify Helm Chart Release Process and Enhance CI Efficiency

.github/actions/update-yaml/action.yaml

@@ -0,0 +1,33 @@
+name: 'Update YAML File'
+description: 'Update values in a YAML file'
+inputs:
+  file:
+    description: 'The path to the YAML file to update'
+    required: true
+  updates:
+    description: 'YAML content with the updates to apply'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Install yq
+      shell: bash
+      run: |
+        wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O yq
+        chmod +x yq
+        sudo mv yq /usr/local/bin/
+
+    - name: Update YAML file
+      shell: bash
+      run: |
+        set -e
+        echo "Applying updates to ${{ inputs.file }}"
+
+        # Write the updates to a temporary YAML file
+        echo "${{ inputs.updates }}" > updates.yaml
+
+        # Merge the updates into the target YAML file
+        yq e -i '. * load("updates.yaml")' "${{ inputs.file }}"
+
+        echo "Updated ${{ inputs.file }}:"
+        cat "${{ inputs.file }}"

.github/workflows/build-image.yaml

@@ -1,18 +1,8 @@
 name: Build docker image
 permissions:
-  actions: read
-  checks: read
   contents: write
-  deployments: read
-  discussions: read
-  id-token: write
-  issues: read
   packages: write
-  pages: read
-  pull-requests: read
-  repository-projects: read
-  statuses: read
-  security-events: read
+  id-token: write
 
 on:
   workflow_call:
@@ -59,9 +49,6 @@ on:
 jobs:
   docker:
     runs-on: ${{ inputs.runs-on }}
-    permissions:
-      contents: read
-      packages: write
     steps:
       - name: Print inputs
         run: |

.github/workflows/cross-build-images.yaml

@@ -1,18 +1,8 @@
 name: Paladin Buildx
 permissions:
-  actions: read
-  checks: read
   contents: write
-  deployments: read
-  discussions: read
-  id-token: write
-  issues: read
   packages: write
-  pages: read
-  pull-requests: read
-  repository-projects: read
-  statuses: read
-  security-events: read
+  id-token: write
 
 on:
   workflow_call:
@@ -23,9 +13,10 @@ on:
       repository:
         required: true
         type: string
-      image_tag:
+      image_tags:
         required: true
         type: string
+        description: 'List of image tags separated by comma'
       push:
         required: false
         type: boolean
@@ -43,7 +34,7 @@ jobs:
       dockerfile: ./Dockerfile
       registry: ${{ inputs.registry }}
       image: ${{ inputs.repository }}/paladin
-      image_tag: "${{ inputs.image_tag }}-arm64"
+      image_tag: "${{ github.sha }}-arm64"
       push: ${{ inputs.push }}
       platforms: linux/arm64
       runs-on: ubuntu-latest
@@ -56,7 +47,7 @@ jobs:
       dockerfile: ./Dockerfile
       registry: ${{ inputs.registry }}
       image: ${{ inputs.repository }}/paladin
-      image_tag: "${{ inputs.image_tag }}-amd64"
+      image_tag: "${{ github.sha }}-amd64"
       push: ${{ inputs.push }}
       platforms: linux/amd64
       runs-on: ubuntu-latest
@@ -70,7 +61,7 @@ jobs:
       dockerfile: ./operator/Dockerfile
       registry: ${{ inputs.registry }}
       image: ${{ inputs.repository }}/paladin-operator
-      image_tag: ${{ inputs.image_tag }}-amd64
+      image_tag: ${{ github.sha }}-amd64
       push: ${{ inputs.push }}
       platforms: linux/amd64
       runs-on: ubuntu-latest
@@ -83,7 +74,7 @@ jobs:
       dockerfile: ./operator/Dockerfile
       registry: ${{ inputs.registry }}
       image: ${{ inputs.repository }}/paladin-operator
-      image_tag: ${{ inputs.image_tag }}-arm64
+      image_tag: ${{ github.sha }}-arm64
       push: ${{ inputs.push }}
       platforms: linux/arm64
       runs-on: ubuntu-latest
@@ -96,47 +87,81 @@ jobs:
     needs: [core-macos, core-ubuntu]
     runs-on: ubuntu-latest
     env:
-      IMAGE: ${{ inputs.registry }}/${{ inputs.repository }}/paladin:${{ inputs.image_tag }}
+      IMAGE: ${{ inputs.registry }}/${{ inputs.repository }}/paladin
     steps:
       - uses: actions/checkout@v4
+
       - name: Set lower case image name
         run: |
-          echo "IMAGE=${IMAGE,,}" >>${GITHUB_ENV}
+          LOWER_IMAGE=${IMAGE,,}
+          echo "IMAGE=$LOWER_IMAGE" >> $GITHUB_ENV
+          echo "TEMP_IMAGE=$LOWER_IMAGE:$GITHUB_SHA" >> $GITHUB_ENV
+          echo "Image: $LOWER_IMAGE"
+          echo "Temp image: $LOWER_IMAGE:$GITHUB_SHA"
+
       - name: Docker registry login
         uses: docker/login-action@v3
         with:
           registry: ${{ inputs.registry }}
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: Create and push Paladin core Docker manifest
+      - name: Create and push core manifest
+        run: |
+          docker manifest create $TEMP_IMAGE \
+            --amend $TEMP_IMAGE-arm64 \
+            --amend $TEMP_IMAGE-amd64
+          docker manifest push $TEMP_IMAGE
+
+      - name: Retag and push core image
+        env:
+          IMAGE_TAGS: ${{ inputs.image_tags }}
         run: |
-          docker manifest create ${{ env.IMAGE }} \
-            --amend ${{ env.IMAGE }}-arm64 \
-            --amend ${{ env.IMAGE }}-amd64
-          docker manifest push ${{ env.IMAGE }}
+          docker pull $TEMP_IMAGE
+          IFS=',' read -ra TAG_ARRAY <<< "$IMAGE_TAGS"
+          for tag in "${TAG_ARRAY[@]}"; do
+            docker tag $TEMP_IMAGE $IMAGE:$tag
+            docker push $IMAGE:$tag
+          done
 
   operator-docker-manifest:
     if: ${{ inputs.push }}
     needs: [operator-macos, operator-ubuntu]
     runs-on: ubuntu-latest
     env:
-      IMAGE: ${{ inputs.registry }}/${{ inputs.repository }}/paladin-operator:${{ inputs.image_tag }}
+      IMAGE: ${{ inputs.registry }}/${{ inputs.repository }}/paladin-operator
     steps:
       - uses: actions/checkout@v4
+
       - name: Set lower case image name
         run: |
-          echo "IMAGE=${IMAGE,,}" >>${GITHUB_ENV}
+          LOWER_IMAGE=${IMAGE,,}
+          echo "IMAGE=$LOWER_IMAGE" >> $GITHUB_ENV
+          echo "TEMP_IMAGE=$LOWER_IMAGE:$GITHUB_SHA" >> $GITHUB_ENV
+          echo "Image: $LOWER_IMAGE"
+          echo "Temp image: $LOWER_IMAGE:$GITHUB_SHA"
+
       - name: Docker registry login
         uses: docker/login-action@v3
         with:
           registry: ${{ inputs.registry }}
           username: ${{ secrets.username }}
           password: ${{ secrets.password }}
 
-      - name: Create and push Operaotr Docker manifest
+      - name: Create and push operator manifest
+        run: |
+          docker manifest create $TEMP_IMAGE \
+            --amend $TEMP_IMAGE-arm64 \
+            --amend $TEMP_IMAGE-amd64
+          docker manifest push $TEMP_IMAGE
+
+      - name: Retag and push operator image
+        env:
+          IMAGE_TAGS: ${{ inputs.image_tags }}
         run: |
-          docker manifest create ${{ env.IMAGE }} \
-            --amend ${{ env.IMAGE }}-arm64 \
-            --amend ${{ env.IMAGE }}-amd64
-          docker manifest push ${{ env.IMAGE }}
+          docker pull $TEMP_IMAGE
+          IFS=',' read -ra TAG_ARRAY <<< "$IMAGE_TAGS"
+          for tag in "${TAG_ARRAY[@]}"; do
+            docker tag $TEMP_IMAGE $IMAGE:$tag
+            docker push $IMAGE:$tag
+          done

.github/workflows/operator.yaml

@@ -1,16 +1,16 @@
 name: Paladin Operator Build
 
 on:
-  push:
-    branches:
-      - main
-    paths:
-      - 'operator/**'
   pull_request:
     paths:
       - 'operator/**'
   workflow_dispatch:
 
+# Ensure this workflow only runs for the most recent commit of a pull-request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   operator-build:
     runs-on: ubuntu-latest
@@ -65,6 +65,9 @@ jobs:
         with:
           install_only: true # only install kind, the cluster creation is managed by the deploy step
           ignore_failed_clean: true
+
+      # The makefile uses kustomize
+      - uses: imranismail/setup-kustomize@v2
 
       - name: Deploy Operator
         run: ./gradlew deploy -PclusterName=${{ env.CLUSTER_NAME }} -Pnamespace=${{ env.NAMESPACE }} 

.github/workflows/paladin-PR-build.yml

@@ -1,25 +1,26 @@
 name: Paladin Build
 permissions:
-  actions: read
-  checks: read
   contents: write
-  deployments: read
-  discussions: read
-  id-token: write
-  issues: read
   packages: write
-  pages: read
-  pull-requests: read
-  repository-projects: read
-  statuses: read
-  security-events: read
+  id-token: write
 
 on:
   push:
     branches: [main]
+    paths-ignore:
+      - '**.md'
+      - 'operator/charts/**'
   pull_request:
+    paths-ignore:
+      - '**.md'
+      - 'operator/charts/**'
   workflow_dispatch:
 
+# Ensure this workflow only runs for the most recent commit of a pull-request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -104,7 +105,7 @@ jobs:
     with:
       registry: ghcr.io
       repository: ${{ github.repository_owner }}
-      image_tag: ${{ github.ref_name }}
+      image_tags: ${{ github.ref_name }}
     secrets:
       username: ${{ github.actor }}
       password: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release-charts.yaml

@@ -7,10 +7,25 @@ on:
         required: false
         type: boolean
         default: false
-
+      tag:
+        required: true
+        type: string
+        description: 'The tag to release the chart and images with'
+
 jobs:
+  build-images:
+    # build and release images
+    uses: ./.github/workflows/release-images.yaml
+    with:
+      tag: ${{ inputs.tag }}
+      latest: ${{ inputs.latest }}
+    secrets:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
   helm-chart-release:
     runs-on: ubuntu-latest
+    needs: build-images
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -54,13 +69,58 @@ jobs:
           git config user.name "$GITHUB_ACTOR"
           git config user.email "[email protected]"
       
+      # The makefile uses kustomize
+      - uses: imranismail/setup-kustomize@v2
+
       - name: Prepare CRD chart
-        working-directory: operator
-        run: gradle prepareCRDsChart
+        run: ./gradlew prepareCRDsChart
 
       - name: Prepare operator chart
         run: ./gradlew prepareOperatorChart
 
+      - name: Set Lowercase Repository Owner
+        run: echo "LOWER_REPOSITORY_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV
+
+      # Override the image tags in the values.yaml file 
+      - name: Override image tags
+        uses: ./.github/actions/update-yaml
+        with:
+          file: 'operator/charts/paladin-operator/values.yaml'
+          updates: |
+            operator:
+              image:
+                repository: docker.io/${{ env.LOWER_REPOSITORY_OWNER }}/paladin-operator
+                tag: ${{ inputs.tag }}
+            paladin:
+              image:
+                repository: docker.io/${{ env.LOWER_REPOSITORY_OWNER }}/paladin
+                tag: ${{ inputs.tag }}
+      
+      # Remove 'v' prefix from the tag
+      - name: Process Tag
+        shell: bash
+        run: |
+          TAG="${{ inputs.tag }}"
+          PROCESSED_TAG="${TAG#v}"
+          echo "PROCESSED_TAG=${PROCESSED_TAG}" >> $GITHUB_ENV
+
+      # Override the chart version in the Chart.yaml file
+      - name: Override chart version
+        uses: ./.github/actions/update-yaml
+        with:
+          file: 'operator/charts/paladin-operator/Chart.yaml'
+          updates: |
+            version: ${{ env.PROCESSED_TAG }}
+            appVersion: ${{ inputs.tag }}
+
+      - name: Confirm Helm temapltes
+        working-directory: operator/charts/paladin-operator
+        run: |
+          helm dependency build
+          helm template .
+          rm -rf charts
+          rm Chart.lock
+
       - name: Run chart-releaser
         uses: helm/[email protected]
         with: