From 9fd484ba7032e032b0f99d1b1c2b4712fb6a06a1 Mon Sep 17 00:00:00 2001
From: Daniela Plascencia <daniela.plascencia@canonical.com>
Date: Wed, 9 Oct 2024 04:07:33 +0200
Subject: [PATCH 1/4] ci: add re-usable workflow for scans from published img
 and automatic reports

This commit adds get-published-images-scan-and-report.yaml, a re-usable workflow
that enables repositories to scan images from a public registry (in the case
of the Analytics team it defaults to charmedkubeflow) and reports back
the security vulnerabilities as Github issues.
This workflow is intended to be used on demand (using a workflow dispatch)
and on schedule, as it will be used for continuous testing of the published
images a rock repository generates.

Part of #69
---
 .../workflows/get-published-image-names.yaml  |  41 +++++++
 .../get-published-images-scan-and-report.yaml |  37 +++++++
 .../scan-from-dockerhub-report-issue.yaml     |  38 +++++++
 .../workflows/scan-from-published-image.yaml  | 104 ++++++++++++++++++
 4 files changed, 220 insertions(+)
 create mode 100644 .github/workflows/get-published-image-names.yaml
 create mode 100644 .github/workflows/get-published-images-scan-and-report.yaml
 create mode 100644 .github/workflows/scan-from-dockerhub-report-issue.yaml
 create mode 100644 .github/workflows/scan-from-published-image.yaml

diff --git a/.github/workflows/get-published-image-names.yaml b/.github/workflows/get-published-image-names.yaml
new file mode 100644
index 0000000..143acce
--- /dev/null
+++ b/.github/workflows/get-published-image-names.yaml
@@ -0,0 +1,41 @@
+name: Get oci-images names from Dockerhub
+
+on:
+  workflow_call:
+    outputs:
+      images-names:
+        description: A JSON array of images that Paths in this repository where rockcraft projects are stored
+        value: ${{ jobs.get-images-names.outputs.images-names }}
+
+jobs:
+  get-images-names:
+    name: Get all rockcraft.yaml paths
+    runs-on: ubuntu-22.04
+    outputs:
+      images-names: ${{ steps.get-images.outputs.images-names }}
+    steps:
+      - name: Checkout repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install tools
+        run: |
+          sudo snap install yq
+          sudo apt install jq
+
+      - name: Get images names
+        id: get-images
+        run: |
+          set -xeu
+          IMAGES_NAMES=()
+          paths=$(find ./ -name "rockcraft.yaml" | sed 's/\.\///g')
+          for d in $paths
+          do
+            short_hash=$(git log -n 1 --pretty=%h -- ${d})
+            image_name=$(cat ${d} | yq -r '.name')
+            image_version=$(cat ${d} | yq -r '.version')
+            IMAGES_NAMES+=(${image_name}:${image_version}-${short_hash})
+          done
+          IMAGES_ARRAY=$(jq -c -n '$ARGS.positional' --args "${IMAGES_NAMES[@]}")
+          echo "images-names=${IMAGES_ARRAY}" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/get-published-images-scan-and-report.yaml b/.github/workflows/get-published-images-scan-and-report.yaml
new file mode 100644
index 0000000..419e9bd
--- /dev/null
+++ b/.github/workflows/get-published-images-scan-and-report.yaml
@@ -0,0 +1,37 @@
+name: Get ROCKs modified and build-scan-test-publish them
+
+on:
+  workflow_call:
+    secrets:
+      GH_TOKEN:
+        required: true
+    inputs:
+      report-vulnerabilities:
+        description: "Whether to report security vulnerabilities through Github issues."
+        required: false
+        default: false
+        type: boolean
+      severity:
+        description: "Comma separated list of severities of vulnerabilities to scanned for and displayed"
+        required: false
+        type: string
+        default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL"
+
+
+jobs:
+  get-published-images-names:
+    name: Get published images names
+    uses: ./.github/workflows/get-published-image-names.yaml
+
+  scan-report-vulnerability:
+    needs: get-published-images-names
+    strategy:
+      fail-fast: false
+      matrix:
+        image-name: ${{ fromJson(needs.get-published-images-names.outputs.images-names) }}
+    uses: ./.github/workflows/scan-from-dockerhub-report-issue.yaml
+    secrets: inherit
+    with:
+      image-name: ${{ matrix.image-name }}
+      report-vulnerabilities: ${{ inputs.report-vulnerabilities }}
+      severity: ${{ inputs.severity }}
diff --git a/.github/workflows/scan-from-dockerhub-report-issue.yaml b/.github/workflows/scan-from-dockerhub-report-issue.yaml
new file mode 100644
index 0000000..85da866
--- /dev/null
+++ b/.github/workflows/scan-from-dockerhub-report-issue.yaml
@@ -0,0 +1,38 @@
+name: Scan from published image and report vulnerabilities
+
+on:
+  workflow_call:
+    inputs:
+      image-name:
+        description: "The published image name to be scanned."
+        required: true
+        type: string
+      report-vulnerabilities:
+        description: "Whether to report security vulnerabilities through Github issues."
+        required: false
+        default: false
+        type: boolean
+      severity:
+        description: "Comma separated list of severities of vulnerabilities to scanned for and displayed"
+        required: false
+        type: string
+        default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL"
+jobs:
+  scan:
+    # uses: canonical/charmed-kubeflow-workflows/.github/workflows/scan-from-published-image.yaml@KF-6331-add-scan-reusable-workflow
+    uses: ./.github/workflows/scan-from-published-image.yaml
+    secrets: inherit
+    with:
+      image-name: ${{ inputs.image-name }}
+      report-vulnerabilities: ${{ inputs.report-vulnerabilities }}
+      severity: ${{ inputs.severity }}
+
+  # Leaving it here in case we ever want to enable scan and reports on_merge
+  report-vulnerability:
+    needs: scan
+    uses: ./.github/workflows/report-vulnerability-in-gh.yaml
+    secrets: inherit
+    if: ${{ always() && (needs.scan.result == 'failure') }}
+    with:
+      issue-title: 'Vulnerabilities found for'
+      image-name: ${{ needs.scan.outputs.image-name-dashes }}
diff --git a/.github/workflows/scan-from-published-image.yaml b/.github/workflows/scan-from-published-image.yaml
new file mode 100644
index 0000000..ec3b662
--- /dev/null
+++ b/.github/workflows/scan-from-published-image.yaml
@@ -0,0 +1,104 @@
+name: Scan
+
+on:
+  workflow_call:
+    outputs:
+      image-name-dashes:
+        description: "The image name with format <image-name>-<version>"
+        value: ${{ jobs.scan.outputs.image-name-dashes }}
+    inputs:
+      container-registry:
+        description: "The name of the container registry where images are hosted."
+        required: false
+        type: string
+        default: "charmedkubeflow"
+      image-name:
+        description: "The published image name to be scanned."
+        required: true
+        type: string
+      report-vulnerabilities:
+        description: "Whether to report security vulnerabilities through Github issues."
+        required: false
+        default: false
+        type: boolean
+      severity:
+        description: "Comma separated list of severities of vulnerabilities to scanned for and displayed"
+        required: false
+        type: string
+        default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL"
+jobs:
+  scan:
+    name: Scan of ${{ inputs.image-name }}
+    runs-on: ubuntu-22.04
+    outputs:
+      image-name: ${{ steps.image-name.outputs.image-name }}
+      image-name-dashes: ${{ steps.image-name.outputs.image-name-dashes }}
+    strategy:
+      fail-fast: false
+    steps:
+      # Ideally we'd use self-hosted runners, but this effort is still not stable.
+      # This action will remove unused software (dotnet, haskell, android libs, codeql,
+      # and docker images) from the GH runner, which will liberate around 60 GB of storage
+      # distributed in 40GB for root and around 20 for a mnt point.
+      # We need it to avoid cases where scanning fails due to "no space left on device".
+      - name: Maximise GH runner space
+        uses: easimon/maximize-build-space@v7
+        with:
+          root-reserve-mb: 29696
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-android: 'true'
+          remove-codeql: 'true'
+
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Set up inputs for scan
+        id: set-up-inputs
+        run: |
+          echo "exit-code=1" >> "$GITHUB_OUTPUT"
+          if ${{ inputs.report-vulnerabilities == false }}; then
+            echo "exit-code=0" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Generate image name
+        id: image-name
+        run: |
+          IMAGE_NAME=$(echo ${{ inputs.image-name }} | rev | cut -f2- -d"-" | rev)
+          IMAGE_NAME_DASHES=$(echo $IMAGE_NAME | sed 's/\:/-/g')
+          echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
+          echo "image-name-dashes=${IMAGE_NAME_DASHES}" >> "$GITHUB_OUTPUT"
+
+      - name: Scan for vulnerabilities
+        id: scan
+        uses: aquasecurity/trivy-action@0.25.0
+        # Workaround for https://github.com/aquasecurity/trivy-action/issues/389
+        env:
+          TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
+          TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1
+        with:
+          scan-type: 'image'
+          image-ref: '${{ inputs.container-registry }}/${{ inputs.image-name }}'
+          format: 'table'
+          output: 'trivy-report-${{ steps.image-name.outputs.image-name-dashes }}.txt'
+          ignore-unfixed: true
+          timeout: '50m0s'
+          exit-code: ${{ steps.set-up-inputs.outputs.exit-code }}
+          severity: ${{ inputs.severity }}
+          # NOTE: pebble is flagged with a HIGH vuln because of golang.org/x/crypto
+          # CVE-2021-43565, CVE-2022-27191
+          skip-files: '/bin/pebble,/usr/bin/pebble,usr/bin/pebble,bin/pebble'
+
+      - name: Print vulnerabilities report
+        # The report should be printed regardless of the success of the previous step
+        if: success() || failure()
+        run: cat trivy-report-${{ steps.image-name.outputs.image-name-dashes }}.txt
+
+      - name: Upload Trivy reports
+        # The report should be uploaded regardless of the success of the previous steps
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          compression-level: 0
+          name: trivy-report-${{ steps.image-name.outputs.image-name-dashes }}
+          path: trivy-report-${{ steps.image-name.outputs.image-name-dashes }}.txt

From 075ce4b980ac727920b7b13bd2aedf02ea607600 Mon Sep 17 00:00:00 2001
From: Daniela Plascencia <daniela.plascencia@canonical.com>
Date: Thu, 10 Oct 2024 18:39:36 +0200
Subject: [PATCH 2/4] skip: changes based on feedback

---
 ...ml => get-image-names-from-rockcraft.yaml} |  6 +--
 .../get-published-images-scan-and-report.yaml | 13 ++-----
 .../scan-from-dockerhub-report-issue.yaml     | 13 ++-----
 .../workflows/scan-from-published-image.yaml  | 39 ++++++-------------
 4 files changed, 22 insertions(+), 49 deletions(-)
 rename .github/workflows/{get-published-image-names.yaml => get-image-names-from-rockcraft.yaml} (85%)

diff --git a/.github/workflows/get-published-image-names.yaml b/.github/workflows/get-image-names-from-rockcraft.yaml
similarity index 85%
rename from .github/workflows/get-published-image-names.yaml
rename to .github/workflows/get-image-names-from-rockcraft.yaml
index 143acce..e780810 100644
--- a/.github/workflows/get-published-image-names.yaml
+++ b/.github/workflows/get-image-names-from-rockcraft.yaml
@@ -1,15 +1,15 @@
-name: Get oci-images names from Dockerhub
+name: Get oci-images names from rockcraft files
 
 on:
   workflow_call:
     outputs:
       images-names:
-        description: A JSON array of images that Paths in this repository where rockcraft projects are stored
+        description: "A JSON array of all images that have their rockcraft project stored in this repository"
         value: ${{ jobs.get-images-names.outputs.images-names }}
 
 jobs:
   get-images-names:
-    name: Get all rockcraft.yaml paths
+    name: Get image names
     runs-on: ubuntu-22.04
     outputs:
       images-names: ${{ steps.get-images.outputs.images-names }}
diff --git a/.github/workflows/get-published-images-scan-and-report.yaml b/.github/workflows/get-published-images-scan-and-report.yaml
index 419e9bd..d2e039f 100644
--- a/.github/workflows/get-published-images-scan-and-report.yaml
+++ b/.github/workflows/get-published-images-scan-and-report.yaml
@@ -1,4 +1,4 @@
-name: Get ROCKs modified and build-scan-test-publish them
+name: Get image names and report vulnerabilties in Github
 
 on:
   workflow_call:
@@ -6,11 +6,6 @@ on:
       GH_TOKEN:
         required: true
     inputs:
-      report-vulnerabilities:
-        description: "Whether to report security vulnerabilities through Github issues."
-        required: false
-        default: false
-        type: boolean
       severity:
         description: "Comma separated list of severities of vulnerabilities to scanned for and displayed"
         required: false
@@ -20,11 +15,12 @@ on:
 
 jobs:
   get-published-images-names:
-    name: Get published images names
-    uses: ./.github/workflows/get-published-image-names.yaml
+    name: Get images names from rockcraft
+    uses: ./.github/workflows/get-image-names-from-rockcraft.yaml
 
   scan-report-vulnerability:
     needs: get-published-images-names
+    name: Scan and report
     strategy:
       fail-fast: false
       matrix:
@@ -33,5 +29,4 @@ jobs:
     secrets: inherit
     with:
       image-name: ${{ matrix.image-name }}
-      report-vulnerabilities: ${{ inputs.report-vulnerabilities }}
       severity: ${{ inputs.severity }}
diff --git a/.github/workflows/scan-from-dockerhub-report-issue.yaml b/.github/workflows/scan-from-dockerhub-report-issue.yaml
index 85da866..75575bf 100644
--- a/.github/workflows/scan-from-dockerhub-report-issue.yaml
+++ b/.github/workflows/scan-from-dockerhub-report-issue.yaml
@@ -1,4 +1,4 @@
-name: Scan from published image and report vulnerabilities
+name: Scan published image and report vulnerabilities
 
 on:
   workflow_call:
@@ -7,11 +7,6 @@ on:
         description: "The published image name to be scanned."
         required: true
         type: string
-      report-vulnerabilities:
-        description: "Whether to report security vulnerabilities through Github issues."
-        required: false
-        default: false
-        type: boolean
       severity:
         description: "Comma separated list of severities of vulnerabilities to scanned for and displayed"
         required: false
@@ -19,15 +14,12 @@ on:
         default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL"
 jobs:
   scan:
-    # uses: canonical/charmed-kubeflow-workflows/.github/workflows/scan-from-published-image.yaml@KF-6331-add-scan-reusable-workflow
     uses: ./.github/workflows/scan-from-published-image.yaml
     secrets: inherit
     with:
       image-name: ${{ inputs.image-name }}
-      report-vulnerabilities: ${{ inputs.report-vulnerabilities }}
       severity: ${{ inputs.severity }}
 
-  # Leaving it here in case we ever want to enable scan and reports on_merge
   report-vulnerability:
     needs: scan
     uses: ./.github/workflows/report-vulnerability-in-gh.yaml
@@ -35,4 +27,5 @@ jobs:
     if: ${{ always() && (needs.scan.result == 'failure') }}
     with:
       issue-title: 'Vulnerabilities found for'
-      image-name: ${{ needs.scan.outputs.image-name-dashes }}
+      image-name: ${{ inputs.image-name }}
+      vulnerability-report-artefact: ${{ needs.scan.outputs.vulnerability-report-artefact-name }}
diff --git a/.github/workflows/scan-from-published-image.yaml b/.github/workflows/scan-from-published-image.yaml
index ec3b662..db7e935 100644
--- a/.github/workflows/scan-from-published-image.yaml
+++ b/.github/workflows/scan-from-published-image.yaml
@@ -3,9 +3,9 @@ name: Scan
 on:
   workflow_call:
     outputs:
-      image-name-dashes:
-        description: "The image name with format <image-name>-<version>"
-        value: ${{ jobs.scan.outputs.image-name-dashes }}
+      vulnerability-report-artefact-name:
+        description: "The name of the artefact that contains the vulnerability report."
+        value: ${{ jobs.scan.outputs.vulnerability-report-artefact-name }}
     inputs:
       container-registry:
         description: "The name of the container registry where images are hosted."
@@ -16,11 +16,6 @@ on:
         description: "The published image name to be scanned."
         required: true
         type: string
-      report-vulnerabilities:
-        description: "Whether to report security vulnerabilities through Github issues."
-        required: false
-        default: false
-        type: boolean
       severity:
         description: "Comma separated list of severities of vulnerabilities to scanned for and displayed"
         required: false
@@ -31,8 +26,7 @@ jobs:
     name: Scan of ${{ inputs.image-name }}
     runs-on: ubuntu-22.04
     outputs:
-      image-name: ${{ steps.image-name.outputs.image-name }}
-      image-name-dashes: ${{ steps.image-name.outputs.image-name-dashes }}
+      vulnerability-report-artefact-name: ${{ steps.report-name.outputs.report-name }}
     strategy:
       fail-fast: false
     steps:
@@ -53,21 +47,12 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v4
 
-      - name: Set up inputs for scan
-        id: set-up-inputs
-        run: |
-          echo "exit-code=1" >> "$GITHUB_OUTPUT"
-          if ${{ inputs.report-vulnerabilities == false }}; then
-            echo "exit-code=0" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Generate image name
-        id: image-name
+      - name: Generate report name
+        id: report-name
         run: |
           IMAGE_NAME=$(echo ${{ inputs.image-name }} | rev | cut -f2- -d"-" | rev)
           IMAGE_NAME_DASHES=$(echo $IMAGE_NAME | sed 's/\:/-/g')
-          echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
-          echo "image-name-dashes=${IMAGE_NAME_DASHES}" >> "$GITHUB_OUTPUT"
+          echo "report-name='trivy-report-${IMAGE_NAME_DASHES}'" >> "$GITHUB_OUTPUT"
 
       - name: Scan for vulnerabilities
         id: scan
@@ -80,10 +65,10 @@ jobs:
           scan-type: 'image'
           image-ref: '${{ inputs.container-registry }}/${{ inputs.image-name }}'
           format: 'table'
-          output: 'trivy-report-${{ steps.image-name.outputs.image-name-dashes }}.txt'
+          output: '${{ steps.report-name.outputs.report-name }}.txt'
           ignore-unfixed: true
           timeout: '50m0s'
-          exit-code: ${{ steps.set-up-inputs.outputs.exit-code }}
+          exit-code: 1
           severity: ${{ inputs.severity }}
           # NOTE: pebble is flagged with a HIGH vuln because of golang.org/x/crypto
           # CVE-2021-43565, CVE-2022-27191
@@ -92,7 +77,7 @@ jobs:
       - name: Print vulnerabilities report
         # The report should be printed regardless of the success of the previous step
         if: success() || failure()
-        run: cat trivy-report-${{ steps.image-name.outputs.image-name-dashes }}.txt
+        run: cat ${{ steps.report-name.outputs.report-name }}.txt
 
       - name: Upload Trivy reports
         # The report should be uploaded regardless of the success of the previous steps
@@ -100,5 +85,5 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           compression-level: 0
-          name: trivy-report-${{ steps.image-name.outputs.image-name-dashes }}
-          path: trivy-report-${{ steps.image-name.outputs.image-name-dashes }}.txt
+          name: ${{ steps.report-name.outputs.report-name }}
+          path: ${{ steps.report-name.outputs.report-name }}.txt

From f71aa72c130c2d8452dbd28b2d8929b9c2b1a09e Mon Sep 17 00:00:00 2001
From: Daniela Plascencia <daniela.plascencia@canonical.com>
Date: Thu, 10 Oct 2024 19:17:06 +0200
Subject: [PATCH 3/4] skip: remove quotes

---
 .github/workflows/scan-from-published-image.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/scan-from-published-image.yaml b/.github/workflows/scan-from-published-image.yaml
index db7e935..5564370 100644
--- a/.github/workflows/scan-from-published-image.yaml
+++ b/.github/workflows/scan-from-published-image.yaml
@@ -52,7 +52,7 @@ jobs:
         run: |
           IMAGE_NAME=$(echo ${{ inputs.image-name }} | rev | cut -f2- -d"-" | rev)
           IMAGE_NAME_DASHES=$(echo $IMAGE_NAME | sed 's/\:/-/g')
-          echo "report-name='trivy-report-${IMAGE_NAME_DASHES}'" >> "$GITHUB_OUTPUT"
+          echo "report-name=trivy-report-${IMAGE_NAME_DASHES}" >> "$GITHUB_OUTPUT"
 
       - name: Scan for vulnerabilities
         id: scan

From 74df13879469e3d2c563aa5fe10a237ad66ed24e Mon Sep 17 00:00:00 2001
From: Daniela Plascencia <daniela.plascencia@canonical.com>
Date: Mon, 14 Oct 2024 16:21:30 +0200
Subject: [PATCH 4/4] skip: add branching strategy

---
 .github/workflows/get-image-names-from-rockcraft.yaml       | 5 +++++
 .github/workflows/get-published-images-scan-and-report.yaml | 6 +++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/get-image-names-from-rockcraft.yaml b/.github/workflows/get-image-names-from-rockcraft.yaml
index e780810..f2d3b09 100644
--- a/.github/workflows/get-image-names-from-rockcraft.yaml
+++ b/.github/workflows/get-image-names-from-rockcraft.yaml
@@ -2,6 +2,10 @@ name: Get oci-images names from rockcraft files
 
 on:
   workflow_call:
+    inputs:
+      branch:
+        required: false
+        type: string
     outputs:
       images-names:
         description: "A JSON array of all images that have their rockcraft project stored in this repository"
@@ -18,6 +22,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ inputs.branch }}
 
       - name: Install tools
         run: |
diff --git a/.github/workflows/get-published-images-scan-and-report.yaml b/.github/workflows/get-published-images-scan-and-report.yaml
index d2e039f..2814d6e 100644
--- a/.github/workflows/get-published-images-scan-and-report.yaml
+++ b/.github/workflows/get-published-images-scan-and-report.yaml
@@ -11,12 +11,16 @@ on:
         required: false
         type: string
         default: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL"
-
+      branch:
+        required: false
+        type: string
 
 jobs:
   get-published-images-names:
     name: Get images names from rockcraft
     uses: ./.github/workflows/get-image-names-from-rockcraft.yaml
+    with:
+      branch: ${{ inputs.branch }}
 
   scan-report-vulnerability:
     needs: get-published-images-names