From eee3c4fd9fec250fa4efcc829fef62d7f7fc62fc Mon Sep 17 00:00:00 2001 From: mkosiarc Date: Thu, 28 Nov 2024 10:12:23 +0100 Subject: [PATCH] Improve generation of base images SBOMs most functional changes are in the related PR that updates the base_images_sbom_script.py https://github.com/konflux-ci/build-tasks-dockerfiles/pull/191 Here, we are just updating on how we generate the inputs for this script. We are now passing the whole parsed Dockerfile in json format to that script, which allows us to better parse/detect base images. Also, the format of the /shared/base_images_digests file was changed. Previously we could rely on the order of the image references with the digests in the file. Now we need to provide a mapping from an image reference as it was used in the Dockerfile to the full image reference with digests that was used during build and generated by buildah. The mapping is done as: Also, the sbom utility image has to be updated together in the same PR/commit, otherwise it would break konflux temporarily KFLUXBUGS-1718 Signed-off-by: mkosiarc --- .../0.1/build-image-index.yaml | 2 +- task/buildah-oci-ta/0.2/buildah-oci-ta.yaml | 18 ++++++++++-------- .../0.2/buildah-remote-oci-ta.yaml | 18 ++++++++++-------- task/buildah-remote/0.2/buildah-remote.yaml | 19 +++++++++++-------- task/buildah/0.2/buildah.yaml | 19 +++++++++++-------- 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/task/build-image-index/0.1/build-image-index.yaml b/task/build-image-index/0.1/build-image-index.yaml index 48855b7937..1c8088c5da 100644 --- a/task/build-image-index/0.1/build-image-index.yaml +++ b/task/build-image-index/0.1/build-image-index.yaml @@ -163,7 +163,7 @@ spec: add: - SETFCAP - - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4 + - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579 name: create-sbom computeResources: limits: diff --git a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml index bcf4db60ee..65f9cdd091 100644 --- a/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml +++ b/task/buildah-oci-ta/0.2/buildah-oci-ta.yaml @@ -348,9 +348,9 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -530,11 +530,13 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens + # if buildah did not use that particular image during build because it was skipped + if [ -n "$base_image_digest" ]; then + echo "$image $base_image_digest" >>/shared/base_images_digests + fi done - - # Needed to generate base images SBOM - echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile computeResources: limits: cpu: "4" @@ -651,7 +653,7 @@ spec: securityContext: runAsUser: 0 - name: prepare-sboms - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4 + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579 workingDir: /var/workdir script: | echo "Merging contents of sbom-source.json and sbom-image.json into sbom-cyclonedx.json" @@ -666,7 +668,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests echo "Adding image reference to sbom" diff --git a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml index c70529c4bd..c9650451b3 100644 --- a/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml +++ b/task/buildah-remote-oci-ta/0.2/buildah-remote-oci-ta.yaml @@ -382,9 +382,9 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" >/shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -564,12 +564,14 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >>/shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens + # if buildah did not use that particular image during build because it was skipped + if [ -n "$base_image_digest" ]; then + echo "$image $base_image_digest" >>/shared/base_images_digests + fi done - # Needed to generate base images SBOM - echo "$BASE_IMAGES" >/shared/base_images_from_dockerfile - buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE" REMOTESSHEOF chmod +x scripts/script-build.sh @@ -769,7 +771,7 @@ spec: requests: cpu: 100m memory: 256Mi - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4 + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579 name: prepare-sboms script: | #!/bin/bash @@ -790,7 +792,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests echo "Adding image reference to sbom" diff --git a/task/buildah-remote/0.2/buildah-remote.yaml b/task/buildah-remote/0.2/buildah-remote.yaml index 88a1ee48f3..e4362df69a 100644 --- a/task/buildah-remote/0.2/buildah-remote.yaml +++ b/task/buildah-remote/0.2/buildah-remote.yaml @@ -358,9 +358,10 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -542,12 +543,14 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens + # if buildah did not use that particular image during build because it was skipped + if [ -n "$base_image_digest" ]; then + echo "$image $base_image_digest" >> /shared/base_images_digests + fi done - # Needed to generate base images SBOM - echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile - buildah push "$IMAGE" "oci:konflux-final-image:$IMAGE" REMOTESSHEOF chmod +x scripts/script-build.sh @@ -749,7 +752,7 @@ spec: requests: cpu: 100m memory: 256Mi - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4 + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579 name: prepare-sboms script: | #!/bin/bash @@ -770,7 +773,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests echo "Adding image reference to sbom" diff --git a/task/buildah/0.2/buildah.yaml b/task/buildah/0.2/buildah.yaml index 34fdcbfa8b..53c0896edf 100644 --- a/task/buildah/0.2/buildah.yaml +++ b/task/buildah/0.2/buildah.yaml @@ -279,9 +279,10 @@ spec: BUILD_ARG_FLAGS+=("--build-arg=$build_arg") done + + dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" > /shared/parsed_dockerfile.json BASE_IMAGES=$( - dockerfile-json "${BUILD_ARG_FLAGS[@]}" "$dockerfile_copy" | - jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' + jq -r '.Stages[] | select(.From | .Stage or .Scratch | not) | .BaseName | select(test("^oci-archive:") | not)' /shared/parsed_dockerfile.json ) BUILDAH_ARGS=() @@ -463,12 +464,14 @@ spec: touch /shared/base_images_digests for image in $BASE_IMAGES; do - buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image" >> /shared/base_images_digests + base_image_digest=$(buildah images --format '{{ .Name }}:{{ .Tag }}@{{ .Digest }}' --filter reference="$image") + # In some cases, there might be BASE_IMAGES, but not any associated digest. This happens + # if buildah did not use that particular image during build because it was skipped + if [ -n "$base_image_digest" ]; then + echo "$image $base_image_digest" >> /shared/base_images_digests + fi done - # Needed to generate base images SBOM - echo "$BASE_IMAGES" > /shared/base_images_from_dockerfile - securityContext: capabilities: add: @@ -599,7 +602,7 @@ spec: runAsUser: 0 - name: prepare-sboms - image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:f1c07f273536a00fa6539e2aa41b3fddab3bb282a157c9676572858bfd19d7e4 + image: quay.io/redhat-appstudio/sbom-utility-scripts-image@sha256:e1347023ef1e83d52813c26384f551e3a03e482539d17a647955603e7ea6b579 computeResources: limits: memory: 512Mi @@ -620,7 +623,7 @@ spec: echo "Adding base images data to sbom-cyclonedx.json" python3 /scripts/base_images_sbom_script.py \ --sbom=sbom-cyclonedx.json \ - --base-images-from-dockerfile=/shared/base_images_from_dockerfile \ + --parsed-dockerfile=/shared/parsed_dockerfile.json \ --base-images-digests=/shared/base_images_digests echo "Adding image reference to sbom"