Add image index task to pipelines

The build-image-index task is added to all pipelines but the generation of an image index is disabled by default. Signed-off-by: arewm <[email protected]>
konflux-ci · Aug 21, 2024 · 47cb196 · 47cb196
1 parent caa47b4
commit 47cb196
Show file tree

Hide file tree

Showing 12 changed files with 446 additions and 270 deletions.
diff --git a/pipelines/docker-build-oci-ta/README.md b/pipelines/docker-build-oci-ta/README.md
@@ -4,13 +4,14 @@
 |---|---|---|---|
 |build-args| Array of --build-arg values ("arg=value" strings) for buildah| []| build-container:0.2:BUILD_ARGS|
 |build-args-file| Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file| | build-container:0.2:BUILD_ARGS_FILE|
+|build-image-index| Add built image into an OCI image index| false| build-image-index:0.1:ALWAYS_BUILD_INDEX|
 |build-source-image| Build a source image.| false| |
 |dockerfile| Path to the Dockerfile inside the context specified by parameter path-context| Dockerfile| build-container:0.2:DOCKERFILE ; push-dockerfile:0.1:DOCKERFILE|
 |git-url| Source Repository URL| None| clone-repository:0.1:url|
 |hermetic| Execute the build with network isolation| false| build-container:0.2:HERMETIC|
-|image-expires-after| Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively.| | clone-repository:0.1:ociArtifactExpiresAfter ; prefetch-dependencies:0.1:ociArtifactExpiresAfter ; build-container:0.2:IMAGE_EXPIRES_AFTER|
+|image-expires-after| Image tag expiration time, time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively.| | clone-repository:0.1:ociArtifactExpiresAfter ; prefetch-dependencies:0.1:ociArtifactExpiresAfter ; build-container:0.2:IMAGE_EXPIRES_AFTER ; build-image-index:0.1:IMAGE_EXPIRES_AFTER|
 |java| Java build| false| |
-|output-image| Fully Qualified Output Image| None| init:0.2:image-url ; clone-repository:0.1:ociStorage ; prefetch-dependencies:0.1:ociStorage ; build-container:0.2:IMAGE ; build-source-image:0.1:BINARY_IMAGE|
+|output-image| Fully Qualified Output Image| None| init:0.2:image-url ; clone-repository:0.1:ociStorage ; prefetch-dependencies:0.1:ociStorage ; build-container:0.2:IMAGE ; build-image-index:0.1:IMAGE ; build-source-image:0.1:BINARY_IMAGE|
 |path-context| Path to the source code of an application's component from where to build image.| .| build-container:0.2:CONTEXT ; push-dockerfile:0.1:CONTEXT|
 |prefetch-input| Build dependencies to be prefetched by Cachi2| | prefetch-dependencies:0.1:input ; build-container:0.2:PREFETCH_INPUT|
 |rebuild| Force rebuild image| false| init:0.2:rebuild|
@@ -23,7 +24,17 @@
 |ADDITIONAL_TAGS| Additional tags that will be applied to the image in the registry.| []| |
 |CA_TRUST_CONFIG_MAP_KEY| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| |
 |CA_TRUST_CONFIG_MAP_NAME| The name of the ConfigMap to read CA bundle data from.| trusted-ca| |
-|IMAGE| Reference of image that was pushed to registry in the buildah task.| None| '$(tasks.build-container.results.IMAGE_URL)'|
+|IMAGE| Reference of image that was pushed to registry in the buildah task.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
+### build-image-index:0.1 task parameters
+|name|description|default value|already set by|
+|---|---|---|---|
+|ALWAYS_BUILD_INDEX| Build an image index even if IMAGES is of length 1. Default true. If the image index generation is skipped, the task will forward values for params.IMAGES[0] to results.IMAGE_*. In order to properly set all results, use the repository:tag@sha256:digest format for the IMAGES parameter.| true| '$(params.build-image-index)'|
+|COMMIT_SHA| The commit the image is built from.| | '$(tasks.clone-repository.results.commit)'|
+|IMAGE| The target image and tag where the image will be pushed to.| None| '$(params.output-image)'|
+|IMAGES| List of Image Manifests to be referenced by the Image Index| None| '['$(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST)']'|
+|IMAGE_EXPIRES_AFTER| Delete image tag after specified time resulting in garbage collection of the digest. Empty means to keep the image tag. Time values could be something like 1h, 2d, 3w for hours, days, and weeks, respectively.| | '$(params.image-expires-after)'|
+|STORAGE_DRIVER| Storage driver to configure for buildah| vfs| |
+|TLSVERIFY| Verify the TLS on the registry endpoint (for push/pull to a non-TLS registry)| true| |
 ### buildah-oci-ta:0.2 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
@@ -58,32 +69,32 @@
 |ca-trust-config-map-key| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| |
 |ca-trust-config-map-name| The name of the ConfigMap to read CA bundle data from.| trusted-ca| |
 |docker-auth| unused, should be removed in next task version.| | |
-|image-digest| Image digest to scan.| None| '$(tasks.build-container.results.IMAGE_DIGEST)'|
-|image-url| Image URL.| None| '$(tasks.build-container.results.IMAGE_URL)'|
+|image-digest| Image digest to scan.| None| '$(tasks.build-image-index.results.IMAGE_DIGEST)'|
+|image-url| Image URL.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
 ### clamav-scan:0.1 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
 |ca-trust-config-map-key| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| |
 |ca-trust-config-map-name| The name of the ConfigMap to read CA bundle data from.| trusted-ca| |
 |docker-auth| unused| | |
-|image-digest| Image digest to scan.| None| '$(tasks.build-container.results.IMAGE_DIGEST)'|
-|image-url| Image URL.| None| '$(tasks.build-container.results.IMAGE_URL)'|
+|image-digest| Image digest to scan.| None| '$(tasks.build-image-index.results.IMAGE_DIGEST)'|
+|image-url| Image URL.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
 ### deprecated-image-check:0.4 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
 |BASE_IMAGES_DIGESTS| Digests of base build images.| | |
 |CA_TRUST_CONFIG_MAP_KEY| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| |
 |CA_TRUST_CONFIG_MAP_NAME| The name of the ConfigMap to read CA bundle data from.| trusted-ca| |
-|IMAGE_DIGEST| Image digest.| None| '$(tasks.build-container.results.IMAGE_DIGEST)'|
-|IMAGE_URL| Fully qualified image name.| None| '$(tasks.build-container.results.IMAGE_URL)'|
+|IMAGE_DIGEST| Image digest.| None| '$(tasks.build-image-index.results.IMAGE_DIGEST)'|
+|IMAGE_URL| Fully qualified image name.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
 |POLICY_DIR| Path to directory containing Conftest policies.| /project/repository/| |
 |POLICY_NAMESPACE| Namespace for Conftest policy.| required_checks| |
 ### ecosystem-cert-preflight-checks:0.1 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
 |ca-trust-config-map-key| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| |
 |ca-trust-config-map-name| The name of the ConfigMap to read CA bundle data from.| trusted-ca| |
-|image-url| Image url to scan.| None| '$(tasks.build-container.results.IMAGE_URL)'|
+|image-url| Image url to scan.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
 ### git-clone-oci-ta:0.1 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
@@ -129,32 +140,32 @@
 |ARTIFACT_TYPE| Artifact type of the Dockerfile image.| application/vnd.konflux.dockerfile| |
 |CONTEXT| Path to the directory to use as context.| .| '$(params.path-context)'|
 |DOCKERFILE| Path to the Dockerfile.| ./Dockerfile| '$(params.dockerfile)'|
-|IMAGE| The built binary image. The Dockerfile is pushed to the same image repository alongside.| None| '$(tasks.build-container.results.IMAGE_URL)'|
-|IMAGE_DIGEST| The built binary image digest, which is used to construct the tag of Dockerfile image.| None| '$(tasks.build-container.results.IMAGE_DIGEST)'|
+|IMAGE| The built binary image. The Dockerfile is pushed to the same image repository alongside.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
+|IMAGE_DIGEST| The built binary image digest, which is used to construct the tag of Dockerfile image.| None| '$(tasks.build-image-index.results.IMAGE_DIGEST)'|
 |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)'|
 |TAG_SUFFIX| Suffix of the Dockerfile image tag.| .dockerfile| |
 ### sast-snyk-check-oci-ta:0.2 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
 |ARGS| Append arguments.| --all-projects --exclude=test*,vendor,deps| |
-|CACHI2_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the prefetched dependencies.| | '$(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)'|
+|CACHI2_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the prefetched dependencies.| | |
 |SNYK_SECRET| Name of secret which contains Snyk token.| snyk-secret| |
 |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| None| '$(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)'|
-|image-digest| Image digest to report findings for.| | '$(tasks.build-container.results.IMAGE_DIGEST)'|
-|image-url| Image URL.| | '$(tasks.build-container.results.IMAGE_URL)'|
+|image-digest| Image digest to report findings for.| | '$(tasks.build-image-index.results.IMAGE_DIGEST)'|
+|image-url| Image URL.| | '$(tasks.build-image-index.results.IMAGE_URL)'|
 ### sbom-json-check:0.1 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
 |CA_TRUST_CONFIG_MAP_KEY| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| |
 |CA_TRUST_CONFIG_MAP_NAME| The name of the ConfigMap to read CA bundle data from.| trusted-ca| |
-|IMAGE_DIGEST| Image digest.| None| '$(tasks.build-container.results.IMAGE_DIGEST)'|
-|IMAGE_URL| Fully qualified image name to verify.| None| '$(tasks.build-container.results.IMAGE_URL)'|
+|IMAGE_DIGEST| Image digest.| None| '$(tasks.build-image-index.results.IMAGE_DIGEST)'|
+|IMAGE_URL| Fully qualified image name to verify.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
 ### show-sbom:0.1 task parameters
 |name|description|default value|already set by|
 |---|---|---|---|
 |CA_TRUST_CONFIG_MAP_KEY| The name of the key in the ConfigMap that contains the CA bundle data.| ca-bundle.crt| |
 |CA_TRUST_CONFIG_MAP_NAME| The name of the ConfigMap to read CA bundle data from.| trusted-ca| |
-|IMAGE_URL| Fully qualified image name to show SBOM for.| None| '$(tasks.build-container.results.IMAGE_URL)'|
+|IMAGE_URL| Fully qualified image name to show SBOM for.| None| '$(tasks.build-image-index.results.IMAGE_URL)'|
 |PLATFORM| Specific architecture to display the SBOM for. An example arch would be "linux/amd64". If IMAGE_URL refers to a multi-arch image and this parameter is empty, the task will default to use "linux/amd64".| linux/amd64| |
 ### source-build-oci-ta:0.1 task parameters
 |name|description|default value|already set by|
@@ -169,16 +180,22 @@
 |---|---|---|
 |CHAINS-GIT_COMMIT| |$(tasks.clone-repository.results.commit)|
 |CHAINS-GIT_URL| |$(tasks.clone-repository.results.url)|
-|IMAGE_DIGEST| |$(tasks.build-container.results.IMAGE_DIGEST)|
-|IMAGE_URL| |$(tasks.build-container.results.IMAGE_URL)|
+|IMAGE_DIGEST| |$(tasks.build-image-index.results.IMAGE_DIGEST)|
+|IMAGE_URL| |$(tasks.build-image-index.results.IMAGE_URL)|
 |JAVA_COMMUNITY_DEPENDENCIES| |$(tasks.build-container.results.JAVA_COMMUNITY_DEPENDENCIES)|
 ## Available results from tasks
-### buildah-oci-ta:0.2 task results
+### build-image-index:0.1 task results
 |name|description|used in params (taskname:taskrefversion:taskparam)
 |---|---|---|
+|IMAGES| List of all referenced image manifests| |
 |IMAGE_DIGEST| Digest of the image just built| deprecated-base-image-check:0.4:IMAGE_DIGEST ; clair-scan:0.1:image-digest ; sast-snyk-check:0.2:image-digest ; clamav-scan:0.1:image-digest ; sbom-json-check:0.1:IMAGE_DIGEST ; push-dockerfile:0.1:IMAGE_DIGEST|
+|IMAGE_URL| Image repository and tag where the built image was pushed| show-sbom:0.1:IMAGE_URL ; deprecated-base-image-check:0.4:IMAGE_URL ; clair-scan:0.1:image-url ; ecosystem-cert-preflight-checks:0.1:image-url ; sast-snyk-check:0.2:image-url ; clamav-scan:0.1:image-url ; sbom-json-check:0.1:IMAGE_URL ; apply-tags:0.1:IMAGE ; push-dockerfile:0.1:IMAGE|
+### buildah-oci-ta:0.2 task results
+|name|description|used in params (taskname:taskrefversion:taskparam)
+|---|---|---|
+|IMAGE_DIGEST| Digest of the image just built| |
 |IMAGE_REF| Image reference of the built image| |
-|IMAGE_URL| Image repository where the built image was pushed| show-sbom:0.1:IMAGE_URL ; deprecated-base-image-check:0.4:IMAGE_URL ; clair-scan:0.1:image-url ; ecosystem-cert-preflight-checks:0.1:image-url ; sast-snyk-check:0.2:image-url ; clamav-scan:0.1:image-url ; sbom-json-check:0.1:IMAGE_URL ; apply-tags:0.1:IMAGE ; push-dockerfile:0.1:IMAGE|
+|IMAGE_URL| Image repository and tag where the built image was pushed| build-image-index:0.1:IMAGES|
 |JAVA_COMMUNITY_DEPENDENCIES| The Java dependencies that came from community sources such as Maven central.| |
 |SBOM_BLOB_URL| Reference of SBOM blob digest to enable digest-based verification from provenance| |
 |SBOM_JAVA_COMPONENTS_COUNT| The counting of Java components by publisher in JSON format| |
@@ -206,7 +223,7 @@
 |name|description|used in params (taskname:taskrefversion:taskparam)
 |---|---|---|
 |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| prefetch-dependencies:0.1:SOURCE_ARTIFACT|
-|commit| The precise commit SHA that was fetched by this Task.| build-container:0.2:COMMIT_SHA|
+|commit| The precise commit SHA that was fetched by this Task.| build-container:0.2:COMMIT_SHA ; build-image-index:0.1:COMMIT_SHA|
 |commit-timestamp| The commit timestamp of the checkout| |
 |url| The precise URL that was fetched by this Task.| |
 ### init:0.2 task results
@@ -216,7 +233,7 @@
 ### prefetch-dependencies-oci-ta:0.1 task results
 |name|description|used in params (taskname:taskrefversion:taskparam)
 |---|---|---|
-|CACHI2_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the prefetched dependencies.| build-container:0.2:CACHI2_ARTIFACT ; build-source-image:0.1:CACHI2_ARTIFACT ; sast-snyk-check:0.2:CACHI2_ARTIFACT|
+|CACHI2_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the prefetched dependencies.| build-container:0.2:CACHI2_ARTIFACT ; build-source-image:0.1:CACHI2_ARTIFACT ; ecosystem-cert-preflight-checks:0.1:CACHI2_ARTIFACT|
 |SOURCE_ARTIFACT| The Trusted Artifact URI pointing to the artifact with the application source code.| build-container:0.2:SOURCE_ARTIFACT ; build-source-image:0.1:SOURCE_ARTIFACT ; sast-snyk-check:0.2:SOURCE_ARTIFACT ; push-dockerfile:0.1:SOURCE_ARTIFACT|
 ### push-dockerfile-oci-ta:0.1 task results
 |name|description|used in params (taskname:taskrefversion:taskparam)

diff --git a/pipelines/docker-build-oci-ta/patch.yaml b/pipelines/docker-build-oci-ta/patch.yaml
@@ -12,19 +12,20 @@
   path: /spec/workspaces/0
 # Order of Tasks from the base docker-build Pipeline:
 # $ kustomize build pipelines/docker-build | yq .spec.tasks.[].name | nl -v 0
-#  0  init
-#  1  clone-repository
-#  2  prefetch-dependencies
-#  3  build-container
-#  4  build-source-image
-#  5  deprecated-base-image-check
-#  6  clair-scan
-#  7  ecosystem-cert-preflight-checks
-#  8  sast-snyk-check
-#  9  clamav-scan
-# 10  sbom-json-check
-# 11  apply-tags
-# 12  push-dockerfile
+#   0  init
+#   1  clone-repository
+#   2  prefetch-dependencies
+#   3  build-container
+#   4  build-image-index
+#   5  build-source-image
+#   6  deprecated-base-image-check
+#   7  clair-scan
+#   8  ecosystem-cert-preflight-checks
+#   9  sast-snyk-check
+#  10  clamav-scan
+#  11  sbom-json-check
+#  12  apply-tags
+#  13  push-dockerfile
 
 # clone-repository Task
 - op: replace
@@ -86,29 +87,29 @@
 
 # build-source-image
 - op: replace
-  path: /spec/tasks/4/taskRef/name
+  path: /spec/tasks/5/taskRef/name
   value: source-build-oci-ta
 - op: add
-  path: /spec/tasks/4/params/-
+  path: /spec/tasks/5/params/-
   value:
     name: SOURCE_ARTIFACT
     value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
 - op: add
-  path: /spec/tasks/4/params/-
+  path: /spec/tasks/5/params/-
   value:
     name: CACHI2_ARTIFACT
     value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
 - op: remove
-  path: /spec/tasks/4/workspaces/0
+  path: /spec/tasks/5/workspaces/0
 
 # sast-snyk-check
 - op: replace
-  path: /spec/tasks/8/taskRef/name
+  path: /spec/tasks/9/taskRef/name
   value: sast-snyk-check-oci-ta
 - op: add
   # In the docker-build Pipeline, the snyk Task does not receive any parameters, so we cannot
   # append to it.
-  path: /spec/tasks/8/params/-
+  path: /spec/tasks/9/params/-
   value:
     name: SOURCE_ARTIFACT
     value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
@@ -118,19 +119,19 @@
     name: CACHI2_ARTIFACT
     value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT)
 - op: remove
-  path: /spec/tasks/8/workspaces/0
+  path: /spec/tasks/9/workspaces/0
 
 # push-dockerfile
 - op: replace
-  path: /spec/tasks/12/taskRef/name
+  path: /spec/tasks/13/taskRef/name
   value: push-dockerfile-oci-ta
 - op: add
-  path: /spec/tasks/12/params/-
+  path: /spec/tasks/13/params/-
   value:
     name: SOURCE_ARTIFACT
     value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT)
 - op: remove
-  path: /spec/tasks/12/workspaces/0
+  path: /spec/tasks/13/workspaces/0
 
 # Order of finally Tasks from the base docker-build Pipeline:
 # $ kustomize build pipelines/docker-build | yq .spec.finally.[].name | nl -v 0