From 6d1fdbbe5e1c046cefeac2cd3f51c3bd5581a3f0 Mon Sep 17 00:00:00 2001 From: jstuart Date: Thu, 11 Jul 2024 13:06:11 -0500 Subject: [PATCH] Tag bundles after acceptable bundles list pushed This change addresses the issue where the acceptable bundles list does not contain new bundle updates. The issue is addressed by tagging new bundles after the acceptable bundle is pushed. This will ensure renovate does not push updates until after the acceptable bundles list has been updated. https://issues.redhat.com/browse/EC-627 --- .tekton/push.yaml | 12 ++- .../build-acceptable-bundles.sh | 30 +++++- hack/build-and-push.sh | 13 +-- hack/push-and-tag.sh | 17 ++++ spec/hack/bundles_spec.sh | 99 +++++++++++++++++++ 5 files changed, 157 insertions(+), 14 deletions(-) rename {.tekton/scripts => hack}/build-acceptable-bundles.sh (69%) create mode 100755 hack/push-and-tag.sh create mode 100644 spec/hack/bundles_spec.sh diff --git a/.tekton/push.yaml b/.tekton/push.yaml index 0fd3e301dc..c8c517106e 100644 --- a/.tekton/push.yaml +++ b/.tekton/push.yaml @@ -125,6 +125,8 @@ spec: value: "$(params.revision)" - name: GIT_URL value: "$(params.git-url)" + - name: OUTPUT_TASK_BUNDLE_LIST + value: $(workspaces.source.path)/full-bundle-list script: | #!/bin/bash set -euo pipefail @@ -138,7 +140,7 @@ spec: [[ -f "$f" ]] && list+=("$f") done - .tekton/scripts/build-acceptable-bundles.sh "${list[@]}" + hack/build-acceptable-bundles.sh "${list[@]}" echo -n "${DATA_BUNDLE_TAG}" > acceptable_bundle_tag args: @@ -168,6 +170,14 @@ spec: - mountPath: /root/.docker/config.json subPath: .dockerconfigjson name: quay-secret + - name: tag-bundles-konflux-ci + image: quay.io/konflux-ci/appstudio-utils:{{ revision }} + workingDir: $(workspaces.source.path)/source + command: ["./hack/push-and-tag.sh"] + env: + - name: OUTPUT_TASK_BUNDLE_LIST + value: $(workspaces.source.path)/full-bundle-list + volumes: - name: quay-secret secret: diff --git a/.tekton/scripts/build-acceptable-bundles.sh b/hack/build-acceptable-bundles.sh similarity index 69% rename from .tekton/scripts/build-acceptable-bundles.sh rename to hack/build-acceptable-bundles.sh index 6a2b9599a3..9d9d5044da 100755 --- a/.tekton/scripts/build-acceptable-bundles.sh +++ b/hack/build-acceptable-bundles.sh @@ -3,10 +3,27 @@ set -o errexit set -o nounset set -o pipefail + +# Function to remove the sha and digest from the image name +# from: quay.io/konflux-ci/task1:0.1-1234@sha256:5678 to quay.io/konflux-ci/task1:0.1 +strip_image_tag() { + sed 's/\(:[^-]*\).*/\1/' <<< "$1" +} + # helps with debugging DATA_BUNDLE_REPO="${DATA_BUNDLE_REPO:-quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles}" mapfile -t BUNDLES < <(cat "$@") +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# File containing the list of images +OUTPUT_TASK_BUNDLE_LIST="${OUTPUT_TASK_BUNDLE_LIST-${SCRIPTDIR}/../task-bundle-list}" +for i in "${!BUNDLES[@]}"; do + original_line="${BUNDLES[$i]}" + modified_line=$(strip_image_tag "$original_line") + BUNDLES[$i]="$modified_line" + echo "$original_line,$modified_line" >> "$OUTPUT_TASK_BUNDLE_LIST" +done + # store a list of changed task files task_records=() # loop over all changed files @@ -24,11 +41,15 @@ for path in $(git log -m -1 --name-only --pretty="format:" "${REVISION}"); do fi done -echo "Tasks to be added:" -printf '%s\n' "${task_records[@]}" +if [ ${#task_records[@]} -gt 0 ]; then + echo "Tasks to be added:" + printf '%s\n' "${task_records[@]}" +fi -echo "Bundles to be added:" -printf '%s\n' "${BUNDLES[@]}" +if [ ${#BUNDLES[@]} -gt 0 ]; then + echo "Bundles to be added:" + printf '%s\n' "${BUNDLES[@]}" +fi # The OPA data bundle is tagged with the current timestamp. This has two main # advantages. First, it prevents the image from accidentally not having any tags, @@ -44,7 +65,6 @@ fi mapfile -t -d ' ' BUNDLES_PARAM < <(printf -- '--bundle=%s ' "${BUNDLES[@]}") PARAMS=("${TASK_PARAM[@]}" "${BUNDLES_PARAM[@]}") - ec track bundle --debug \ --input "oci:${DATA_BUNDLE_REPO}:latest" \ --output "oci:${DATA_BUNDLE_REPO}:${DATA_BUNDLE_TAG}" \ diff --git a/hack/build-and-push.sh b/hack/build-and-push.sh index 5703435ef3..6bdda849cc 100755 --- a/hack/build-and-push.sh +++ b/hack/build-and-push.sh @@ -20,6 +20,7 @@ tkn_bundle_push() { local retry=0 local -r interval=${RETRY_INTERVAL:-5} local -r max_retries=5 + while true; do tkn bundle push "$@" && break status=$? @@ -42,7 +43,6 @@ tkn_bundle_push() { function save_ref() { local output output="$(< /dev/stdin)" - echo "${output}" local digest digest="$(echo "${output}" | grep -Po '@\K(sha256:[a-f0-9]*)')" @@ -51,7 +51,6 @@ function save_ref() { local refFile refFile="$2" echo "${tagRef}@${digest}" >> "${refFile}" - echo "Created:" echo "${tagRef}@${digest}" } @@ -114,20 +113,18 @@ do echo Unknown task in "$task_dir" continue fi + + # the task files have been copied to the $prepared_task_file location at this point repository=${TEST_REPO_NAME:-task-${task_name}} tag=${TEST_REPO_NAME:+${task_name}-}${task_version} - task_bundle=quay.io/$QUAY_NAMESPACE/${repository}:${tag} + task_bundle=quay.io/$QUAY_NAMESPACE/${repository}:${tag}-${task_file_sha} - if digest=$(skopeo inspect --no-tags --format='{{.Digest}}' docker://"${task_bundle}-${task_file_sha}" 2>/dev/null); then + if digest=$(skopeo inspect --no-tags --format='{{.Digest}}' docker://"${task_bundle}" 2>/dev/null); then task_bundle_with_digest=${task_bundle}@${digest} else output=$(tkn_bundle_push -f "$prepared_task_file" "$task_bundle" | save_ref "$task_bundle" "$OUTPUT_TASK_BUNDLE_LIST") echo "$output" task_bundle_with_digest="${output##*$'\n'}" - - # copy task to new tag pointing to commit where the file was changed lastly, so that image persists - # even when original tag is updated - skopeo copy "docker://${task_bundle}" "docker://${task_bundle}-${task_file_sha}" fi # version placeholder is removed naturally by the substitution. real_task_name=$(yq e '.metadata.name' "$prepared_task_file") diff --git a/hack/push-and-tag.sh b/hack/push-and-tag.sh new file mode 100755 index 0000000000..eaa695cb9d --- /dev/null +++ b/hack/push-and-tag.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# File containing the list of images +OUTPUT_TASK_BUNDLE_LIST="${OUTPUT_TASK_BUNDLE_LIST-${SCRIPTDIR}/../task-bundle-list}" + +# Read the file and process each line +while IFS=, read -r original_image new_image; do + # Remove the quotes from the strings + original_image=$(echo "$original_image" | tr -d '"' | xargs) + new_image=$(echo "$new_image" | tr -d '"' | xargs) + + # Run the skopeo copy command + echo "Copying from $original_image to $new_image" + skopeo copy "docker://$original_image" "docker://$new_image" + +done < "$OUTPUT_TASK_BUNDLE_LIST" diff --git a/spec/hack/bundles_spec.sh b/spec/hack/bundles_spec.sh new file mode 100644 index 0000000000..f699322b0f --- /dev/null +++ b/spec/hack/bundles_spec.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# this spec file tests creating Tekton bundles and the acceptable bundles list + +set -o errexit +set -o pipefail +set -o nounset + +eval "$(shellspec - -c) exit 1" + +check_tkn_push_url() { + while read -r line; do + if [[ "$line" == quay.io/* ]] && [[ ! "$line" =~ ^quay\.io/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+:[0-9a-zA-Z\.-]+@sha256:[a-fA-F0-9]+$ ]]; then + return 1 + fi + done +} + +create_test_tasks() { + mkdir -p tmp/task1/0.1 + mkdir -p tmp/task2/0.1 + touch tmp/task1/0.1/task1.yaml + touch tmp/task2/0.1/task2.yaml +} + +cleanup_test_data() { + rm -rf tmp + rm -f test-task-bundle-list + rm -f test-task-bundle-list.csv +} + +Describe "Creating new acceptable bundles" + AfterAll 'cleanup_test_data' + + Mock skopeo + # Make the skopeo inspect command fail + if [ "$1" = "inspect" ]; then + return 1 + fi + End + + Mock tkn + echo "${5}@sha256:5678" + End + + Mock sha256sum + echo "1234" + End + + Mock ec + End + + Mock find + echo "tmp/task1/0.1/\ntmp/task2/0.1/" + End + + It "builds bundles with the correct sha as the tag" + create_test_tasks + Mock git + echo "1234" + End + export OUTPUT_TASK_BUNDLE_LIST=test-task-bundle-list + export QUAY_NAMESPACE=konflux-ci + export SKIP_BUILD=true + + When call "hack/build-and-push.sh" + The status should be success + # each task and pipeline bundle ends with file checksum @ digest + The output should satisfy check_tkn_push_url + End + + It 'processes the bundles and generates the correct output file' + # this is only used for the task_records var which is unused in this test + Mock git + echo "task/task1/task1.yaml\ntask/task2/task2.yaml" + End + + export OUTPUT_TASK_BUNDLE_LIST="test-task-bundle-list.csv" + export GIT_URL="https://my-url/org/repo" + export REVISION="abcd1234" + export DATA_BUNDLE_REPO="quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles" + + When call hack/build-acceptable-bundles.sh "test-task-bundle-list" + The status should be success + The path test-task-bundle-list.csv should be file + The contents of file "test-task-bundle-list.csv" should equal "quay.io/konflux-ci/task-task1:0.1-1234@sha256:5678,quay.io/konflux-ci/task-task1:0.1 +quay.io/konflux-ci/task-task2:0.1-1234@sha256:5678,quay.io/konflux-ci/task-task2:0.1" + The stdout should include "Bundles to be added:" + The stdout should include "quay.io/konflux-ci/task-task1:0.1" + The stdout should include "quay.io/konflux-ci/task-task2:0.1" + End + + It "copies to the right image locations" + export OUTPUT_TASK_BUNDLE_LIST="test-task-bundle-list.csv" + When call "hack/push-and-tag.sh" + The status should be success + The output should include "Copying from quay.io/konflux-ci/task-task1:0.1-1234@sha256:5678 to quay.io/konflux-ci/task-task1:0.1" + The output should include "Copying from quay.io/konflux-ci/task-task2:0.1-1234@sha256:5678 to quay.io/konflux-ci/task-task2:0.1" + End +End