diff --git a/.tekton/push.yaml b/.tekton/push.yaml index 0fd3e301dc..c8c517106e 100644 --- a/.tekton/push.yaml +++ b/.tekton/push.yaml @@ -125,6 +125,8 @@ spec: value: "$(params.revision)" - name: GIT_URL value: "$(params.git-url)" + - name: OUTPUT_TASK_BUNDLE_LIST + value: $(workspaces.source.path)/full-bundle-list script: | #!/bin/bash set -euo pipefail @@ -138,7 +140,7 @@ spec: [[ -f "$f" ]] && list+=("$f") done - .tekton/scripts/build-acceptable-bundles.sh "${list[@]}" + hack/build-acceptable-bundles.sh "${list[@]}" echo -n "${DATA_BUNDLE_TAG}" > acceptable_bundle_tag args: @@ -168,6 +170,14 @@ spec: - mountPath: /root/.docker/config.json subPath: .dockerconfigjson name: quay-secret + - name: tag-bundles-konflux-ci + image: quay.io/konflux-ci/appstudio-utils:{{ revision }} + workingDir: $(workspaces.source.path)/source + command: ["./hack/push-and-tag.sh"] + env: + - name: OUTPUT_TASK_BUNDLE_LIST + value: $(workspaces.source.path)/full-bundle-list + volumes: - name: quay-secret secret: diff --git a/.tekton/scripts/build-acceptable-bundles.sh b/hack/build-acceptable-bundles.sh similarity index 69% rename from .tekton/scripts/build-acceptable-bundles.sh rename to hack/build-acceptable-bundles.sh index 6a2b9599a3..9d9d5044da 100755 --- a/.tekton/scripts/build-acceptable-bundles.sh +++ b/hack/build-acceptable-bundles.sh @@ -3,10 +3,27 @@ set -o errexit set -o nounset set -o pipefail + +# Function to remove the sha and digest from the image name +# from: quay.io/konflux-ci/task1:0.1-1234@sha256:5678 to quay.io/konflux-ci/task1:0.1 +strip_image_tag() { + sed 's/\(:[^-]*\).*/\1/' <<< "$1" +} + # helps with debugging DATA_BUNDLE_REPO="${DATA_BUNDLE_REPO:-quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles}" mapfile -t BUNDLES < <(cat "$@") +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# File containing the list of images +OUTPUT_TASK_BUNDLE_LIST="${OUTPUT_TASK_BUNDLE_LIST-${SCRIPTDIR}/../task-bundle-list}" +for i in "${!BUNDLES[@]}"; do + original_line="${BUNDLES[$i]}" + modified_line=$(strip_image_tag "$original_line") + BUNDLES[$i]="$modified_line" + echo "$original_line,$modified_line" >> "$OUTPUT_TASK_BUNDLE_LIST" +done + # store a list of changed task files task_records=() # loop over all changed files @@ -24,11 +41,15 @@ for path in $(git log -m -1 --name-only --pretty="format:" "${REVISION}"); do fi done -echo "Tasks to be added:" -printf '%s\n' "${task_records[@]}" +if [ ${#task_records[@]} -gt 0 ]; then + echo "Tasks to be added:" + printf '%s\n' "${task_records[@]}" +fi -echo "Bundles to be added:" -printf '%s\n' "${BUNDLES[@]}" +if [ ${#BUNDLES[@]} -gt 0 ]; then + echo "Bundles to be added:" + printf '%s\n' "${BUNDLES[@]}" +fi # The OPA data bundle is tagged with the current timestamp. This has two main # advantages. First, it prevents the image from accidentally not having any tags, @@ -44,7 +65,6 @@ fi mapfile -t -d ' ' BUNDLES_PARAM < <(printf -- '--bundle=%s ' "${BUNDLES[@]}") PARAMS=("${TASK_PARAM[@]}" "${BUNDLES_PARAM[@]}") - ec track bundle --debug \ --input "oci:${DATA_BUNDLE_REPO}:latest" \ --output "oci:${DATA_BUNDLE_REPO}:${DATA_BUNDLE_TAG}" \ diff --git a/hack/build-and-push.sh b/hack/build-and-push.sh index 5703435ef3..6bdda849cc 100755 --- a/hack/build-and-push.sh +++ b/hack/build-and-push.sh @@ -20,6 +20,7 @@ tkn_bundle_push() { local retry=0 local -r interval=${RETRY_INTERVAL:-5} local -r max_retries=5 + while true; do tkn bundle push "$@" && break status=$? @@ -42,7 +43,6 @@ tkn_bundle_push() { function save_ref() { local output output="$(< /dev/stdin)" - echo "${output}" local digest digest="$(echo "${output}" | grep -Po '@\K(sha256:[a-f0-9]*)')" @@ -51,7 +51,6 @@ function save_ref() { local refFile refFile="$2" echo "${tagRef}@${digest}" >> "${refFile}" - echo "Created:" echo "${tagRef}@${digest}" } @@ -114,20 +113,18 @@ do echo Unknown task in "$task_dir" continue fi + + # the task files have been copied to the $prepared_task_file location at this point repository=${TEST_REPO_NAME:-task-${task_name}} tag=${TEST_REPO_NAME:+${task_name}-}${task_version} - task_bundle=quay.io/$QUAY_NAMESPACE/${repository}:${tag} + task_bundle=quay.io/$QUAY_NAMESPACE/${repository}:${tag}-${task_file_sha} - if digest=$(skopeo inspect --no-tags --format='{{.Digest}}' docker://"${task_bundle}-${task_file_sha}" 2>/dev/null); then + if digest=$(skopeo inspect --no-tags --format='{{.Digest}}' docker://"${task_bundle}" 2>/dev/null); then task_bundle_with_digest=${task_bundle}@${digest} else output=$(tkn_bundle_push -f "$prepared_task_file" "$task_bundle" | save_ref "$task_bundle" "$OUTPUT_TASK_BUNDLE_LIST") echo "$output" task_bundle_with_digest="${output##*$'\n'}" - - # copy task to new tag pointing to commit where the file was changed lastly, so that image persists - # even when original tag is updated - skopeo copy "docker://${task_bundle}" "docker://${task_bundle}-${task_file_sha}" fi # version placeholder is removed naturally by the substitution. real_task_name=$(yq e '.metadata.name' "$prepared_task_file") diff --git a/hack/push-and-tag.sh b/hack/push-and-tag.sh new file mode 100755 index 0000000000..eaa695cb9d --- /dev/null +++ b/hack/push-and-tag.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +# File containing the list of images +OUTPUT_TASK_BUNDLE_LIST="${OUTPUT_TASK_BUNDLE_LIST-${SCRIPTDIR}/../task-bundle-list}" + +# Read the file and process each line +while IFS=, read -r original_image new_image; do + # Remove the quotes from the strings + original_image=$(echo "$original_image" | tr -d '"' | xargs) + new_image=$(echo "$new_image" | tr -d '"' | xargs) + + # Run the skopeo copy command + echo "Copying from $original_image to $new_image" + skopeo copy "docker://$original_image" "docker://$new_image" + +done < "$OUTPUT_TASK_BUNDLE_LIST" diff --git a/spec/hack/bundles_spec.sh b/spec/hack/bundles_spec.sh new file mode 100644 index 0000000000..f699322b0f --- /dev/null +++ b/spec/hack/bundles_spec.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# this spec file tests creating Tekton bundles and the acceptable bundles list + +set -o errexit +set -o pipefail +set -o nounset + +eval "$(shellspec - -c) exit 1" + +check_tkn_push_url() { + while read -r line; do + if [[ "$line" == quay.io/* ]] && [[ ! "$line" =~ ^quay\.io/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+:[0-9a-zA-Z\.-]+@sha256:[a-fA-F0-9]+$ ]]; then + return 1 + fi + done +} + +create_test_tasks() { + mkdir -p tmp/task1/0.1 + mkdir -p tmp/task2/0.1 + touch tmp/task1/0.1/task1.yaml + touch tmp/task2/0.1/task2.yaml +} + +cleanup_test_data() { + rm -rf tmp + rm -f test-task-bundle-list + rm -f test-task-bundle-list.csv +} + +Describe "Creating new acceptable bundles" + AfterAll 'cleanup_test_data' + + Mock skopeo + # Make the skopeo inspect command fail + if [ "$1" = "inspect" ]; then + return 1 + fi + End + + Mock tkn + echo "${5}@sha256:5678" + End + + Mock sha256sum + echo "1234" + End + + Mock ec + End + + Mock find + echo "tmp/task1/0.1/\ntmp/task2/0.1/" + End + + It "builds bundles with the correct sha as the tag" + create_test_tasks + Mock git + echo "1234" + End + export OUTPUT_TASK_BUNDLE_LIST=test-task-bundle-list + export QUAY_NAMESPACE=konflux-ci + export SKIP_BUILD=true + + When call "hack/build-and-push.sh" + The status should be success + # each task and pipeline bundle ends with file checksum @ digest + The output should satisfy check_tkn_push_url + End + + It 'processes the bundles and generates the correct output file' + # this is only used for the task_records var which is unused in this test + Mock git + echo "task/task1/task1.yaml\ntask/task2/task2.yaml" + End + + export OUTPUT_TASK_BUNDLE_LIST="test-task-bundle-list.csv" + export GIT_URL="https://my-url/org/repo" + export REVISION="abcd1234" + export DATA_BUNDLE_REPO="quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles" + + When call hack/build-acceptable-bundles.sh "test-task-bundle-list" + The status should be success + The path test-task-bundle-list.csv should be file + The contents of file "test-task-bundle-list.csv" should equal "quay.io/konflux-ci/task-task1:0.1-1234@sha256:5678,quay.io/konflux-ci/task-task1:0.1 +quay.io/konflux-ci/task-task2:0.1-1234@sha256:5678,quay.io/konflux-ci/task-task2:0.1" + The stdout should include "Bundles to be added:" + The stdout should include "quay.io/konflux-ci/task-task1:0.1" + The stdout should include "quay.io/konflux-ci/task-task2:0.1" + End + + It "copies to the right image locations" + export OUTPUT_TASK_BUNDLE_LIST="test-task-bundle-list.csv" + When call "hack/push-and-tag.sh" + The status should be success + The output should include "Copying from quay.io/konflux-ci/task-task1:0.1-1234@sha256:5678 to quay.io/konflux-ci/task-task1:0.1" + The output should include "Copying from quay.io/konflux-ci/task-task2:0.1-1234@sha256:5678 to quay.io/konflux-ci/task-task2:0.1" + End +End