Skip to content

Commit

Permalink
Tag bundles after acceptable bundles list pushed
Browse files Browse the repository at this point in the history
This change addresses the issue where the
acceptable bundles list does not contain new
bundle updates. The issue is addressed by
tagging new bundles after the acceptable bundle
is pushed. This will ensure renovate
does not push updates until after the acceptable
bundles list has been updated.

https://issues.redhat.com/browse/EC-627
  • Loading branch information
joejstuart committed Jul 22, 2024
1 parent 3118a13 commit 6d1fdbb
Show file tree
Hide file tree
Showing 5 changed files with 157 additions and 14 deletions.
12 changes: 11 additions & 1 deletion .tekton/push.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ spec:
value: "$(params.revision)"
- name: GIT_URL
value: "$(params.git-url)"
- name: OUTPUT_TASK_BUNDLE_LIST
value: $(workspaces.source.path)/full-bundle-list
script: |
#!/bin/bash
set -euo pipefail
Expand All @@ -138,7 +140,7 @@ spec:
[[ -f "$f" ]] && list+=("$f")
done
.tekton/scripts/build-acceptable-bundles.sh "${list[@]}"
hack/build-acceptable-bundles.sh "${list[@]}"
echo -n "${DATA_BUNDLE_TAG}" > acceptable_bundle_tag
args:
Expand Down Expand Up @@ -168,6 +170,14 @@ spec:
- mountPath: /root/.docker/config.json
subPath: .dockerconfigjson
name: quay-secret
- name: tag-bundles-konflux-ci
image: quay.io/konflux-ci/appstudio-utils:{{ revision }}
workingDir: $(workspaces.source.path)/source
command: ["./hack/push-and-tag.sh"]
env:
- name: OUTPUT_TASK_BUNDLE_LIST
value: $(workspaces.source.path)/full-bundle-list

volumes:
- name: quay-secret
secret:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,27 @@ set -o errexit
set -o nounset
set -o pipefail


# Function to remove the sha and digest from the image name
# from: quay.io/konflux-ci/task1:0.1-1234@sha256:5678 to quay.io/konflux-ci/task1:0.1
strip_image_tag() {
sed 's/\(:[^-]*\).*/\1/' <<< "$1"
}

# helps with debugging
DATA_BUNDLE_REPO="${DATA_BUNDLE_REPO:-quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles}"
mapfile -t BUNDLES < <(cat "$@")

SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
# File containing the list of images
OUTPUT_TASK_BUNDLE_LIST="${OUTPUT_TASK_BUNDLE_LIST-${SCRIPTDIR}/../task-bundle-list}"
for i in "${!BUNDLES[@]}"; do
original_line="${BUNDLES[$i]}"
modified_line=$(strip_image_tag "$original_line")
BUNDLES[$i]="$modified_line"
echo "$original_line,$modified_line" >> "$OUTPUT_TASK_BUNDLE_LIST"
done

# store a list of changed task files
task_records=()
# loop over all changed files
Expand All @@ -24,11 +41,15 @@ for path in $(git log -m -1 --name-only --pretty="format:" "${REVISION}"); do
fi
done

echo "Tasks to be added:"
printf '%s\n' "${task_records[@]}"
if [ ${#task_records[@]} -gt 0 ]; then
echo "Tasks to be added:"
printf '%s\n' "${task_records[@]}"
fi

echo "Bundles to be added:"
printf '%s\n' "${BUNDLES[@]}"
if [ ${#BUNDLES[@]} -gt 0 ]; then
echo "Bundles to be added:"
printf '%s\n' "${BUNDLES[@]}"
fi

# The OPA data bundle is tagged with the current timestamp. This has two main
# advantages. First, it prevents the image from accidentally not having any tags,
Expand All @@ -44,7 +65,6 @@ fi
mapfile -t -d ' ' BUNDLES_PARAM < <(printf -- '--bundle=%s ' "${BUNDLES[@]}")

PARAMS=("${TASK_PARAM[@]}" "${BUNDLES_PARAM[@]}")

ec track bundle --debug \
--input "oci:${DATA_BUNDLE_REPO}:latest" \
--output "oci:${DATA_BUNDLE_REPO}:${DATA_BUNDLE_TAG}" \
Expand Down
13 changes: 5 additions & 8 deletions hack/build-and-push.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ tkn_bundle_push() {
local retry=0
local -r interval=${RETRY_INTERVAL:-5}
local -r max_retries=5

while true; do
tkn bundle push "$@" && break
status=$?
Expand All @@ -42,7 +43,6 @@ tkn_bundle_push() {
function save_ref() {
local output
output="$(< /dev/stdin)"
echo "${output}"
local digest
digest="$(echo "${output}" | grep -Po '@\K(sha256:[a-f0-9]*)')"

Expand All @@ -51,7 +51,6 @@ function save_ref() {
local refFile
refFile="$2"
echo "${tagRef}@${digest}" >> "${refFile}"
echo "Created:"
echo "${tagRef}@${digest}"
}

Expand Down Expand Up @@ -114,20 +113,18 @@ do
echo Unknown task in "$task_dir"
continue
fi

# the task files have been copied to the $prepared_task_file location at this point
repository=${TEST_REPO_NAME:-task-${task_name}}
tag=${TEST_REPO_NAME:+${task_name}-}${task_version}
task_bundle=quay.io/$QUAY_NAMESPACE/${repository}:${tag}
task_bundle=quay.io/$QUAY_NAMESPACE/${repository}:${tag}-${task_file_sha}

if digest=$(skopeo inspect --no-tags --format='{{.Digest}}' docker://"${task_bundle}-${task_file_sha}" 2>/dev/null); then
if digest=$(skopeo inspect --no-tags --format='{{.Digest}}' docker://"${task_bundle}" 2>/dev/null); then
task_bundle_with_digest=${task_bundle}@${digest}
else
output=$(tkn_bundle_push -f "$prepared_task_file" "$task_bundle" | save_ref "$task_bundle" "$OUTPUT_TASK_BUNDLE_LIST")
echo "$output"
task_bundle_with_digest="${output##*$'\n'}"

# copy task to new tag pointing to commit where the file was changed lastly, so that image persists
# even when original tag is updated
skopeo copy "docker://${task_bundle}" "docker://${task_bundle}-${task_file_sha}"
fi
# version placeholder is removed naturally by the substitution.
real_task_name=$(yq e '.metadata.name' "$prepared_task_file")
Expand Down
17 changes: 17 additions & 0 deletions hack/push-and-tag.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
# File containing the list of images
OUTPUT_TASK_BUNDLE_LIST="${OUTPUT_TASK_BUNDLE_LIST-${SCRIPTDIR}/../task-bundle-list}"

# Read the file and process each line
while IFS=, read -r original_image new_image; do
# Remove the quotes from the strings
original_image=$(echo "$original_image" | tr -d '"' | xargs)
new_image=$(echo "$new_image" | tr -d '"' | xargs)

# Run the skopeo copy command
echo "Copying from $original_image to $new_image"
skopeo copy "docker://$original_image" "docker://$new_image"

done < "$OUTPUT_TASK_BUNDLE_LIST"
99 changes: 99 additions & 0 deletions spec/hack/bundles_spec.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
#!/bin/bash
# this spec file tests creating Tekton bundles and the acceptable bundles list

set -o errexit
set -o pipefail
set -o nounset

eval "$(shellspec - -c) exit 1"

check_tkn_push_url() {
while read -r line; do
if [[ "$line" == quay.io/* ]] && [[ ! "$line" =~ ^quay\.io/[a-zA-Z0-9_-]+/[a-zA-Z0-9_-]+:[0-9a-zA-Z\.-]+@sha256:[a-fA-F0-9]+$ ]]; then
return 1
fi
done
}

create_test_tasks() {
mkdir -p tmp/task1/0.1
mkdir -p tmp/task2/0.1
touch tmp/task1/0.1/task1.yaml
touch tmp/task2/0.1/task2.yaml
}

cleanup_test_data() {
rm -rf tmp
rm -f test-task-bundle-list
rm -f test-task-bundle-list.csv
}

Describe "Creating new acceptable bundles"
AfterAll 'cleanup_test_data'

Mock skopeo
# Make the skopeo inspect command fail
if [ "$1" = "inspect" ]; then
return 1
fi
End

Mock tkn
echo "${5}@sha256:5678"
End

Mock sha256sum
echo "1234"
End

Mock ec
End

Mock find
echo "tmp/task1/0.1/\ntmp/task2/0.1/"
End

It "builds bundles with the correct sha as the tag"
create_test_tasks
Mock git
echo "1234"
End
export OUTPUT_TASK_BUNDLE_LIST=test-task-bundle-list
export QUAY_NAMESPACE=konflux-ci
export SKIP_BUILD=true

When call "hack/build-and-push.sh"
The status should be success
# each task and pipeline bundle ends with file checksum @ digest
The output should satisfy check_tkn_push_url
End

It 'processes the bundles and generates the correct output file'
# this is only used for the task_records var which is unused in this test
Mock git
echo "task/task1/task1.yaml\ntask/task2/task2.yaml"
End

export OUTPUT_TASK_BUNDLE_LIST="test-task-bundle-list.csv"
export GIT_URL="https://my-url/org/repo"
export REVISION="abcd1234"
export DATA_BUNDLE_REPO="quay.io/konflux-ci/tekton-catalog/data-acceptable-bundles"

When call hack/build-acceptable-bundles.sh "test-task-bundle-list"
The status should be success
The path test-task-bundle-list.csv should be file
The contents of file "test-task-bundle-list.csv" should equal "quay.io/konflux-ci/task-task1:0.1-1234@sha256:5678,quay.io/konflux-ci/task-task1:0.1
quay.io/konflux-ci/task-task2:0.1-1234@sha256:5678,quay.io/konflux-ci/task-task2:0.1"
The stdout should include "Bundles to be added:"
The stdout should include "quay.io/konflux-ci/task-task1:0.1"
The stdout should include "quay.io/konflux-ci/task-task2:0.1"
End

It "copies to the right image locations"
export OUTPUT_TASK_BUNDLE_LIST="test-task-bundle-list.csv"
When call "hack/push-and-tag.sh"
The status should be success
The output should include "Copying from quay.io/konflux-ci/task-task1:0.1-1234@sha256:5678 to quay.io/konflux-ci/task-task1:0.1"
The output should include "Copying from quay.io/konflux-ci/task-task2:0.1-1234@sha256:5678 to quay.io/konflux-ci/task-task2:0.1"
End
End

0 comments on commit 6d1fdbb

Please sign in to comment.