-
Notifications
You must be signed in to change notification settings - Fork 5
261 lines (234 loc) · 13.3 KB
/
trigger-nightly-fbc-build.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
name: Trigger Nightly FBC Build
run-name: Trigger Nightly FBC Build
on:
workflow_dispatch:
push:
paths:
- schedule/catalog-github-trigger.txt
branches:
- 'rhoai-2.1[6-9]+'
permissions:
contents: write
env:
GITHUB_ORG: red-hat-data-services
GITHUB_RKA_ORG: rhoai-rhtap
jobs:
process-fbc:
if: ${{ github.ref_name != 'main' }}
runs-on: ubuntu-latest
container:
image: quay.io/rhoai/rhoai-task-toolset:latest
options: --privileged
steps:
- name: Get Current branch name
shell: bash
run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
id: get_branch
- name: Git checkout RBC main
uses: actions/checkout@v4
with:
ref: main
path: main
sparse-checkout: |
pcc
catalog/${{ steps.get_branch.outputs.branch }}
sparse-checkout-cone-mode: false
- name: Git checkout RBC release
uses: actions/checkout@v4
with:
ref: ${{ steps.get_branch.outputs.branch }}
path: ${{ steps.get_branch.outputs.branch }}
- name: Git checkout utils
uses: actions/checkout@v4
with:
repository: ${{ env.GITHUB_RKA_ORG }}/RHOAI-Konflux-Automation
ref: main
path: utils
- name: Install dependencies
env:
RHOAI_QUAY_RO_USERNAME: ${{ secrets.RHOAI_QUAY_RO_USERNAME }}
RHOAI_QUAY_RO_TOKEN: ${{ secrets.RHOAI_QUAY_RO_TOKEN }}
run: |
os="$(uname -s | tr '[:upper:]' '[:lower:]')"
arch="$(uname -m | sed 's/x86_64/amd64/')"
yq_version="v4.44.3"
yq_filename="yq-$yq_version"
echo "-> Downloading yq" >&2
curl -sSfLo "$yq_filename" "https://github.com/mikefarah/yq/releases/download/$yq_version/yq_${os}_${arch}"
chmod +x $yq_filename
ln -s $yq_filename yq
cp $yq_filename /usr/local/bin/yq
opm_version="v1.47.0"
opm_filename="opm-$opm_version"
echo "-> Downloading opm" >&2
curl -sSfLo "$opm_filename" "https://github.com/operator-framework/operator-registry/releases/download/$opm_version/$os-$arch-opm"
chmod +x "$opm_filename"
ln -fs "$opm_filename" opm
cp "$opm_filename" /usr/local/bin/opm
pip install -r utils/utils/fbc-processor/requirements.txt
microdnf install -y skopeo && \
microdnf clean all && rm -rf /var/cache/dnf/*
skopeo login -u "${RHOAI_QUAY_RO_USERNAME}" -p "${RHOAI_QUAY_RO_TOKEN}" quay.io/rhoai
- name: Check if PCC Cache Valid
id: check-if-pcc-cache-valid
env:
RHOAI_CATALOG_SA_USERNAME: ${{ secrets.RHOAI_CATALOG_SA_USERNAME }}
RHOAI_CATALOG_SA_TOKEN: ${{ secrets.RHOAI_CATALOG_SA_TOKEN }}
run: |
microdnf install -y coreutils-single && \
microdnf clean all && rm -rf /var/cache/dnf/*
PCC_CACHE_VALID=YES
skopeo login registry.redhat.io -u "${RHOAI_CATALOG_SA_USERNAME}" -p "${RHOAI_CATALOG_SA_TOKEN}"
skopeo list-tags docker://registry.redhat.io/rhoai/odh-operator-bundle | jq -r '.Tags | .[] | select(. | startswith ("v"))' | sort > latest_shipped_rhoai_versions.txt
echo "latest_shipped_rhoai_versions = "
cat latest_shipped_rhoai_versions.txt
echo "shipped_rhoai_versions = "
cat main/pcc/shipped_rhoai_versions.txt
#diff=$(cmp --silent latest_shipped_rhoai_versions.txt main/pcc/shipped_rhoai_versions.txt || echo "files are different")
diff=$(python -c 'print(list(set(open("latest_shipped_rhoai_versions.txt").readlines()) - set(open("main/pcc/shipped_rhoai_versions.txt").readlines())).__len__())')
if [[ $diff -gt 0 ]]
then
diff=$(python -c 'print(list(set(open("latest_shipped_rhoai_versions.txt").readlines()) - set(open("main/pcc/shipped_rhoai_versions.txt").readlines())))')
echo "following new versions are shipped - $diff"
cp latest_shipped_rhoai_versions.txt main/pcc/shipped_rhoai_versions.txt
PCC_CACHE_VALID=NO
fi
echo "PCC_CACHE_VALID=${PCC_CACHE_VALID}" >> $GITHUB_OUTPUT
- name: Regenerate PCC Cache
id: regenerate-pcc-cache
if: ${{ steps.check-if-pcc-cache-valid.outputs.PCC_CACHE_VALID == 'NO' }}
env:
BRANCH: ${{ github.event.inputs.release_branch }}
RHOAI_VERSION: ${{ github.event.inputs.rhoai_version }}
run: |
#install opm cli
os="$(uname -s | tr '[:upper:]' '[:lower:]')"
arch="$(uname -m | sed 's/x86_64/amd64/')"
opm_version="v1.47.0"
opm_filename="opm-$opm_version"
echo "-> Downloading opm" >&2
curl -sSfLo "$opm_filename" "https://github.com/operator-framework/operator-registry/releases/download/$opm_version/$os-$arch-opm"
chmod +x "$opm_filename"
ln -fs "$opm_filename" opm
cp "$opm_filename" /usr/local/bin/opm
microdnf install -y findutils && \
microdnf clean all && rm -rf /var/cache/dnf/*
#Declare basic variables
COMPONENT_SUFFIX=${RHOAI_VERSION/./-}
OPERATOR_BUNDLE_COMPONENT_NAME=odh-operator-bundle
#Declare FBC processing variables
BUILD_CONFIG_PATH=${BRANCH}/config/build-config.yaml
PATCH_YAML_PATH=${BRANCH}/catalog/catalog-patch.yaml
CATALOG_GENERATION_REF_OCP_VERSION=v4.17
CSV_META_MIN_OCP_VERSION=417
BUNDLE_OBJECT_CATALOG_YAML_PATH=main/pcc/bundle_object_catalog.yaml
CSV_META_CATALOG_YAML_PATH=main/pcc/csv_meta_catalog.yaml
opm migrate registry.redhat.io/redhat/redhat-operator-index:${CATALOG_GENERATION_REF_OCP_VERSION} ./catalog-migrate
opm alpha convert-template basic catalog-migrate/rhods-operator/catalog.json -o yaml > catalog-template.yaml
opm alpha render-template basic catalog-template.yaml -o yaml > ${BUNDLE_OBJECT_CATALOG_YAML_PATH}
opm alpha render-template basic catalog-template.yaml --migrate-level=bundle-object-to-csv-metadata -o yaml > ${CSV_META_CATALOG_YAML_PATH}
#while IFS= read -r value;
#do
# OPENSHIFT_VERSION=$value
# NUMERIC_OCP_VERSION=${OPENSHIFT_VERSION/v4./4}
# echo "OPENSHIFT_VERSION=$OPENSHIFT_VERSION"
#
# CATALOG_YAML_PATH=${BUNDLE_OBJECT_CATALOG_YAML_PATH}
# CATALOG_DIR=main/pcc/${OPENSHIFT_VERSION}/rhods-operator
# mkdir -p ${CATALOG_DIR}
#
# if [[ $NUMERIC_OCP_VERSION -ge $CSV_META_MIN_OCP_VERSION ]]
# then
# CATALOG_YAML_PATH=${CSV_META_CATALOG_YAML_PATH}
# fi
# cp ${CATALOG_YAML_PATH} ${CATALOG_DIR}/catalog.yaml
#done < <(find main/pcc/ -maxdepth 1 -mindepth 1 -type d -printf '%f\n')
ls -l main/pcc/
- name: Push latest PCC Cache
if: ${{ steps.check-if-pcc-cache-valid.outputs.PCC_CACHE_VALID == 'NO' }}
uses: actions-js/push@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: main
message: "Regeneratd the PCC Cache"
repository: ${{ env.GITHUB_ORG }}/RHOAI-Build-Config
directory: main
author_name: Openshift-AI DevOps
author_email: [email protected]
- name: Process FBC Fragment
env:
BRANCH: ${{ steps.get_branch.outputs.branch }}
OC_TOKEN: ${{ secrets.KONFLUX_INTERNAL_OC_TOKEN }}
CLUSTER: p02
RHOAI_QUAY_RO_USERNAME: ${{ secrets.RHOAI_QUAY_RO_USERNAME }}
RHOAI_QUAY_RO_TOKEN: ${{ secrets.RHOAI_QUAY_RO_TOKEN }}
RHOAI_QUAY_API_TOKEN: ${{ secrets.RHOAI_QUAY_API_TOKEN }}
run: |
#Declare basic variables
BUILD_CONFIG_PATH=${BRANCH}/config/build-config.yaml
RHOAI_VERSION=v${BRANCH/rhoai-/}
COMPONENT_SUFFIX=${RHOAI_VERSION/./-}
#OPERATOR_BUNDLE_COMPONENT_NAME=odh-operator-bundle-${COMPONENT_SUFFIX}
OPERATOR_BUNDLE_COMPONENT_NAME=odh-operator-bundle
PATCH_YAML_PATH=${BRANCH}/catalog/catalog-patch.yaml
OC_TOKEN=$(echo $OC_TOKEN | awk '{$1=$1};1' | tr -d '\n')
OPERATOR_BUNDLE_IMAGE_NAME=RELATED_IMAGE_ODH_OPERATOR_BUNDLE_IMAGE
echo "OPERATOR_BUNDLE_IMAGE_NAME = $OPERATOR_BUNDLE_IMAGE_NAME"
#Generate the catalog for the latest bundle
#podman login quay.io -u ${RHOAI_QUAY_RO_USERNAME} -p ${RHOAI_QUAY_RO_TOKEN}
BASE64_AUTH=$(echo -n "${RHOAI_QUAY_RO_USERNAME}:${RHOAI_QUAY_RO_TOKEN}" | base64 -w 0)
mkdir ${HOME}/.docker
echo '{"auths":{"quay.io/rhoai/odh-operator-bundle":{"username":"'"${RHOAI_QUAY_RO_USERNAME}"'","password":"'"${RHOAI_QUAY_RO_TOKEN}"'","email":"","auth":"'"${BASE64_AUTH}"'"}}}' > ${HOME}/.docker/config.json
APPLICATION_NAME=rhoai-${COMPONENT_SUFFIX}
echo "APPLICATION_NAME = $APPLICATION_NAME"
#Invoke the FBC processor to extract the snapshot images
CATALOG_BUILD_ARGS_FILE_PATH=${BRANCH}/catalog/catalog_build_args.map
python3 utils/utils/fbc-processor/fbc-processor.py -op extract-snapshot-images -o utils/snapshot_images.json -v ${BRANCH} -f ${OPERATOR_BUNDLE_COMPONENT_NAME} -b ${BUILD_CONFIG_PATH} --catalog-build-args-file-path ${CATALOG_BUILD_ARGS_FILE_PATH} --build-type nightly
LATEST_BUNDLE_IMAGE=$(jq --arg OPERATOR_BUNDLE_IMAGE_NAME "$OPERATOR_BUNDLE_IMAGE_NAME" -r '.[] | select(.name == $OPERATOR_BUNDLE_IMAGE_NAME) | .value' utils/snapshot_images.json)
echo "LATEST_BUNDLE_IMAGE = $LATEST_BUNDLE_IMAGE"
yq e -i '.stable.bundles[0].image = "$LATEST_BUNDLE_IMAGE"' utils/utils/fbc-processor/fbc-semver-template.yaml
LATEST_BUNDLE_IMAGE="$LATEST_BUNDLE_IMAGE" yq e -i '.stable.bundles[0].image = env(LATEST_BUNDLE_IMAGE)' utils/utils/fbc-processor/fbc-semver-template.yaml
#Generate the single bundle catalog === sbc
CSV_META_MIN_OCP_VERSION=417
WORK_DIR=utils/utils/fbc
mkdir -p ${WORK_DIR}
BUNDLE_OBJECT_SINGLE_BUNDLE_PATH=${WORK_DIR}/bundle_object_sbc_semver.yaml
CSV_META_SINGLE_BUNDLE_PATH=${WORK_DIR}/csv_meta_sbc_semver.yaml
CSV_META_OPM_FLAG="--migrate-level=bundle-object-to-csv-metadata"
DOCKER_CONFIG=${HOME}/.docker/ opm alpha render-template semver -o yaml utils/utils/fbc-processor/fbc-semver-template.yaml > ${BUNDLE_OBJECT_SINGLE_BUNDLE_PATH}
DOCKER_CONFIG=${HOME}/.docker/ opm alpha render-template semver ${CSV_META_OPM_FLAG} -o yaml utils/utils/fbc-processor/fbc-semver-template.yaml > ${CSV_META_SINGLE_BUNDLE_PATH}
PCC_BUNDLE_OBJECT_CATALOG_YAML_PATH=main/pcc/bundle_object_catalog.yaml
PCC_CSV_META_CATALOG_YAML_PATH=main/pcc/csv_meta_catalog.yaml
BUNDLE_OBJECT_CATALOG_YAML_PATH=${WORK_DIR}/bundle_object_catalog.yaml
CSV_META_CATALOG_YAML_PATH=${WORK_DIR}/csv_meta_catalog.yaml
PUSH_PIPELINE_PATH=${BRANCH}/.tekton/rhoai-fbc-fragment-${COMPONENT_SUFFIX}-push.yaml
#Invoke FBC processor to patch the catalog
python3 utils/utils/fbc-processor/fbc-processor.py -op catalog-patch -b ${BUILD_CONFIG_PATH} -c ${PCC_BUNDLE_OBJECT_CATALOG_YAML_PATH} -p ${PATCH_YAML_PATH} -s ${BUNDLE_OBJECT_SINGLE_BUNDLE_PATH} -o ${BUNDLE_OBJECT_CATALOG_YAML_PATH} --push-pipeline-yaml-path ${PUSH_PIPELINE_PATH} --push-pipeline-operation disable
python3 utils/utils/fbc-processor/fbc-processor.py -op catalog-patch -b ${BUILD_CONFIG_PATH} -c ${PCC_CSV_META_CATALOG_YAML_PATH} -p ${PATCH_YAML_PATH} -s ${CSV_META_SINGLE_BUNDLE_PATH} -o ${CSV_META_CATALOG_YAML_PATH} --push-pipeline-yaml-path ${PUSH_PIPELINE_PATH} --push-pipeline-operation disable
while IFS= read -r value;
do
#Declare FBC processing variables
OPENSHIFT_VERSION=$value
NUMERIC_OCP_VERSION=${OPENSHIFT_VERSION/v4./4}
echo "OPENSHIFT_VERSION=$OPENSHIFT_VERSION"
OUTPUT_CATALOG_PATH=${BRANCH}/catalog/${OPENSHIFT_VERSION}/rhods-operator/catalog.yaml
SOURCE_CATALOG=${BUNDLE_OBJECT_CATALOG_YAML_PATH}
if [[ $NUMERIC_OCP_VERSION -ge $CSV_META_MIN_OCP_VERSION ]]
then
SOURCE_CATALOG=${CSV_META_CATALOG_YAML_PATH}
fi
cp ${SOURCE_CATALOG} ${OUTPUT_CATALOG_PATH}
#cat ${OUTPUT_CATALOG_PATH}
done < <(yq eval '.config.supported-ocp-versions.build[].name' $BUILD_CONFIG_PATH)
# Update the schedule file to trigger the full fbc build
echo $(date +'%d-%m-%Y %H:%M:%S:%3N') > ${BRANCH}/schedule/catalog-tekton-trigger.txt
- name: Commit and push the changes to release branch
uses: actions-js/push@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: ${{ steps.get_branch.outputs.branch }}
message: "Updating the catalog.yaml with latest images and patches"
repository: ${{ env.GITHUB_ORG }}/RHOAI-Build-Config
directory: ${{ steps.get_branch.outputs.branch }}
author_name: Openshift-AI DevOps
author_email: [email protected]