From 665e3273e60962f99638b1882831a3d5ec5dc9e1 Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Mon, 24 Jul 2023 08:39:14 -0400 Subject: [PATCH 1/3] Add push artifact work around. The workaround identifies artifacts in workspace and pushes them to s3 storage. Signed-off-by: Humair Khan (cherry picked from commit 08eb98dbb3f6b47740608e050ad40bc93779f740) --- config/internal/apiserver/artifact_script.yaml.tmpl | 13 ++++++++++--- .../expected/created/configmap_artifact_script.yaml | 13 ++++++++++--- .../expected/created/configmap_artifact_script.yaml | 13 ++++++++++--- .../expected/created/configmap_artifact_script.yaml | 13 ++++++++++--- .../expected/created/configmap_artifact_script.yaml | 13 ++++++++++--- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/config/internal/apiserver/artifact_script.yaml.tmpl b/config/internal/apiserver/artifact_script.yaml.tmpl index f54d5558..9af903b2 100644 --- a/config/internal/apiserver/artifact_script.yaml.tmpl +++ b/config/internal/apiserver/artifact_script.yaml.tmpl @@ -3,9 +3,16 @@ data: artifact_script: |- #!/usr/bin/env sh push_artifact() { - if [ -f "$2" ]; then - tar -cvzf $1.tgz $2 - aws s3 --endpoint {{.ObjectStorageConnection.Endpoint}} cp $1.tgz s3://{{.ObjectStorageConnection.Bucket}}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g") + workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name) + artifact_name=$(basename $2) + if [ -f "$workspace_dest/$artifact_name" ]; then + echo sending to: ${workspace_dest}/${artifact_name} + tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + elif [ -f "$2" ]; then + tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz else echo "$2 file does not exist. Skip artifact tracking for $1" fi diff --git a/controllers/testdata/declarative/case_0/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_0/expected/created/configmap_artifact_script.yaml index a6e1e365..30771108 100644 --- a/controllers/testdata/declarative/case_0/expected/created/configmap_artifact_script.yaml +++ b/controllers/testdata/declarative/case_0/expected/created/configmap_artifact_script.yaml @@ -3,9 +3,16 @@ data: artifact_script: |- #!/usr/bin/env sh push_artifact() { - if [ -f "$2" ]; then - tar -cvzf $1.tgz $2 - aws s3 --endpoint http://minio-testdsp0.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g") + workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name) + artifact_name=$(basename $2) + if [ -f "$workspace_dest/$artifact_name" ]; then + echo sending to: ${workspace_dest}/${artifact_name} + tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + elif [ -f "$2" ]; then + tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz else echo "$2 file does not exist. Skip artifact tracking for $1" fi diff --git a/controllers/testdata/declarative/case_2/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_2/expected/created/configmap_artifact_script.yaml index 82048ee1..88659df8 100644 --- a/controllers/testdata/declarative/case_2/expected/created/configmap_artifact_script.yaml +++ b/controllers/testdata/declarative/case_2/expected/created/configmap_artifact_script.yaml @@ -3,9 +3,16 @@ data: artifact_script: |- #!/usr/bin/env sh push_artifact() { - if [ -f "$2" ]; then - tar -cvzf $1.tgz $2 - aws s3 --endpoint http://minio-testdsp2.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g") + workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name) + artifact_name=$(basename $2) + if [ -f "$workspace_dest/$artifact_name" ]; then + echo sending to: ${workspace_dest}/${artifact_name} + tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + elif [ -f "$2" ]; then + tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz else echo "$2 file does not exist. Skip artifact tracking for $1" fi diff --git a/controllers/testdata/declarative/case_4/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_4/expected/created/configmap_artifact_script.yaml index b53a2455..b00c143c 100644 --- a/controllers/testdata/declarative/case_4/expected/created/configmap_artifact_script.yaml +++ b/controllers/testdata/declarative/case_4/expected/created/configmap_artifact_script.yaml @@ -3,9 +3,16 @@ data: artifact_script: |- #!/usr/bin/env sh push_artifact() { - if [ -f "$2" ]; then - tar -cvzf $1.tgz $2 - aws s3 --endpoint http://minio-testdsp4.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g") + workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name) + artifact_name=$(basename $2) + if [ -f "$workspace_dest/$artifact_name" ]; then + echo sending to: ${workspace_dest}/${artifact_name} + tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + elif [ -f "$2" ]; then + tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz else echo "$2 file does not exist. Skip artifact tracking for $1" fi diff --git a/controllers/testdata/declarative/case_5/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_5/expected/created/configmap_artifact_script.yaml index 1220718c..bf1f028c 100644 --- a/controllers/testdata/declarative/case_5/expected/created/configmap_artifact_script.yaml +++ b/controllers/testdata/declarative/case_5/expected/created/configmap_artifact_script.yaml @@ -3,9 +3,16 @@ data: artifact_script: |- #!/usr/bin/env sh push_artifact() { - if [ -f "$2" ]; then - tar -cvzf $1.tgz $2 - aws s3 --endpoint http://minio-testdsp5.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g") + workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name) + artifact_name=$(basename $2) + if [ -f "$workspace_dest/$artifact_name" ]; then + echo sending to: ${workspace_dest}/${artifact_name} + tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz + elif [ -f "$2" ]; then + tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name} + aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz else echo "$2 file does not exist. Skip artifact tracking for $1" fi From 15e7b0648cf172ae17c4948d11f3c1cf736a503f Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Thu, 27 Jul 2023 16:28:43 -0400 Subject: [PATCH 2/3] Use unauthenticated images for go-toolset. Signed-off-by: Humair Khan (cherry picked from commit a67b16bd037ae0c2c63de8f86e82298423621df2) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d3428ff4..2a2ae627 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM registry.redhat.io/ubi8/go-toolset:1.18.9-8 as builder +FROM registry.access.redhat.com/ubi8/go-toolset:1.18.9-8 as builder ARG TARGETOS ARG TARGETARCH From beb5063607f26e38f10d6762ce1d2a021f351f52 Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Fri, 28 Jul 2023 15:24:17 -0400 Subject: [PATCH 3/3] Add v1.0.1 shas. Signed-off-by: Humair Khan --- config/base/params.env | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/config/base/params.env b/config/base/params.env index aded367d..e94b402f 100644 --- a/config/base/params.env +++ b/config/base/params.env @@ -1,12 +1,12 @@ -IMAGES_APISERVER=quay.io/opendatahub/ds-pipelines-api-server:v1.0.0 -IMAGES_ARTIFACT=quay.io/opendatahub/ds-pipelines-artifact-manager:v1.0.0 -IMAGES_PERSISTENTAGENT=quay.io/opendatahub/ds-pipelines-persistenceagent:v1.0.0 -IMAGES_SCHEDULEDWORKFLOW=quay.io/opendatahub/ds-pipelines-scheduledworkflow:v1.0.0 -IMAGES_CACHE=registry.access.redhat.com/ubi8/ubi-minimal -IMAGES_MOVERESULTSIMAGE=registry.access.redhat.com/ubi8/ubi-micro -IMAGES_MARIADB=registry.redhat.io/rhel8/mariadb-103:1-188 -IMAGES_DSPO=quay.io/opendatahub/data-science-pipelines-operator:v1.0.0 -IMAGES_OAUTHPROXY=registry.redhat.io/openshift4/ose-oauth-proxy:v4.12.0 -IMAGES_MLMDENVOY=quay.io/opendatahub/ds-pipelines-metadata-envoy:v1.0.0 -IMAGES_MLMDGRPC=quay.io/opendatahub/ds-pipelines-metadata-grpc:v1.0.0 -IMAGES_MLMDWRITER=quay.io/opendatahub/ds-pipelines-metadata-writer:v1.0.0 +IMAGES_APISERVER=quay.io/opendatahub/ds-pipelines-api-server@sha256:e310024243fa47dc30f89b6b53f633db79c69c2ae7202e483f3bc37bfd95d526 +IMAGES_ARTIFACT=quay.io/opendatahub/ds-pipelines-artifact-manager@sha256:bf13a1d4117e3fadc63ce03eaba3230f5286195ac88c3951696d53448808e36e +IMAGES_PERSISTENTAGENT=quay.io/opendatahub/ds-pipelines-persistenceagent@sha256:5b98125e99bfa1ddb6197ddf2b520e0a8807b551a24ef96a0ccf2cb0257ab700 +IMAGES_SCHEDULEDWORKFLOW=quay.io/opendatahub/ds-pipelines-scheduledworkflow@sha256:31262aaf1abadfe0f707224a9b7b0b9b666539d10cd76804f79804b1aa2763cd +IMAGES_CACHE=registry.access.redhat.com/ubi8/ubi-minimal@sha256:e52fc1de73dc2879516431ff1865e0fb61b1a32f57b6f914bdcddb13c62f84e6 +IMAGES_MOVERESULTSIMAGE=registry.access.redhat.com/ubi8/ubi-micro@sha256:443db9a646aaf9374f95d266ba0c8656a52d70d0ffcc386a782cea28fa32e55d +IMAGES_MARIADB=registry.redhat.io/rhel8/mariadb-103@sha256:cafc7364494fb7206c373a1235fd5da74399c19b5c34d87dd02aa07e8f343fa2 +IMAGES_DSPO=quay.io/opendatahub/data-science-pipelines-operator@sha256:2a7a4afcd5deca392b297bc996b27c58158f2428491671d29213dab0ae9c2256 +IMAGES_OAUTHPROXY=registry.redhat.io/openshift4/ose-oauth-proxy@sha256:d0f2f1ef0bdc3aa1a70794ac8ac779271b634af83e939029ac5224ec0c815d7a +IMAGES_MLMDENVOY=quay.io/opendatahub/ds-pipelines-metadata-envoy@sha256:851386f25bec1051a472e87eb98b3b8016f80e1d2e05a4f5d0c4323cb1c99563 +IMAGES_MLMDGRPC=quay.io/opendatahub/ds-pipelines-metadata-grpc@sha256:f2ff89ac664916789e690f8939b5fb0881e6662211a9c40712779236b862735d +IMAGES_MLMDWRITER=quay.io/opendatahub/ds-pipelines-metadata-writer@sha256:aea738408d1a4334671b165ad54196836530e7602764be066924c2afd02abf41