fix(gha): prevent race condition on port-forward (kubeflow#485)

* fix(gha): prevent race condition on port-forward Signed-off-by: Alessio Pragliola <[email protected]> * chore(gha): make it faster by waiting for db before mr Signed-off-by: Alessio Pragliola <[email protected]> Co-authored-by: Matteo Mortari <[email protected]> --------- Signed-off-by: Alessio Pragliola <[email protected]> Co-authored-by: Matteo Mortari <[email protected]>
opendatahub-io · Oct 17, 2024 · cc9b33d · cc9b33d
1 parent d53528c
commit cc9b33d
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 5 deletions.
diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml
@@ -125,15 +125,15 @@ jobs:
         uses: helm/[email protected]
         with:
           node_image: "kindest/node:v1.27.11"
+          cluster_name: chart-testing-py-${{ matrix.python }}
       - name: Load Local Registry Test Image
-        if: matrix.session == 'tests'
         env:
-          IMG: "docker.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
+          IMG: "${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
         run: |
-          kind load docker-image -n chart-testing ${IMG}
+          kind load docker-image -n chart-testing-py-${{ matrix.python }} ${IMG}
       - name: Deploy Model Registry using manifests
         env:
-          IMG: "docker.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
+          IMG: "${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}"
         run: ./scripts/deploy_on_kind.sh
       - name: Nox test end-to-end
         working-directory: clients/python

diff --git a/scripts/deploy_on_kind.sh b/scripts/deploy_on_kind.sh
@@ -2,8 +2,11 @@
 
 set -e
 
+DIR="$(dirname "$0")"
 MR_NAMESPACE="${MR_NAMESPACE:-kubeflow}"
 
+source ./${DIR}/utils.sh
+
 # modularity to allow re-use this script against a remote k8s cluster
 if [[ -n "$LOCAL" ]]; then
     CLUSTER_NAME="${CLUSTER_NAME:-kind}"
@@ -33,6 +36,14 @@ else
 fi
 
 kubectl apply -k manifests/kustomize/overlays/db
-kubectl set image -n kubeflow deployment/model-registry-deployment rest-container="$IMG"
+kubectl patch deployment -n "$MR_NAMESPACE" model-registry-deployment \
+--patch '{"spec": {"template": {"spec": {"containers": [{"name": "rest-container", "image": "'$IMG'", "imagePullPolicy": "IfNotPresent"}]}}}}'
+
 kubectl wait --for=condition=available -n "$MR_NAMESPACE" deployment/model-registry-db --timeout=5m
+
+kubectl delete pod -n "$MR_NAMESPACE" --selector='component=model-registry-server'
+
+repeat_cmd_until "kubectl get pod -n "$MR_NAMESPACE" --selector='component=model-registry-server' \
+-o jsonpath=\"{.items[*].spec.containers[?(@.name=='rest-container')].image}\"" "= $IMG" 300 "kubectl describe pod -n $MR_NAMESPACE --selector='component=model-registry-server'"
+
 kubectl wait --for=condition=available -n "$MR_NAMESPACE" deployment/model-registry-deployment --timeout=5m
diff --git a/scripts/utils.sh b/scripts/utils.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+repeat_cmd_until() {
+  local cmd=$1
+  local condition=$2
+  local max_wait_secs=$3
+  local debug_cmd=$4
+  local interval_secs=2
+  local start_time=$(date +%s)
+  local output
+
+  while true; do
+
+    current_time=$(date +%s)
+    if (( (current_time - start_time) > max_wait_secs )); then
+      echo "Waited for expression "$1" to satisfy condition "$2" for $max_wait_secs seconds without luck. Returning with error."
+      if [ -n "$debug_cmd" ]; then
+        echo "Running debug command: $debug_cmd"
+        eval $debug_cmd
+      fi
+      return 1
+    fi
+
+    output=$(eval $cmd)
+
+    if [ $output $condition ]; then
+      break
+    else
+      sleep $interval_secs
+    fi
+  done
+}