diff --git a/.github/pull.yml b/.github/pull.yml new file mode 100644 index 00000000..638294b6 --- /dev/null +++ b/.github/pull.yml @@ -0,0 +1,5 @@ +version: "1" +rules: + - base: main + upstream: kubeflow:main +label: "tide/merge-method-merge" diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml index 45c5e34c..ec18b55d 100644 --- a/.github/workflows/build-and-push-image.yml +++ b/.github/workflows/build-and-push-image.yml @@ -14,10 +14,10 @@ on: - '.github/dependabot.yml' - 'docs/**' env: - IMG_ORG: kubeflow + IMG_ORG: opendatahub IMG_REPO: model-registry - DOCKER_USER: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKER_PWD: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKER_USER: ${{ secrets.QUAY_USERNAME }} + DOCKER_PWD: ${{ secrets.QUAY_PASSWORD }} PUSH_IMAGE: true jobs: build-image: @@ -50,7 +50,7 @@ jobs: if: env.BUILD_CONTEXT == 'main' shell: bash env: - IMG: ${{ env.IMG_ORG }}/${{ env.IMG_REPO }} + IMG: quay.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }} BUILD_IMAGE: false # image is already built in "Build and Push Image" step run: | docker tag ${{ env.IMG }}:$VERSION ${{ env.IMG }}:latest @@ -60,7 +60,7 @@ jobs: if: env.BUILD_CONTEXT == 'main' shell: bash env: - IMG: ${{ env.IMG_ORG }}/${{ env.IMG_REPO }} + IMG: quay.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }} BUILD_IMAGE: false # image is already built in "Build and Push Image" step run: | docker tag ${{ env.IMG }}:$VERSION ${{ env.IMG }}:main diff --git a/.github/workflows/build-image-pr.yml b/.github/workflows/build-image-pr.yml index ea9e53f5..c06c0af2 100644 --- a/.github/workflows/build-image-pr.yml +++ b/.github/workflows/build-image-pr.yml @@ -11,7 +11,7 @@ on: - "docs/**" - "clients/python/docs/**" env: - IMG_ORG: kubeflow + IMG_ORG: opendatahub IMG_REPO: model-registry PUSH_IMAGE: false BRANCH: ${{ github.base_ref }} @@ -38,29 +38,23 @@ jobs: node_image: "kindest/node:v1.27.11" - name: Load Local Registry Test Image env: - IMG: "${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" + IMG: "quay.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" run: | kind load docker-image -n chart-testing ${IMG} - - name: Create Test Registry + - name: Deploy Operator With Test Image env: - IMG: "${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" + IMG: "quay.io/${{ env.IMG_ORG }}/${{ env.IMG_REPO }}:${{ steps.tags.outputs.tag }}" + run: | + echo "Deploying operator from model-registry-operator branch ${BRANCH}" + kubectl apply -k "https://github.com/opendatahub-io/model-registry-operator.git/config/default?ref=${BRANCH}" + kubectl set env -n model-registry-operator-system deployment/model-registry-operator-controller-manager REST_IMAGE="${IMG}" + - name: Create Test Registry run: | - echo "Download kustomize 5.2.1" - mkdir $GITHUB_WORKSPACE/kustomize - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s "5.2.1" "$GITHUB_WORKSPACE/kustomize" - PATH=$GITHUB_WORKSPACE/kustomize:$PATH - echo "Display Kustomize version" - kustomize version - echo "Deploying Model Registry using Manifests; branch ${BRANCH}" - kubectl create namespace kubeflow - cd manifests/kustomize/overlays/db - kustomize edit set image kubeflow/model-registry:latest $IMG - kustomize build | kubectl apply -f - + kubectl apply -k "https://github.com/opendatahub-io/model-registry-operator.git/config/samples/postgres?ref=${BRANCH}" + kubectl get mr - name: Wait for Test Registry Deployment run: | - kubectl wait --for=condition=available -n kubeflow deployment/model-registry-db --timeout=5m - kubectl wait --for=condition=available -n kubeflow deployment/model-registry-deployment --timeout=5m - kubectl logs -n kubeflow deployment/model-registry-deployment + kubectl wait --for=condition=Available=true modelregistries/modelregistry-sample --timeout=5m - name: Set up Python uses: actions/setup-python@v5 with: @@ -80,6 +74,6 @@ jobs: pip install dist/*.whl - name: Connect with Python client run: | - kubectl port-forward -n kubeflow service/model-registry-service 9090:9090 & + kubectl port-forward service/modelregistry-sample 9090:9090 & sleep 5 python test/python/test_mr_conn.py localhost 9090 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 19aa441b..901d3489 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,3 +50,10 @@ jobs: fi - name: Unit tests run: make test-cover + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4.0.1 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: coverage.txt + fail_ci_if_error: true diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index a653cde3..4dae9047 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -60,6 +60,14 @@ jobs: else nox --python=${{ matrix.python }} fi + - name: Upload coverage report + uses: codecov/codecov-action@v4.0.1 + if: always() && matrix.session == 'tests' + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: coverage.xml + fail_ci_if_error: true - name: Upload documentation if: matrix.session == 'docs-build' uses: actions/upload-artifact@v4 diff --git a/Makefile b/Makefile index c0b6e681..103d7ca8 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,9 @@ DOCKER ?= docker # default Dockerfile DOCKERFILE ?= Dockerfile # container registry, default to empty (dockerhub) if not explicitly set -IMG_REGISTRY ?= +IMG_REGISTRY ?= quay.io # container image organization -IMG_ORG ?= kubeflow +IMG_ORG ?= opendatahub # container image version IMG_VERSION ?= main # container image repository diff --git a/OWNERS b/OWNERS index 24f60adf..2ff7cbf4 100644 --- a/OWNERS +++ b/OWNERS @@ -1,9 +1,25 @@ +# DO NOT EDIT; this file is auto-generated using https://github.com/openshift/ci-tools. +# Fetched from https://github.com/opendatahub-io/codeflare-operator root OWNERS +# If the repo had OWNERS_ALIASES then the aliases were expanded +# Logins who are not members of 'openshift' organization were filtered out +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + approvers: - - andreyvelich - - ckadner - - tarilabs - - rareddy - - Tomcli - - zijianjoy +- dhirajsb +- isinyaaa +- lampajr +- nehachopra27 +- rareddy +- rkubis +- tarilabs +- tonyxrmdavidson +options: {} reviewers: - - andreyvelich +- dhirajsb +- isinyaaa +- lampajr +- nehachopra27 +- rareddy +- rkubis +- tarilabs +- tonyxrmdavidson diff --git a/csi/Dockerfile b/csi/Dockerfile index 706489b9..d9fb65b8 100644 --- a/csi/Dockerfile +++ b/csi/Dockerfile @@ -23,7 +23,7 @@ RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 make build # Use distroless as minimal base image to package the model-registry storage initializer binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8 +FROM registry.access.redhat.com/ubi8/ubi-minimal:latest WORKDIR / # copy the storage initializer binary COPY --from=builder /workspace/bin/mr-storage-initializer . diff --git a/manifests/kustomize/OWNERS b/manifests/kustomize/OWNERS index 4e580875..ea99ec7a 100644 --- a/manifests/kustomize/OWNERS +++ b/manifests/kustomize/OWNERS @@ -1,8 +1,6 @@ approvers: - tarilabs - rareddy - - Tomcli reviewers: - tarilabs - rareddy - - Tomcli diff --git a/manifests/kustomize/base/model-registry-deployment.yaml b/manifests/kustomize/base/model-registry-deployment.yaml index 206cae4d..7022ee17 100644 --- a/manifests/kustomize/base/model-registry-deployment.yaml +++ b/manifests/kustomize/base/model-registry-deployment.yaml @@ -24,7 +24,7 @@ spec: command: - /model-registry - proxy - image: kubeflow/model-registry:latest + image: quay.io/opendatahub/model-registry:latest # empty placeholder environment for patching env: [] ports: diff --git a/openshift-ci/README.md b/openshift-ci/README.md new file mode 100644 index 00000000..149e036d --- /dev/null +++ b/openshift-ci/README.md @@ -0,0 +1,34 @@ +# Model Registry Deployment and Deployment Test + +This deployment of model-registry is deployed via Opendatahub and used the ODH nightly images deployed to an openshfit cluster. + +The script will do the following: +* Create a catalogue source pointing to the latest successful nightly ODH image. +* Deploy the opendatahub operator using the catalogue source. +* Deploy a Data Science Cluster. +* Test the model-registry-operator-contoller-manager pods are running. +* Clone the model-registry-operator repository. +* Deploy model-registry using config/samples/mysql configuration files. +* Test the model-registry-db mysql database pod is running +* Test the modelregistry-sample pods are running + +## Pre-requisites: + +You will need to have an openshift cluster deployed and be oc logged into you cluster as admin. + +## Runing the script: + +From the root of the repository +``` +./openshift-ci/scripts/oc-model-registry-deploy.sh +``` + +## Runing the openshift-ci + +You can start the openshift-ci job to test changes in your Pull Request. To do so put the following command into a comment in your Pull Request +``` +/test e2e-odh-mro-optional +``` + +Previous jobs can be seen [here](https://prow.ci.openshift.org/job-history/gs/test-platform-results/pr-logs/directory/rehearse-49999-pull-ci-opendatahub-io-model-registry-main-e2e-odh-mro-optional) + diff --git a/openshift-ci/resources/model-registry-DSCInitialization.yaml b/openshift-ci/resources/model-registry-DSCInitialization.yaml new file mode 100644 index 00000000..5d1fd548 --- /dev/null +++ b/openshift-ci/resources/model-registry-DSCInitialization.yaml @@ -0,0 +1,16 @@ +apiVersion: dscinitialization.opendatahub.io/v1 +kind: DSCInitialization +metadata: + creationTimestamp: '2024-03-27T16:16:55Z' + finalizers: + - dscinitialization.opendatahub.io/finalizer + generation: 1 + labels: + app.kubernetes.io/created-by: opendatahub-operator + app.kubernetes.io/instance: default + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: dscinitialization + app.kubernetes.io/part-of: opendatahub-operator + name: default-dsci +spec: + applicationsNamespace: opendatahub diff --git a/openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml b/openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml new file mode 100644 index 00000000..b77ecfdd --- /dev/null +++ b/openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml @@ -0,0 +1,24 @@ +apiVersion: modelregistry.opendatahub.io/v1alpha1 +kind: ModelRegistry +metadata: + labels: + app.kubernetes.io/name: modelregistry + app.kubernetes.io/instance: modelregistry-sample + app.kubernetes.io/part-of: model-registry-operator + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/created-by: model-registry-operator + name: modelregistry-sample +spec: + # TODO(user): Add fields here + grpc: + port: 9090 + rest: + port: 8080 + serviceRoute: enabled + mysql: + host: model-registry-db + database: model_registry + username: mlmduser + passwordSecret: + name: model-registry-db + key: database-password diff --git a/openshift-ci/resources/model-registry-operator/mysql-db.yaml b/openshift-ci/resources/model-registry-operator/mysql-db.yaml new file mode 100644 index 00000000..3fda2ef4 --- /dev/null +++ b/openshift-ci/resources/model-registry-operator/mysql-db.yaml @@ -0,0 +1,126 @@ +apiVersion: v1 +items: +- apiVersion: v1 + kind: Service + metadata: + annotations: + template.openshift.io/expose-uri: mysql://{.spec.clusterIP}:{.spec.ports[?(.name==\mysql\)].port} + name: model-registry-db + spec: + ports: + - name: mysql + nodePort: 0 + port: 3306 + protocol: TCP + targetPort: 3306 + selector: + name: model-registry-db + sessionAffinity: None + type: ClusterIP +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: model-registry-db + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi +- apiVersion: apps/v1 + kind: Deployment + metadata: + annotations: + template.alpha.openshift.io/wait-for-ready: "true" + name: model-registry-db + spec: + replicas: 1 + revisionHistoryLimit: 0 + selector: + matchLabels: + name: model-registry-db + strategy: + type: Recreate + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + labels: + name: model-registry-db + spec: + containers: + - env: + - name: MYSQL_USER + valueFrom: + secretKeyRef: + key: database-user + name: model-registry-db + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: model-registry-db + - name: MYSQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: model-registry-db + args: + - --datadir + - /var/lib/mysql/datadir + - --default-authentication-plugin=mysql_native_password + image: mysql:8.3.0 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /bin/bash + - -c + - mysqladmin -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} ping + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 + name: mysql + ports: + - containerPort: 3306 + protocol: TCP + readinessProbe: + exec: + command: + - /bin/bash + - -c + - mysql -D ${MYSQL_DATABASE} -u${MYSQL_USER} -p${MYSQL_ROOT_PASSWORD} -e 'SELECT 1' + initialDelaySeconds: 10 + timeoutSeconds: 5 + securityContext: + capabilities: {} + privileged: false + terminationMessagePath: /dev/termination-log + volumeMounts: + - mountPath: /var/lib/mysql + name: model-registry-db-data + dnsPolicy: ClusterFirst + restartPolicy: Always + volumes: + - name: model-registry-db-data + persistentVolumeClaim: + claimName: model-registry-db +- apiVersion: v1 + kind: Secret + metadata: + annotations: + template.openshift.io/expose-database_name: '{.data[''database-name'']}' + template.openshift.io/expose-password: '{.data[''database-password'']}' + template.openshift.io/expose-username: '{.data[''database-user'']}' + name: model-registry-db + stringData: + database-name: "model_registry" + database-password: "TheBlurstOfTimes" + database-user: "mlmduser" +kind: List +metadata: {} diff --git a/openshift-ci/resources/opendatahub-data-science-cluster.yaml b/openshift-ci/resources/opendatahub-data-science-cluster.yaml new file mode 100644 index 00000000..b5d56c09 --- /dev/null +++ b/openshift-ci/resources/opendatahub-data-science-cluster.yaml @@ -0,0 +1,38 @@ +kind: DataScienceCluster +apiVersion: datasciencecluster.opendatahub.io/v1 +metadata: + labels: + app.kubernetes.io/created-by: opendatahub-operator + app.kubernetes.io/instance: default + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: datasciencecluster + app.kubernetes.io/part-of: opendatahub-operator + name: default-dsc +spec: + components: + codeflare: + managementState: Removed + dashboard: + managementState: Managed + datasciencepipelines: + managementState: Managed + kserve: + managementState: Managed + serving: + ingressGateway: + certificate: + type: SelfSigned + managementState: Managed + name: knative-serving + kueue: + managementState: Removed + modelmeshserving: + managementState: Managed + modelregistry: + managementState: Managed + ray: + managementState: Removed + trustyai: + managementState: Removed + workbenches: + managementState: Managed diff --git a/openshift-ci/resources/opendatahub-subscription.yaml b/openshift-ci/resources/opendatahub-subscription.yaml new file mode 100644 index 00000000..3a0f1325 --- /dev/null +++ b/openshift-ci/resources/opendatahub-subscription.yaml @@ -0,0 +1,10 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: model-registry-test-source-subscription + namespace: openshift-operators +spec: + channel: fast + name: opendatahub-operator + source: community-operators + sourceNamespace: openshift-marketplace \ No newline at end of file diff --git a/openshift-ci/scripts/colour_text_variables.sh b/openshift-ci/scripts/colour_text_variables.sh new file mode 100644 index 00000000..c1be2c09 --- /dev/null +++ b/openshift-ci/scripts/colour_text_variables.sh @@ -0,0 +1,4 @@ +export RED='\033[0;31m' +export GREEN='\033[0;32m' +export NC='\033[0m' +export YELLOW='\033[0;33m' \ No newline at end of file diff --git a/openshift-ci/scripts/oci-model-registry-deploy.sh b/openshift-ci/scripts/oci-model-registry-deploy.sh new file mode 100755 index 00000000..6b78c835 --- /dev/null +++ b/openshift-ci/scripts/oci-model-registry-deploy.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +# Define variables for ODH deployment deployment +OPENDATAHUB_SUBSCRIPTION="openshift-ci/resources/opendatahub-subscription.yaml" +DSC_INITIALIZATION_MANIFEST="openshift-ci/resources/model-registry-DSCInitialization.yaml" +DATA_SCIENCE_CLUSTER_MANIFEST="openshift-ci/resources/opendatahub-data-science-cluster.yaml" +MODEL_REGISTRY_DB_MANIFEST="openshift-ci/resources/model-registry-operator/mysql-db.yaml" +MODEL_REGISTRY_SAMPLE_MANIFEST="openshift-ci/resources/model-registry-operator/modelregistry_v1alpha1_modelregistry.yaml" +OPENDATAHUB_CRDS="datascienceclusters.datasciencecluster.opendatahub.io,dscinitializations.dscinitialization.opendatahub.io,featuretrackers.features.opendatahub.io" +DATA_SCIENCE_CLUSTER_CRDS="acceleratorprofiles.dashboard.opendatahub.io,datasciencepipelinesapplications.datasciencepipelinesapplications.opendatahub.io,odhapplications.dashboard.opendatahub.io,odhdashboardconfigs.opendatahub.io,odhdocuments.dashboard.opendatahub.io" +MODEL_REGISTRY_CRDS="modelregistries.modelregistry.opendatahub.io" +source "openshift-ci/scripts/colour_text_variables.sh" + +# Function to monitor CRDS creation and deployment. +# The function takes two arguments, reference to manifest and a wait time in seconds. +monitoring_crds_installation() { + IFS=',' read -ra crds_array <<< "$1" + local timeout=$2 + + echo "Monitoring the installation of CRDs: ${crds_array[*]}" + echo "Timeout set to ${timeout}s" + + local start_time=$(date +%s) + + # Loop until all specified CRDs are installed or timeout is reached + while true; do + local elapsed_time=$(($(date +%s) - start_time)) + + # Check if timeout has been reached + if [ "$elapsed_time" -ge "$timeout" ]; then + echo -e "${RED}X Error:${NC} Timeout reached. Installation of CRDs failed." + return 1 + fi + + # Get the list of installed CRDs + local installed_crds=($(oc get crd -o=name | cut -d '/' -f2)) + + # Check if all CRDs are installed + local all_installed=true + for crd in "${crds_array[@]}"; do + if ! [[ " ${installed_crds[@]} " =~ " ${crd} " ]]; then + all_installed=false + break + fi + done + + # If all CRDs are installed, break out of the loop + if [ "$all_installed" = true ]; then + echo -e "${GREEN}✔ Success:${NC} All specified CRDs are installed." + return 0 + fi + + # Print the status of each CRD + for crd in "${crds_array[@]}"; do + if [[ " ${installed_crds[@]} " =~ " ${crd} " ]]; then + echo "CRD '$crd' is installed." + else + echo "CRD '$crd' is not installed." + fi + done + + # Wait for a few seconds before checking again + sleep 5 + done +} + +# Function to deploy and wait for deployment +# The function takes two arguments, reference to manifest and a wait time in seconds. +deploy_and_wait() { + local namespace=$1 + local manifest=$2 + local resource_name=$(basename -s .yaml $manifest) + local wait_time=$3 + + sleep $wait_time + + echo "Deploying $resource_name from $manifest..." + + if oc apply -f $manifest $namespace --wait=true --timeout=300s; then + echo -e "${GREEN}✔ Success:${NC} Deployment of $resource_name succeeded." + else + echo -e "${RED}X Error:${NC} Deployment of $resource_name failed or timed out." >&2 + return 1 + fi +} + +check_deployment_availability() { + local namespace="$1" + local deployment="$2" + local timeout=300 # Timeout in seconds + local start_time=$(date +%s) + + # Loop until timeout + while (( $(date +%s) - start_time < timeout )); do + # Get the availability status of the deployment + local deployment_status=$(oc get deployment "$deployment" -n "$namespace" --no-headers -o custom-columns=:.status.availableReplicas) + + # Check if the deployment is available + if [[ $deployment_status != "" ]]; then + echo -e "${GREEN}✔ Success:${NC} Deployment $deployment is available" + return 0 # Success + fi + + sleep 5 # Wait for 5 seconds before checking again + done + + echo -e "${RED}X Error:${NC} Timeout reached. Deployment $deployment did not become available within $timeout seconds" + return 1 # Failure +} + +# Function to check the status of deploying pods +# The function takes three arguments, namespace, descriptor to identify the component and number of containers expected. +check_pod_status() { + local namespace="$1" + local pod_selector="$2" + local expected_ready_containers="$3" + local timeout=300 # Timeout in seconds + local start_time=$(date +%s) + + # Loop until timeout + while (( $(date +%s) - start_time < timeout )); do + # Get the list of pods in the specified namespace matching the provided partial names + local pod_list=$(oc get pods -n $namespace $pod_selector --no-headers -o custom-columns=NAME:.metadata.name) + + # Iterate over each pod in the list + while IFS= read -r pod_name; do + # Get the pod info + local pod_info=$(oc get pod "$pod_name" -n "$namespace" --no-headers) + + # Extract pod status and ready status from the info + local pod_name=$(echo "$pod_info" | awk '{print $1}') + local pod_status=$(echo "$pod_info" | awk '{print $3}') + local pod_ready=$(echo "$pod_info" | awk '{print $2}') + local ready_containers=$(echo "$pod_ready" | cut -d'/' -f1) + + # Check if the pod is Running and all containers are ready + if [[ $pod_status == "Running" ]] && [[ $ready_containers -eq $expected_ready_containers ]]; then + echo -e "${GREEN}✔ Success:${NC} Pod $pod_name is running and $ready_containers out of $expected_ready_containers containers are ready" + return 0 # Success + else + echo -e "${YELLOW}! INFO:${NC} Pod $pod_name is not running or does not have $expected_ready_containers containers ready" + fi + done <<< "$pod_list" + + sleep 5 # Wait for 5 seconds before checking again + done + + echo -e "${RED}X Failure:${NC} Timeout reached. No pod matching '$pod_name_partial' became ready within $timeout seconds" + return 1 # Failure +} + +# Function to check the status of a route +# The function takes two arguments, namespace and route name. +check_route_status() { + local namespace="$1" + local route_name="$2" + local key="items" + local interval=5 + local timeout=300 + local start_time=$(date +%s) + + while (( $(date +%s) - start_time < timeout )); do + # Get the route URL + local route=$(oc get route -n "$namespace" "$route_name" -o jsonpath='{.spec.host}') + local route_url="http://$route" + + if [[ -z "$route_url" ]]; then + echo -e "${RED}X Error:${NC} Route '$route_name' does not exist in namespace '$namespace'" + return 1 + else + echo -e "${GREEN}✔ Success:${NC} Route '$route_name' exists in namespace '$namespace'" + fi + + # Test if the route is live + local response=$(curl -s -o /dev/null -w "%{http_code}" "$route_url/api/model_registry/v1alpha3/registered_models") + + # Check if the response status code is 200 OK or 404 Not Found + if [[ "$response" == "200" ]]; then + echo -e "${GREEN}✔ Success:${NC} Route server is reachable. Status code: 200 OK" + return 0 + elif [[ "$response" == "404" ]]; then + echo -e "${YELLOW}! WARNING:${NC} Route server is reachable, but service is not. Status code: 404 Not Found" + return 0 + else + echo -e "${RED}X Error:${NC} Route server is unreachable. Status code: $response" + fi + + sleep "$interval" + done + + echo -e "${RED}X Error:${NC} Timeout reached. Route '$route_name' did not become live within $timeout seconds." + return 1 +} + +# Function to source the rest api tests and run them. +run_api_tests() { + ./test/scripts/rest.sh "-n opendatahub" +} + +# Run the deployment tests. +run_deployment_tests() { + check_deployment_availability "opendatahub" model-registry-db + check_deployment_availability "opendatahub" modelregistry-sample + check_pod_status "opendatahub" "-l name=model-registry-db" 1 + check_pod_status "opendatahub" "-l app=modelregistry-sample" 2 + check_route_status "opendatahub" "modelregistry-sample-http" +} + +# Main function for orchestrating deployments +main() { + deploy_and_wait "" $OPENDATAHUB_SUBSCRIPTION 0 + monitoring_crds_installation $OPENDATAHUB_CRDS 120 + deploy_and_wait "" $DSC_INITIALIZATION_MANIFEST 20 + deploy_and_wait "" $DATA_SCIENCE_CLUSTER_MANIFEST 10 + monitoring_crds_installation $DATA_SCIENCE_CLUSTER_CRDS 120 + check_pod_status "opendatahub" "-l component.opendatahub.io/name=model-registry-operator" 2 + deploy_and_wait "-n opendatahub" $MODEL_REGISTRY_SAMPLE_MANIFEST 20 + monitoring_crds_installation $MODEL_REGISTRY_CRDS 120 + deploy_and_wait "-n opendatahub" $MODEL_REGISTRY_DB_MANIFEST 20 + run_deployment_tests + run_api_tests "-n opendatahub" +} + +# Execute main function +main \ No newline at end of file diff --git a/scripts/build_deploy.sh b/scripts/build_deploy.sh index 3a3dbf37..9ede030f 100755 --- a/scripts/build_deploy.sh +++ b/scripts/build_deploy.sh @@ -3,8 +3,8 @@ set -e # see Makefile for the IMG_ variables semantic -IMG_REGISTRY="" -IMG_ORG="${IMG_ORG:-kubeflow}" +IMG_REGISTRY="quay.io" +IMG_ORG="${IMG_ORG:-opendatahub}" IMG_REPO="${IMG_REPO:-model-registry}" DOCKER_USER="${DOCKER_USER}" DOCKER_PWD="${DOCKER_PWD}" diff --git a/test/scripts/populateDatabase.sh b/test/scripts/populateDatabase.sh new file mode 100755 index 00000000..ec0d2b5a --- /dev/null +++ b/test/scripts/populateDatabase.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +# This script will push data to model-registry-db. It targets the default model-registry +# To run the script from the root of the repo : ./test/scripts/populateDatabase.sh +# Example : Putting 5 entries into the database : ./test/scripts/populateDatabase.sh 5 + +LOOPS=$1 +MR_HOSTNAME="http://$(oc get route modelregistry-sample-http -n opendatahub --template='{{.spec.host}}')" + +# Function to send data and extract ID from response +make_post_extract_id() { + local url="$1" + local data="$2" + local id=$(curl -s -X POST "$url" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d "$data" | jq -r '.id') + + if [ -z "$id" ]; then + echo -e "Error: Failed to extract ID from response" + exit 1 + else + echo "$id" + fi +} + +# Function to post model registry data +post_model_registry_data() { + test_data_number=$1 + timestamp=$(date +"%Y%m%d%H%M%S") + rm_name="test-data-2-$test_data_number" + + rm_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" '{ + "description": "lorem ipsum registered model", + "name": "'"$rm_name"'" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Registered Model ID not returned" + exit 1 + else + echo -e "Success: Registered Model ID: $rm_id" + fi + + mv_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" '{ + "description": "lorem ipsum model version", + "name": "v1", + "author": "John Doe", + "registeredModelId": "'"$rm_id"'" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Model Version ID not returned" + exit 1 + else + echo -e "Success: Model Version ID: $mv_id" + fi + + RAW_ML_MODEL_URI='https://huggingface.co/tarilabs/mnist/resolve/v1.nb20231206162408/mnist.onnx' + ma_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/$mv_id/artifacts" '{ + "description": "lorem ipsum model artifact", + "uri": "'"$RAW_ML_MODEL_URI"'", + "name": "mnist", + "modelFormatName": "onnx", + "modelFormatVersion": "1", + "storageKey": "aws-connection-unused", + "storagePath": "unused just demo", + "artifactType": "model-artifact" + }') + + if [ $? -ne 0 ]; then + echo -e "Error: Model Artifact ID not returned" + exit 1 + else + echo -e "Success: Model Artifact ID: $ma_id" + fi +} + +# Main function for populating database +main() { + for i in {$(seq 1 $LOOPS)}; do + post_model_registry_data $i + done +} + +# Execute main function +main diff --git a/test/scripts/rest.sh b/test/scripts/rest.sh index 81e36f44..b3945305 100755 --- a/test/scripts/rest.sh +++ b/test/scripts/rest.sh @@ -1,5 +1,8 @@ #!/bin/bash +FLAG_WITH_NAMESPACE=$1 +MR_HOSTNAME="http://$(oc get route modelregistry-sample-http $FLAG_WITH_NAMESPACE --template='{{.spec.host}}')" +# Function to send data and extract ID from response make_post_extract_id() { local url="$1" local data="$2" @@ -9,89 +12,113 @@ make_post_extract_id() { -d "$data" | jq -r '.id') if [ -z "$id" ]; then - echo "Error: Failed to extract ID from response" + echo -e "Error: Failed to extract ID from response" exit 1 + else + echo "$id" fi - - echo "$id" } -# TODO: finalize using openshift-ci values. -OCP_CLUSTER_NAME="PROVIDE OCP CLUSTER NAME FOR OPENSHIFT-CI" -MR_NAMESPACE="shared-modelregistry-ns" -MR_HOSTNAME="http://modelregistry-sample-http-$MR_NAMESPACE.apps.$OCP_CLUSTER_NAME" - -timestamp=$(date +"%Y%m%d%H%M%S") -rm_name="demo-$timestamp" +# Function to post model registry data +post_model_registry_data() { + timestamp=$(date +"%Y%m%d%H%M%S") + rm_name="demo-$timestamp" -rm_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" '{ - "description": "lorem ipsum registered model", - "name": "'"$rm_name"'" -}') + rm_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/registered_models" '{ + "description": "lorem ipsum registered model", + "name": "'"$rm_name"'" + }') -if [ $? -ne 0 ]; then - exit 1 -fi -echo "Registered Model ID: $rm_id" + if [ $? -ne 0 ]; then + echo -e "Error: Registered Model ID not returned" + exit 1 + else + echo -e "Success: Registered Model ID: $rm_id" + fi -mv_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" '{ - "description": "lorem ipsum model version", - "name": "v1", - "author": "John Doe", - "registeredModelId": "'"$rm_id"'" -}') + mv_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions" '{ + "description": "lorem ipsum model version", + "name": "v1", + "author": "John Doe", + "registeredModelId": "'"$rm_id"'" + }') -if [ $? -ne 0 ]; then - exit 1 -fi -echo "Model Version ID: $mv_id" + if [ $? -ne 0 ]; then + echo -e "Error: Model Version ID not returned" + exit 1 + else + echo -e "Success: Model Version ID: $mv_id" + fi -RAW_ML_MODEL_URI='https://huggingface.co/tarilabs/mnist/resolve/v1.nb20231206162408/mnist.onnx' -ma_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/$mv_id/artifacts" '{ - "description": "lorem ipsum model artifact", - "uri": "'"$RAW_ML_MODEL_URI"'", - "name": "mnist", - "modelFormatName": "onnx", - "modelFormatVersion": "1", - "storageKey": "aws-connection-unused", - "storagePath": "unused just demo", - "artifactType": "model-artifact" -}') + RAW_ML_MODEL_URI='https://huggingface.co/tarilabs/mnist/resolve/v1.nb20231206162408/mnist.onnx' + ma_id=$(make_post_extract_id "$MR_HOSTNAME/api/model_registry/v1alpha3/model_versions/$mv_id/artifacts" '{ + "description": "lorem ipsum model artifact", + "uri": "'"$RAW_ML_MODEL_URI"'", + "name": "mnist", + "modelFormatName": "onnx", + "modelFormatVersion": "1", + "storageKey": "aws-connection-unused", + "storagePath": "unused just demo", + "artifactType": "model-artifact" + }') -if [ $? -ne 0 ]; then - exit 1 -fi -echo "Model Artifact ID: $ma_id" + if [ $? -ne 0 ]; then + echo -e "Error: Model Artifact ID not returned" + exit 1 + else + echo -e "Success: Model Artifact ID: $ma_id" + fi +} -ISVC_TARGET_NS=odh-project-b -MODEL_SERVER_NAME=modelserverb +# Function to test deployment of second odh-project +odh_project_b() { + ISVC_TARGET_NS=odh-project-b + MODEL_SERVER_NAME=modelserverb -oc apply -n $ISVC_TARGET_NS -f - <