From 8ce98fd060e3a5cce8390f05076dde6ce18ba8f2 Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 18 Sep 2024 17:18:47 +0300 Subject: [PATCH 01/32] Add tests --- .../ODH/ODHDashboard/ODHModelServing.resource | 24 +++++------ .../1008_model_serving_cross_auth.robot | 43 ++++++++++++------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 66a86b725..b5af18e8b 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -387,19 +387,19 @@ Clean Up Model Serving Page [Documentation] Deletes all currently deployed models, if any are present. # Returns an empty list if no matching elements found Switch Model Serving Project project_name=All projects - ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] - FOR ${project} IN @{projects} - ${project}= Get Text ${project} - @{project description}= Split String ${project} - Switch Model Serving Project ${project description}[0] - ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a - FOR ${model} IN @{models} - ${model}= Get Text ${model} - Delete Model Via UI ${model} - Sleep 1s - END - Switch Model Serving Project project_name=All projects +# ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] +# FOR ${project} IN @{projects} +# ${project}= Get Text ${project} +# @{project description}= Split String ${project} +# Switch Model Serving Project ${project description}[0] + ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a + FOR ${model} IN @{models} + ${model}= Get Text ${model} + Delete Model Via UI ${model} + Sleep 1s END +# Switch Model Serving Project project_name=All projects +# END Add Namespace To ServiceMeshMemberRoll [Arguments] ${namespace} ${servicemesh_ns}=istio-system diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 91dbabb40..8a1af8b80 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -13,7 +13,7 @@ Resource ../../Resources/OCP.resource Resource ../../Resources/CLI/ModelServing/modelmesh.resource Suite Setup Cross Auth On Kserve Suite Setup Suite Teardown Cross Auth On Kserve Suite Teardown -Test Tags Kserve Modelmesh +Test Tags Kserve Modelmesh Sanity ProductBug *** Variables *** @@ -26,39 +26,54 @@ ${SECOND_MODEL_NAME}= test-model-second ${RUNTIME_NAME}= Model Serving Test ${EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${SECOND_EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${SECOND_MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable +${SINGLE_MODE_ENABLED}= ${TRUE} +${SINGLE_MODE_DISABLED}= ${FALSE} *** Test Cases *** Test Cross Model Authentication On Kserve [Documentation] Tests for the presence of CVE-2024-7557 when using Kserve - [Tags] Sanity ProductBug - ... RHOAIENG-11007 RHOAIENG-12048 + Set Test Variable ${project_name} ${PRJ_TITLE}-kserve + Template with embedded arguments + ... mode=${SINGLE_MODE_ENABLED} + [Tags] RHOAIENG-11007 RHOAIENG-12048 + +Test Cross Model Authentication On ModelMesh + [Documentation] Tests for the presence of CVE-2024-7557 when using ModelMesh + Set Test Variable ${project_name} ${PRJ_TITLE}-modelmesh + Template with embedded arguments + ... mode=${SINGLE_MODE_DISABLED} + [Tags] RHOAIENG-12314 RHOAIENG-12853 + +*** Keywords *** +Template with embedded arguments + [Arguments] ${mode} Open Data Science Projects Home Page - Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} + Create Data Science Project title=${project_name} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} - Recreate S3 Data Connection project_title=${PRJ_TITLE} dc_name=model-serving-connection + Recreate S3 Data Connection project_title=${project_name} dc_name=model-serving-connection ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} ... aws_bucket_name=ods-ci-s3 Deploy Kserve Model Via UI model_name=${MODEL_NAME} serving_runtime=OpenVINO Model Server ... data_connection=model-serving-connection path=test-dir model_framework=onnx ... service_account_name=first_account token=${TRUE} Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${MODEL_NAME} - ... namespace=${PRJ_TITLE} + ... namespace=${project_name} ${first_token}= Get Model Serving Access Token via UI service_account_name=first_account single_model=${TRUE} ... model_name=${MODEL_NAME} Deploy Kserve Model Via UI model_name=${SECOND_MODEL_NAME} serving_runtime=OpenVINO Model Server ... data_connection=model-serving-connection path=test-dir model_framework=onnx ... service_account_name=second_account token=${TRUE} Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${SECOND_MODEL_NAME} - ... namespace=${PRJ_TITLE} + ... namespace=${project_name} ${second_token}= Get Model Serving Access Token via UI service_account_name=second_account - ... single_model=${TRUE} model_name=${SECOND_MODEL_NAME} + ... single_model=${mode} model_name=${SECOND_MODEL_NAME} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} - ... project_title=${PRJ_TITLE} + ... project_title=${project_name} Verify Model Inference model_name=${SECOND_MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${SECOND_EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${second_token} - ... project_title=${PRJ_TITLE} + ... project_title=${project_name} # Should not be able to query first model with second token # Will fail at this step until CVE is fixed from dashboard side ${inf_out}= Get Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} @@ -67,11 +82,9 @@ Test Cross Model Authentication On Kserve ${inf_out}= Get Model Inference model_name=${SECOND_MODEL_NAME} inference_input=${INFERENCE_INPUT} ... token_auth=${TRUE} token=${first_token} Run Keyword And Warn On Failure Should Contain ${inf_out} Log in with OpenShift - [Teardown] Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs - ... model_name=${MODEL_NAME} project_title=${PRJ_TITLE} AND Clean All Models Of Current User + [Teardown] Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs + ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User - -*** Keywords *** Cross Auth On Kserve Suite Setup [Documentation] Suite setup steps for testing DSG. It creates some test variables ... and runs RHOSi setup @@ -93,7 +106,7 @@ Cross Auth On Kserve Suite Teardown ELSE Log Model not deployed, skipping deletion step during teardown console=true END - ${projects}= Create List ${PRJ_TITLE} + ${projects}= Create List ${project_name} Delete List Of Projects Via CLI ocp_projects=${projects} # Will only be present on SM cluster runs, but keyword passes # if file does not exist From 73b24a5a120050b4a0e66a774de1f5538e2c7435 Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 18 Sep 2024 17:27:26 +0300 Subject: [PATCH 02/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../1008_model_serving_cross_auth.robot | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 8a1af8b80..6d3119088 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -11,8 +11,8 @@ Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Mo Resource ../../Resources/Page/ODH/Monitoring/Monitoring.resource Resource ../../Resources/OCP.resource Resource ../../Resources/CLI/ModelServing/modelmesh.resource -Suite Setup Cross Auth On Kserve Suite Setup -Suite Teardown Cross Auth On Kserve Suite Teardown +#Suite Setup Cross Auth On Kserve Suite Setup +#Suite Teardown Cross Auth On Kserve Suite Teardown Test Tags Kserve Modelmesh Sanity ProductBug @@ -33,21 +33,22 @@ ${SINGLE_MODE_DISABLED}= ${FALSE} *** Test Cases *** Test Cross Model Authentication On Kserve [Documentation] Tests for the presence of CVE-2024-7557 when using Kserve - Set Test Variable ${project_name} ${PRJ_TITLE}-kserve Template with embedded arguments - ... mode=${SINGLE_MODE_ENABLED} + ... project_name=${PRJ_TITLE}-kserve + ... mode=${SINGLE_MODE_ENABLED} [Tags] RHOAIENG-11007 RHOAIENG-12048 Test Cross Model Authentication On ModelMesh [Documentation] Tests for the presence of CVE-2024-7557 when using ModelMesh - Set Test Variable ${project_name} ${PRJ_TITLE}-modelmesh Template with embedded arguments - ... mode=${SINGLE_MODE_DISABLED} + ... project_name=${PRJ_TITLE}-modelmesh + ... mode=${SINGLE_MODE_DISABLED} [Tags] RHOAIENG-12314 RHOAIENG-12853 *** Keywords *** Template with embedded arguments - [Arguments] ${mode} + [Arguments] ${project_name} ${mode} + Cross Auth On Kserve Suite Setup Open Data Science Projects Home Page Create Data Science Project title=${project_name} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} @@ -84,6 +85,7 @@ Template with embedded arguments Run Keyword And Warn On Failure Should Contain ${inf_out} Log in with OpenShift [Teardown] Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User + ... AND Cross Auth On Kserve Suite Teardown project_title=${project_name} Cross Auth On Kserve Suite Setup [Documentation] Suite setup steps for testing DSG. It creates some test variables @@ -99,6 +101,7 @@ Cross Auth On Kserve Suite Setup Cross Auth On Kserve Suite Teardown [Documentation] Suite teardown steps after testing DSG. It Deletes ... all the DS projects created by the tests and run RHOSi teardown + [Arguments] ${project_name} # Even if kw fails, deleting the whole project will also delete the model # Failure will be shown in the logs of the run nonetheless IF ${MODEL_CREATED} From 3f83bc63d5fae9a2bc5b68e64f418c262563c04a Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 23 Sep 2024 18:41:53 +0300 Subject: [PATCH 03/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHModelServing.resource | 2 +- .../1008_model_serving_cross_auth.robot | 45 ++++++++++++------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index b5af18e8b..3ae226f3c 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -399,7 +399,7 @@ Clean Up Model Serving Page Sleep 1s END # Switch Model Serving Project project_name=All projects -# END +# END Add Namespace To ServiceMeshMemberRoll [Arguments] ${namespace} ${servicemesh_ns}=istio-system diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 6d3119088..677019334 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -1,6 +1,6 @@ # robocop: off=too-long-test-case,too-many-calls-in-test-case,wrong-case-in-keyword-name *** Settings *** -Documentation Suite of test cases for OVMS in Kserve +Documentation Suite of test cases for OVMS in Kserve and ModelMesh Library OperatingSystem Library ../../../libs/Helpers.py Resource ../../Resources/Page/ODH/JupyterHub/HighAvailability.robot @@ -13,6 +13,7 @@ Resource ../../Resources/OCP.resource Resource ../../Resources/CLI/ModelServing/modelmesh.resource #Suite Setup Cross Auth On Kserve Suite Setup #Suite Teardown Cross Auth On Kserve Suite Teardown +Test Teardown Run Keywords Cross Auth On Kserve Test Teardown Test Tags Kserve Modelmesh Sanity ProductBug @@ -26,29 +27,32 @@ ${SECOND_MODEL_NAME}= test-model-second ${RUNTIME_NAME}= Model Serving Test ${EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${SECOND_EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${SECOND_MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable -${SINGLE_MODE_ENABLED}= ${TRUE} -${SINGLE_MODE_DISABLED}= ${FALSE} *** Test Cases *** Test Cross Model Authentication On Kserve [Documentation] Tests for the presence of CVE-2024-7557 when using Kserve - Template with embedded arguments - ... project_name=${PRJ_TITLE}-kserve - ... mode=${SINGLE_MODE_ENABLED} [Tags] RHOAIENG-11007 RHOAIENG-12048 + Set Test Variable $serving_mode kserve + Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} + Template with embedded arguments + ... project_name=${project_name} + ... mode=${serving_mode} Test Cross Model Authentication On ModelMesh [Documentation] Tests for the presence of CVE-2024-7557 when using ModelMesh - Template with embedded arguments - ... project_name=${PRJ_TITLE}-modelmesh - ... mode=${SINGLE_MODE_DISABLED} [Tags] RHOAIENG-12314 RHOAIENG-12853 + Set Test Variable $serving_mode modelmeshserving + Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} + Template with embedded arguments + ... project_name=${project_name} + ... mode=${serving_mode} *** Keywords *** Template with embedded arguments [Arguments] ${project_name} ${mode} - Cross Auth On Kserve Suite Setup + ${single_model}= Set Variable If "${mode}" == "kserve" ${True} ${False} + Cross Auth On Kserve Suite Setup mode=${mode} Open Data Science Projects Home Page Create Data Science Project title=${project_name} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} @@ -60,7 +64,7 @@ Template with embedded arguments ... service_account_name=first_account token=${TRUE} Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${MODEL_NAME} ... namespace=${project_name} - ${first_token}= Get Model Serving Access Token via UI service_account_name=first_account single_model=${TRUE} + ${first_token}= Get Model Serving Access Token via UI service_account_name=first_account single_model=${single_model} ... model_name=${MODEL_NAME} Deploy Kserve Model Via UI model_name=${SECOND_MODEL_NAME} serving_runtime=OpenVINO Model Server ... data_connection=model-serving-connection path=test-dir model_framework=onnx @@ -68,7 +72,7 @@ Template with embedded arguments Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${SECOND_MODEL_NAME} ... namespace=${project_name} ${second_token}= Get Model Serving Access Token via UI service_account_name=second_account - ... single_model=${mode} model_name=${SECOND_MODEL_NAME} + ... single_model=${single_model} model_name=${SECOND_MODEL_NAME} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} ... project_title=${project_name} @@ -83,15 +87,16 @@ Template with embedded arguments ${inf_out}= Get Model Inference model_name=${SECOND_MODEL_NAME} inference_input=${INFERENCE_INPUT} ... token_auth=${TRUE} token=${first_token} Run Keyword And Warn On Failure Should Contain ${inf_out} Log in with OpenShift - [Teardown] Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs - ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User - ... AND Cross Auth On Kserve Suite Teardown project_title=${project_name} +# [Teardown] Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs +# ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User +# ... AND Cross Auth On Kserve Suite Teardown project_title=${project_name} Cross Auth On Kserve Suite Setup [Documentation] Suite setup steps for testing DSG. It creates some test variables ... and runs RHOSi setup + [Arguments] ${mode} Set Library Search Order SeleniumLibrary - Skip If Component Is Not Enabled kserve + Skip If Component Is Not Enabled ${mode} RHOSi Setup Launch Dashboard ${TEST_USER.USERNAME} ${TEST_USER.PASSWORD} ${TEST_USER.AUTH_TYPE} ... ${ODH_DASHBOARD_URL} ${BROWSER.NAME} ${BROWSER.OPTIONS} @@ -116,3 +121,11 @@ Cross Auth On Kserve Suite Teardown Remove File openshift_ca_istio_knative.crt SeleniumLibrary.Close All Browsers RHOSi Teardown + + +Cross Auth On Kserve Test Teardown + [Documentation] Test teardown steps after testing DSG. Collects logs and + ... events. + Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs + ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User + ... AND Cross Auth On Kserve Suite Teardown project_title=${project_name} From b82028d338b87a684d75ebac87db97d58932c15d Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 24 Sep 2024 12:16:00 +0300 Subject: [PATCH 04/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHModelServing.resource | 63 +++++++++++-- .../1008_model_serving_cross_auth.robot | 91 ++++++++++--------- 2 files changed, 104 insertions(+), 50 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 3ae226f3c..3e72c11a2 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -26,8 +26,10 @@ ${MS_TABLE_STATUS_SUCCESS}= //span[contains(@class,"pf-v5-c-icon__content")] ${MS_TABLE_STATUS_FAILURE}= //span[contains(@class,"pf-v5-c-icon__content")][contains(@class,"pf-m-danger")] ${KSERVE_MODAL_HEADER}= //header[@class="pf-v5-c-modal-box__header"]/h1[.="Deploy model"] ${KSERVE_RUNTIME_DROPDOWN}= //button[@data-testid="serving-runtime-template-selection"] +${MODELMESH_MODAL_HEADER}= //header[@class="pf-v5-c-modal-box__header"]/h1[.="Add model server"] ${LLM_RESOURCES_DIRPATH}= tests/Resources/Files/llm ${DEPLOY_MODEL_BTN}= //button[contains(@data-testid,"deploy")] +${DEPLOY_MODELMESH_MODEL_BTN}= //button[contains(@data-testid,"add")] ${TOKEN_AUTH_CHECKBOX_XP}= xpath://input[@id="alt-form-checkbox-auth"] @@ -387,19 +389,12 @@ Clean Up Model Serving Page [Documentation] Deletes all currently deployed models, if any are present. # Returns an empty list if no matching elements found Switch Model Serving Project project_name=All projects -# ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] -# FOR ${project} IN @{projects} -# ${project}= Get Text ${project} -# @{project description}= Split String ${project} -# Switch Model Serving Project ${project description}[0] ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a FOR ${model} IN @{models} ${model}= Get Text ${model} Delete Model Via UI ${model} Sleep 1s END -# Switch Model Serving Project project_name=All projects -# END Add Namespace To ServiceMeshMemberRoll [Arguments] ${namespace} ${servicemesh_ns}=istio-system @@ -643,3 +638,57 @@ Set Up Project ELSE Log message=Skipping UserWorkloadMonitoring enablement. END + +Set Modelmesh Model Name + [Documentation] Sets the model name in the "add model server" modal + [Arguments] ${model_name} + Input Text xpath://input[@id="serving-runtime-name-input"] ${model_name} + +Add Model Server Via UI + [Documentation] Add model server (multi-model) using the Data Science Projects UI + [Arguments] ${model_name} ${serving_runtime} + ... ${replicas}=1 ${size}=Small + ... ${token}=${FALSE} ${multi_token}=${FALSE} ${multi_service_account_name}=default-name2 + ... ${public_endpoint}=${TRUE} ${service_account_name}=${NONE} + Move To Tab Models + SeleniumLibrary.Click Button ${DEPLOY_MODELMESH_MODEL_BTN} + SeleniumLibrary.Wait Until Page Contains Element xpath=${MODELMESH_MODAL_HEADER} + Set Modelmesh Model Name ${model_name} + Set Model Server Runtime ${serving_runtime} + Set Replicas Number With Buttons ${replicas} + Set Server Size ${size} + IF ${public_endpoint} + Enable External Serving Route + # By default this also enables Token Authentication. Let's disable it and let it get re-enable by the next + # IF block if needed. + ${token_enabled}= Run Keyword And Return Status SeleniumLibrary.Checkbox Should Be Selected + ... ${TOKEN_AUTH_CHECKBOX_XP} + IF ${token_enabled} + Disable Token Authentication + ELSE + Log Token Authentication was supposed to be automatically enabled, but it wasn't level=ERROR + ... console=${True} + END + END + IF ${token} + Enable Token Authentication service_account_name=${service_account_name} + IF ${multi_token} + Add Another Service Account ${multi_service_account_name} + END + END + Click Button Add + Wait Until Page Does Not Contain Element xpath=${KSERVE_MODAL_HEADER} timeout=60s + +Deploy Model Via UI + [Documentation] Deploy a model using the Data Science Projects UI + [Arguments] ${model_name} ${model_framework} ${data_connection} ${path} + Move To Tab Models + SeleniumLibrary.Click Button ${DEPLOY_MODEL_BTN} + SeleniumLibrary.Wait Until Page Contains Element xpath=${KSERVE_MODAL_HEADER} + Set Model Name ${model_name} + Select Framework ${model_framework} + Select Existing Data Connection ${data_connection} + Set Folder Path ${path} + Click Button Deploy + Wait Until Page Does Not Contain Element xpath=${KSERVE_MODAL_HEADER} timeout=60s + diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 677019334..a2e79a25d 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -11,9 +11,7 @@ Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Mo Resource ../../Resources/Page/ODH/Monitoring/Monitoring.resource Resource ../../Resources/OCP.resource Resource ../../Resources/CLI/ModelServing/modelmesh.resource -#Suite Setup Cross Auth On Kserve Suite Setup -#Suite Teardown Cross Auth On Kserve Suite Teardown -Test Teardown Run Keywords Cross Auth On Kserve Test Teardown +Test Teardown Cross Auth Test Teardown Test Tags Kserve Modelmesh Sanity ProductBug @@ -27,6 +25,8 @@ ${SECOND_MODEL_NAME}= test-model-second ${RUNTIME_NAME}= Model Serving Test ${EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${SECOND_EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${SECOND_MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable +${FIRST_SERVICE_ACCOUNT}= first_account +${SECOND_SERVICE_ACCOUNT}= second_account *** Test Cases *** @@ -36,8 +36,6 @@ Test Cross Model Authentication On Kserve Set Test Variable $serving_mode kserve Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} Template with embedded arguments - ... project_name=${project_name} - ... mode=${serving_mode} Test Cross Model Authentication On ModelMesh [Documentation] Tests for the presence of CVE-2024-7557 when using ModelMesh @@ -45,40 +43,35 @@ Test Cross Model Authentication On ModelMesh Set Test Variable $serving_mode modelmeshserving Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} Template with embedded arguments - ... project_name=${project_name} - ... mode=${serving_mode} *** Keywords *** Template with embedded arguments - [Arguments] ${project_name} ${mode} - ${single_model}= Set Variable If "${mode}" == "kserve" ${True} ${False} - Cross Auth On Kserve Suite Setup mode=${mode} + [Documentation] Template for cross-auth test cases + Cross Auth Test Setup + ${single_model}= Set Variable If "${serving_mode}" == "kserve" ${True} ${False} Open Data Science Projects Home Page Create Data Science Project title=${project_name} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} - Recreate S3 Data Connection project_title=${project_name} dc_name=model-serving-connection - ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} - ... aws_bucket_name=ods-ci-s3 - Deploy Kserve Model Via UI model_name=${MODEL_NAME} serving_runtime=OpenVINO Model Server - ... data_connection=model-serving-connection path=test-dir model_framework=onnx - ... service_account_name=first_account token=${TRUE} - Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${MODEL_NAME} - ... namespace=${project_name} - ${first_token}= Get Model Serving Access Token via UI service_account_name=first_account single_model=${single_model} + Cross Auth Model Deployment single_model=${single_model} + ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} + + ${first_token}= Get Model Serving Access Token via UI service_account_name=${FIRST_SERVICE_ACCOUNT} single_model=${single_model} ... model_name=${MODEL_NAME} - Deploy Kserve Model Via UI model_name=${SECOND_MODEL_NAME} serving_runtime=OpenVINO Model Server - ... data_connection=model-serving-connection path=test-dir model_framework=onnx - ... service_account_name=second_account token=${TRUE} - Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${SECOND_MODEL_NAME} - ... namespace=${project_name} - ${second_token}= Get Model Serving Access Token via UI service_account_name=second_account + + Cross Auth Model Deployment single_model=${single_model} + ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} + + ${second_token}= Get Model Serving Access Token via UI service_account_name=${SECOND_SERVICE_ACCOUNT} ... single_model=${single_model} model_name=${SECOND_MODEL_NAME} + Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} ... project_title=${project_name} + Verify Model Inference model_name=${SECOND_MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${SECOND_EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${second_token} ... project_title=${project_name} + # Should not be able to query first model with second token # Will fail at this step until CVE is fixed from dashboard side ${inf_out}= Get Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} @@ -87,26 +80,46 @@ Template with embedded arguments ${inf_out}= Get Model Inference model_name=${SECOND_MODEL_NAME} inference_input=${INFERENCE_INPUT} ... token_auth=${TRUE} token=${first_token} Run Keyword And Warn On Failure Should Contain ${inf_out} Log in with OpenShift -# [Teardown] Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs -# ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User -# ... AND Cross Auth On Kserve Suite Teardown project_title=${project_name} -Cross Auth On Kserve Suite Setup - [Documentation] Suite setup steps for testing DSG. It creates some test variables +Cross Auth Model Deployment + [Documentation] Deploys a model with cross auth enabled + [Arguments] ${single_model} ${model_name} ${service_account_name} + ${dc_name}= Set Variable model-serving-connection-${serving_mode} + Recreate S3 Data Connection project_title=${project_name} dc_name=${dc_name} + ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} + ... aws_bucket_name=ods-ci-s3 + IF ${single_model} == ${TRUE} + Deploy Kserve Model Via UI model_name=${model_name} serving_runtime=OpenVINO Model Server + ... data_connection=${dc_name} path=test-dir model_framework=onnx + ... service_account_name=${service_account_name} token=${TRUE} + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} namespace=${project_name} + ELSE + Add Model Server Via UI model_name=${model_name} serving_runtime=OpenVINO Model Server + ... service_account_name=${service_account_name} token=${TRUE} + Deploy Model Via UI model_name=${model_name} model_framework=onnx + ... data_connection=${dc_name} path=test-dir + Wait For Pods To Be Ready label_selector=name=modelmesh-serving-${model_name} + ... namespace=${project_name} + END + +Cross Auth Test Setup + [Documentation] Test setup steps for testing DSG. It creates some test variables ... and runs RHOSi setup - [Arguments] ${mode} + Set Library Search Order SeleniumLibrary - Skip If Component Is Not Enabled ${mode} + Skip If Component Is Not Enabled ${serving_mode} RHOSi Setup Launch Dashboard ${TEST_USER.USERNAME} ${TEST_USER.PASSWORD} ${TEST_USER.AUTH_TYPE} ... ${ODH_DASHBOARD_URL} ${BROWSER.NAME} ${BROWSER.OPTIONS} Fetch Knative CA Certificate filename=openshift_ca_istio_knative.crt Clean All Models Of Current User -Cross Auth On Kserve Suite Teardown - [Documentation] Suite teardown steps after testing DSG. It Deletes +Cross Auth Test Teardown + [Documentation] Test teardown steps after testing DSG. It Deletes ... all the DS projects created by the tests and run RHOSi teardown - [Arguments] ${project_name} + Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs + ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User + # Even if kw fails, deleting the whole project will also delete the model # Failure will be shown in the logs of the run nonetheless IF ${MODEL_CREATED} @@ -121,11 +134,3 @@ Cross Auth On Kserve Suite Teardown Remove File openshift_ca_istio_knative.crt SeleniumLibrary.Close All Browsers RHOSi Teardown - - -Cross Auth On Kserve Test Teardown - [Documentation] Test teardown steps after testing DSG. Collects logs and - ... events. - Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs - ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User - ... AND Cross Auth On Kserve Suite Teardown project_title=${project_name} From 8000cbb38cf621d1ec8a34c6d5116e0e5b717d48 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 24 Sep 2024 12:19:28 +0300 Subject: [PATCH 05/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 1 + 1 file changed, 1 insertion(+) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 3e72c11a2..3bed6c4c6 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -642,6 +642,7 @@ Set Up Project Set Modelmesh Model Name [Documentation] Sets the model name in the "add model server" modal [Arguments] ${model_name} + Log To Console ${model_name} xxxxxxx Input Text xpath://input[@id="serving-runtime-name-input"] ${model_name} Add Model Server Via UI From 2c0720d61b333b1fb6ce2a1de33b30205cd7c798 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 24 Sep 2024 12:21:11 +0300 Subject: [PATCH 06/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 1 - 1 file changed, 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 3bed6c4c6..3e72c11a2 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -642,7 +642,6 @@ Set Up Project Set Modelmesh Model Name [Documentation] Sets the model name in the "add model server" modal [Arguments] ${model_name} - Log To Console ${model_name} xxxxxxx Input Text xpath://input[@id="serving-runtime-name-input"] ${model_name} Add Model Server Via UI From 118abdd8ba519c43e0c73924d4d527c281fd788b Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 24 Sep 2024 18:55:30 +0300 Subject: [PATCH 07/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHModelServing.resource | 56 ------------------- .../1008_model_serving_cross_auth.robot | 30 +++++----- 2 files changed, 17 insertions(+), 69 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 3e72c11a2..6476c74ad 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -26,10 +26,8 @@ ${MS_TABLE_STATUS_SUCCESS}= //span[contains(@class,"pf-v5-c-icon__content")] ${MS_TABLE_STATUS_FAILURE}= //span[contains(@class,"pf-v5-c-icon__content")][contains(@class,"pf-m-danger")] ${KSERVE_MODAL_HEADER}= //header[@class="pf-v5-c-modal-box__header"]/h1[.="Deploy model"] ${KSERVE_RUNTIME_DROPDOWN}= //button[@data-testid="serving-runtime-template-selection"] -${MODELMESH_MODAL_HEADER}= //header[@class="pf-v5-c-modal-box__header"]/h1[.="Add model server"] ${LLM_RESOURCES_DIRPATH}= tests/Resources/Files/llm ${DEPLOY_MODEL_BTN}= //button[contains(@data-testid,"deploy")] -${DEPLOY_MODELMESH_MODEL_BTN}= //button[contains(@data-testid,"add")] ${TOKEN_AUTH_CHECKBOX_XP}= xpath://input[@id="alt-form-checkbox-auth"] @@ -638,57 +636,3 @@ Set Up Project ELSE Log message=Skipping UserWorkloadMonitoring enablement. END - -Set Modelmesh Model Name - [Documentation] Sets the model name in the "add model server" modal - [Arguments] ${model_name} - Input Text xpath://input[@id="serving-runtime-name-input"] ${model_name} - -Add Model Server Via UI - [Documentation] Add model server (multi-model) using the Data Science Projects UI - [Arguments] ${model_name} ${serving_runtime} - ... ${replicas}=1 ${size}=Small - ... ${token}=${FALSE} ${multi_token}=${FALSE} ${multi_service_account_name}=default-name2 - ... ${public_endpoint}=${TRUE} ${service_account_name}=${NONE} - Move To Tab Models - SeleniumLibrary.Click Button ${DEPLOY_MODELMESH_MODEL_BTN} - SeleniumLibrary.Wait Until Page Contains Element xpath=${MODELMESH_MODAL_HEADER} - Set Modelmesh Model Name ${model_name} - Set Model Server Runtime ${serving_runtime} - Set Replicas Number With Buttons ${replicas} - Set Server Size ${size} - IF ${public_endpoint} - Enable External Serving Route - # By default this also enables Token Authentication. Let's disable it and let it get re-enable by the next - # IF block if needed. - ${token_enabled}= Run Keyword And Return Status SeleniumLibrary.Checkbox Should Be Selected - ... ${TOKEN_AUTH_CHECKBOX_XP} - IF ${token_enabled} - Disable Token Authentication - ELSE - Log Token Authentication was supposed to be automatically enabled, but it wasn't level=ERROR - ... console=${True} - END - END - IF ${token} - Enable Token Authentication service_account_name=${service_account_name} - IF ${multi_token} - Add Another Service Account ${multi_service_account_name} - END - END - Click Button Add - Wait Until Page Does Not Contain Element xpath=${KSERVE_MODAL_HEADER} timeout=60s - -Deploy Model Via UI - [Documentation] Deploy a model using the Data Science Projects UI - [Arguments] ${model_name} ${model_framework} ${data_connection} ${path} - Move To Tab Models - SeleniumLibrary.Click Button ${DEPLOY_MODEL_BTN} - SeleniumLibrary.Wait Until Page Contains Element xpath=${KSERVE_MODAL_HEADER} - Set Model Name ${model_name} - Select Framework ${model_framework} - Select Existing Data Connection ${data_connection} - Set Folder Path ${path} - Click Button Deploy - Wait Until Page Does Not Contain Element xpath=${KSERVE_MODAL_HEADER} timeout=60s - diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index a2e79a25d..4060dd76a 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -12,7 +12,7 @@ Resource ../../Resources/Page/ODH/Monitoring/Monitoring.resource Resource ../../Resources/OCP.resource Resource ../../Resources/CLI/ModelServing/modelmesh.resource Test Teardown Cross Auth Test Teardown -Test Tags Kserve Modelmesh Sanity ProductBug +Test Tags Sanity ProductBug *** Variables *** @@ -32,14 +32,14 @@ ${SECOND_SERVICE_ACCOUNT}= second_account *** Test Cases *** Test Cross Model Authentication On Kserve [Documentation] Tests for the presence of CVE-2024-7557 when using Kserve - [Tags] RHOAIENG-11007 RHOAIENG-12048 + [Tags] Kserve RHOAIENG-11007 RHOAIENG-12048 Set Test Variable $serving_mode kserve Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} Template with embedded arguments Test Cross Model Authentication On ModelMesh [Documentation] Tests for the presence of CVE-2024-7557 when using ModelMesh - [Tags] RHOAIENG-12314 RHOAIENG-12853 + [Tags] ModelMesh RHOAIENG-11007 RHOAIENG-12853 Set Test Variable $serving_mode modelmeshserving Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} Template with embedded arguments @@ -55,14 +55,14 @@ Template with embedded arguments Cross Auth Model Deployment single_model=${single_model} ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} - ${first_token}= Get Model Serving Access Token via UI service_account_name=${FIRST_SERVICE_ACCOUNT} single_model=${single_model} - ... model_name=${MODEL_NAME} + ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} single_model=${single_model} + ... model_name=${MODEL_NAME} project_name=${project_name} Cross Auth Model Deployment single_model=${single_model} ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} - ${second_token}= Get Model Serving Access Token via UI service_account_name=${SECOND_SERVICE_ACCOUNT} - ... single_model=${single_model} model_name=${SECOND_MODEL_NAME} + ${second_token}= Get Access Token Via UI single_model=${single_model} + ... model_name=${MODEL_NAME} project_name=${project_name} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} @@ -94,12 +94,16 @@ Cross Auth Model Deployment ... service_account_name=${service_account_name} token=${TRUE} Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} namespace=${project_name} ELSE - Add Model Server Via UI model_name=${model_name} serving_runtime=OpenVINO Model Server - ... service_account_name=${service_account_name} token=${TRUE} - Deploy Model Via UI model_name=${model_name} model_framework=onnx - ... data_connection=${dc_name} path=test-dir - Wait For Pods To Be Ready label_selector=name=modelmesh-serving-${model_name} - ... namespace=${project_name} + Create Model Server token=${TRUE} server_name=${model_name} + Serve Model project_name=${project_name} model_name=${model_name} framework=onnx + ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} + ... model_path=openvino-example-model + ${runtime_pod_name}= Replace String Using Regexp string=${project_name} pattern=\\s replace_with=- + ${runtime_pod_name}= Convert To Lower Case ${runtime_pod_name} + Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${model_name} + ... project_name=${project_name} + Wait Until Keyword Succeeds 5 min 10 sec Verify Serving Service project_name=${project_name} + Verify Model Status ${MODEL_NAME} success END Cross Auth Test Setup From f7d526c10ca470593266f1924f45c08c68ffa0cd Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 25 Sep 2024 14:43:23 +0300 Subject: [PATCH 08/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDataScienceProject/ModelServer.resource | 12 +++++++----- .../1008_model_serving_cross_auth.robot | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 2333463d4..484b01c3d 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -33,7 +33,7 @@ Create Model Server [Documentation] Keyword to create a Model Server in a Data Science Project [Arguments] ${no_replicas}=1 ${server_size}=Small ${ext_route}=${TRUE} ... ${token}=${TRUE} ${runtime}=OpenVINO Model Server ${server_name}=Model Serving Test - ... ${no_gpus}=0 ${existing_server}=${FALSE} + ... ${no_gpus}=0 ${existing_server}=${FALSE} ${service_account_name}=${NONE} Move To Tab Models IF ${existing_server} ${existing_server}= Run Keyword And Return Status Wait Until Page Contains Element //button[.="${server_name}"] @@ -63,8 +63,9 @@ Create Model Server Log message=Token Auth should be enabled by default..(from v2.5) SeleniumLibrary.Checkbox Should Be Selected ${TOKEN_AUTH_CHECKBOX_XP} END - ELSE IF ${token}==${TRUE} - Enable Token Authentication + END + IF ${token}==${TRUE} + Enable Token Authentication service_account_name=${service_account_name} END SeleniumLibrary.Wait Until Element Is Enabled //button[contains(text(),"Add")] SeleniumLibrary.Click Button Add @@ -181,9 +182,10 @@ Enable Token Authentication [Documentation] Enables Token authentication to serving route [Arguments] ${service_account_name}=${NONE} SeleniumLibrary.Select Checkbox ${TOKEN_AUTH_CHECKBOX_XP} - IF "${service_account_name}" != "${NONE}" - Input Service Account Name ${service_account_name} + IF "${service_account_name}" == "${NONE}" + ${service_account_name}= Set Variable default-name END + Input Service Account Name ${service_account_name} Disable Token Authentication [Documentation] Disable Token authentication to serving route diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 4060dd76a..56fe8079c 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -88,13 +88,14 @@ Cross Auth Model Deployment Recreate S3 Data Connection project_title=${project_name} dc_name=${dc_name} ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} ... aws_bucket_name=ods-ci-s3 + Open Data Science Project Details Page ${project_name} tab_id=model-server IF ${single_model} == ${TRUE} Deploy Kserve Model Via UI model_name=${model_name} serving_runtime=OpenVINO Model Server ... data_connection=${dc_name} path=test-dir model_framework=onnx ... service_account_name=${service_account_name} token=${TRUE} Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} namespace=${project_name} ELSE - Create Model Server token=${TRUE} server_name=${model_name} + Create Model Server token=${TRUE} server_name=${model_name} service_account_name=${service_account_name} Serve Model project_name=${project_name} model_name=${model_name} framework=onnx ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} ... model_path=openvino-example-model From 4048abffc62d1b33674df42bc0fc0d722777be1f Mon Sep 17 00:00:00 2001 From: rnetser Date: Wed, 25 Sep 2024 16:04:59 +0300 Subject: [PATCH 09/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDataScienceProject/ModelServer.resource | 2 +- .../ODH/ODHDashboard/ODHModelServing.resource | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 484b01c3d..984b45cea 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -183,7 +183,7 @@ Enable Token Authentication [Arguments] ${service_account_name}=${NONE} SeleniumLibrary.Select Checkbox ${TOKEN_AUTH_CHECKBOX_XP} IF "${service_account_name}" == "${NONE}" - ${service_account_name}= Set Variable default-name + ${service_account_name} default-name END Input Service Account Name ${service_account_name} diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 6476c74ad..66a86b725 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -387,11 +387,18 @@ Clean Up Model Serving Page [Documentation] Deletes all currently deployed models, if any are present. # Returns an empty list if no matching elements found Switch Model Serving Project project_name=All projects - ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a - FOR ${model} IN @{models} - ${model}= Get Text ${model} - Delete Model Via UI ${model} - Sleep 1s + ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] + FOR ${project} IN @{projects} + ${project}= Get Text ${project} + @{project description}= Split String ${project} + Switch Model Serving Project ${project description}[0] + ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a + FOR ${model} IN @{models} + ${model}= Get Text ${model} + Delete Model Via UI ${model} + Sleep 1s + END + Switch Model Serving Project project_name=All projects END Add Namespace To ServiceMeshMemberRoll From 1e8f59b78cb2fff12962e3217a4cd28d8c2f25c4 Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 26 Sep 2024 07:43:11 +0300 Subject: [PATCH 10/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 1 + .../1000__model_serving/1008_model_serving_cross_auth.robot | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 66a86b725..4e1db2ec5 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -235,6 +235,7 @@ Open ${section} Options Menu ... Valid sections are: "Name", "Model framework (name - version)", "Project" ${optionsmenu}= Run Keyword And Return Status Page Should Contain Element ... xpath://span[.="${section}"]/../../..//button[@aria-label="Options menu"] + Sleep 5s IF ${optionsmenu} == ${TRUE} SeleniumLibrary.Click Element xpath://span[.="${section}"]/../../..//button[@aria-label="Options menu"] # Remove attribute which is not present anymore ELSE diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 56fe8079c..36ef57e05 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -61,7 +61,7 @@ Template with embedded arguments Cross Auth Model Deployment single_model=${single_model} ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} - ${second_token}= Get Access Token Via UI single_model=${single_model} + ${second_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} single_model=${single_model} ... model_name=${MODEL_NAME} project_name=${project_name} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} From 15c0e394796aa92d871bda0d71eb8489e625040e Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 26 Sep 2024 08:17:15 +0300 Subject: [PATCH 11/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHModelServing.resource | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 4e1db2ec5..63a9b8aa5 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -388,18 +388,11 @@ Clean Up Model Serving Page [Documentation] Deletes all currently deployed models, if any are present. # Returns an empty list if no matching elements found Switch Model Serving Project project_name=All projects - ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] - FOR ${project} IN @{projects} - ${project}= Get Text ${project} - @{project description}= Split String ${project} - Switch Model Serving Project ${project description}[0] - ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a - FOR ${model} IN @{models} - ${model}= Get Text ${model} - Delete Model Via UI ${model} - Sleep 1s - END - Switch Model Serving Project project_name=All projects + ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a + FOR ${model} IN @{models} + ${model}= Get Text ${model} + Delete Model Via UI ${model} + Sleep 1s END Add Namespace To ServiceMeshMemberRoll From 35fe0d1237788284d931c51f45130d11ea93beb9 Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 26 Sep 2024 10:02:57 +0300 Subject: [PATCH 12/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource | 2 +- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 984b45cea..73187ec91 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -183,7 +183,7 @@ Enable Token Authentication [Arguments] ${service_account_name}=${NONE} SeleniumLibrary.Select Checkbox ${TOKEN_AUTH_CHECKBOX_XP} IF "${service_account_name}" == "${NONE}" - ${service_account_name} default-name + ${service_account_name} = Set Variable default-name END Input Service Account Name ${service_account_name} diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 63a9b8aa5..507af8239 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -233,6 +233,7 @@ Get Model Route Via CLI Open ${section} Options Menu [Documentation] Opens the "Options menu" dropdown for different sections ... Valid sections are: "Name", "Model framework (name - version)", "Project" + ${optionsmenu}= Run Keyword And Return Status Page Should Contain Element ... xpath://span[.="${section}"]/../../..//button[@aria-label="Options menu"] Sleep 5s From 62cb00ec21c26b9f013d3e3977235b52447dbaf5 Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 26 Sep 2024 20:58:23 +0300 Subject: [PATCH 13/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHModelServing.resource | 59 ++++++++++++------- .../1008_model_serving_cross_auth.robot | 4 +- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 507af8239..1a6fe5731 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -7,6 +7,7 @@ Resource ODHDataScienceProject/ModelServer.resource Resource ODHDataScienceProject/DataConnections.resource Resource ../../../CLI/ModelServing/llm.resource Library ../../../../../libs/Helpers.py +Library RPA.Salesforce *** Variables *** @@ -75,26 +76,8 @@ Serve Model SeleniumLibrary.Click Button ${DEPLOY_MODEL_BTN} END SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] - Set Model Name ${model_name} - Select Model Server ${model_server} - IF "${serving_runtime}" != "${NONE}" Set Model Server Runtime ${serving_runtime} - SeleniumLibrary.Wait Until Page Contains Element xpath://span[.="Model framework (name - version)"] - Select Framework ${framework} - IF ${existing_data_connection}==${TRUE} - # Select Radio Button group_name=radiogroup value=existing-data-connection-radio - # Selected by default, let's skip for now - Select Existing Data Connection ${data_connection_name} - Set Folder Path ${model_path} - ELSE - # Existing connection radio is selected by default; for now blindly click on new connection radio - SeleniumLibrary.Click Element //input[@id="new-data-connection-radio"] - # Select Radio Button group_name=radiogroup value=new-data-connection-radio - Set Up New Data Connection dc_name=${data_connection_name} - Set Folder Path ${model_path} - END - SeleniumLibrary.Wait Until Element Is Enabled //button[contains(text(),"Deploy")] - SeleniumLibrary.Click Button Deploy - SeleniumLibrary.Wait Until Page Does Not Contain xpath://h1[.="Deploy model"] + Fill Deploy Model Form model_name=${model_name} model_server=${model_server} + ... serving_runtime=${serving_runtime} framework=${framework} Select Project [Documentation] Selects a project in the "deploy model" modal. @@ -236,7 +219,6 @@ Open ${section} Options Menu ${optionsmenu}= Run Keyword And Return Status Page Should Contain Element ... xpath://span[.="${section}"]/../../..//button[@aria-label="Options menu"] - Sleep 5s IF ${optionsmenu} == ${TRUE} SeleniumLibrary.Click Element xpath://span[.="${section}"]/../../..//button[@aria-label="Options menu"] # Remove attribute which is not present anymore ELSE @@ -638,3 +620,38 @@ Set Up Project ELSE Log message=Skipping UserWorkloadMonitoring enablement. END + +Deploy Model From Models Tab + [Documentation] Deploys a model from the project's Models tab + [Arguments] ${project_name} ${data_connection_name} ${model_name} + ... ${framework} ${existing_data_connection} ${model_path} ${model_server} + ... ${serving_runtime}=${NONE} + Open Data Science Project Details Page ${project_name} tab_id=model-server + SeleniumLibrary.Click Button xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@class='pf-v5-c-table__td']//button + SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] + Fill Deploy Model Form model_name=${model_name} model_server=${model_server} + ... serving_runtime=${serving_runtime} framework=${framework} + +Fill Deploy Model Form + [Documentation] Fills the deploy model form + [Arguments] ${model_name} ${model_server} ${serving_runtime} ${framework} + Set Model Name ${model_name} + Select Model Server ${model_server} + IF "${serving_runtime}" != "${NONE}" Set Model Server Runtime ${serving_runtime} + SeleniumLibrary.Wait Until Page Contains Element xpath://span[.="Model framework (name - version)"] + Select Framework ${framework} + IF ${existing_data_connection}==${TRUE} + # Select Radio Button group_name=radiogroup value=existing-data-connection-radio + # Selected by default, let's skip for now + Select Existing Data Connection ${data_connection_name} + Set Folder Path ${model_path} + ELSE + # Existing connection radio is selected by default; for now blindly click on new connection radio + SeleniumLibrary.Click Element //input[@id="new-data-connection-radio"] + # Select Radio Button group_name=radiogroup value=new-data-connection-radio + Set Up New Data Connection dc_name=${data_connection_name} + Set Folder Path ${model_path} + END + SeleniumLibrary.Wait Until Element Is Enabled //button[contains(text(),"Deploy")] + SeleniumLibrary.Click Button Deploy + SeleniumLibrary.Wait Until Page Does Not Contain xpath://h1[.="Deploy model"] diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 36ef57e05..8a745b516 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -61,7 +61,7 @@ Template with embedded arguments Cross Auth Model Deployment single_model=${single_model} ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} - ${second_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} single_model=${single_model} + ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} single_model=${single_model} ... model_name=${MODEL_NAME} project_name=${project_name} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} @@ -96,7 +96,7 @@ Cross Auth Model Deployment Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} namespace=${project_name} ELSE Create Model Server token=${TRUE} server_name=${model_name} service_account_name=${service_account_name} - Serve Model project_name=${project_name} model_name=${model_name} framework=onnx + Deploy Model From Models Tab project_name=${project_name} model_name=${model_name} framework=onnx ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} ... model_path=openvino-example-model ${runtime_pod_name}= Replace String Using Regexp string=${project_name} pattern=\\s replace_with=- From ccd8051696890bb73814665a9d4601bd51200b64 Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 26 Sep 2024 21:02:39 +0300 Subject: [PATCH 14/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 1a6fe5731..c6f33303e 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -78,6 +78,8 @@ Serve Model SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] Fill Deploy Model Form model_name=${model_name} model_server=${model_server} ... serving_runtime=${serving_runtime} framework=${framework} + ... data_connection_name=${data_connection_name} existing_data_connection=${existing_data_connection} + ... model_path=${model_path} Select Project [Documentation] Selects a project in the "deploy model" modal. @@ -631,10 +633,13 @@ Deploy Model From Models Tab SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] Fill Deploy Model Form model_name=${model_name} model_server=${model_server} ... serving_runtime=${serving_runtime} framework=${framework} + ... data_connection_name=${data_connection_name} existing_data_connection=${existing_data_connection} + ... model_path=${model_path} Fill Deploy Model Form [Documentation] Fills the deploy model form [Arguments] ${model_name} ${model_server} ${serving_runtime} ${framework} + ... ${data_connection_name} ${existing_data_connection} ${model_path} Set Model Name ${model_name} Select Model Server ${model_server} IF "${serving_runtime}" != "${NONE}" Set Model Server Runtime ${serving_runtime} From a8d618ad79837b85d2b56a4f515e8a67a84cd85f Mon Sep 17 00:00:00 2001 From: rnetser Date: Thu, 26 Sep 2024 21:07:02 +0300 Subject: [PATCH 15/32] add modelmesh tests Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../Page/ODH/ODHDashboard/ODHModelServing.resource | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index c6f33303e..93a71b309 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -7,7 +7,6 @@ Resource ODHDataScienceProject/ModelServer.resource Resource ODHDataScienceProject/DataConnections.resource Resource ../../../CLI/ModelServing/llm.resource Library ../../../../../libs/Helpers.py -Library RPA.Salesforce *** Variables *** @@ -76,7 +75,7 @@ Serve Model SeleniumLibrary.Click Button ${DEPLOY_MODEL_BTN} END SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] - Fill Deploy Model Form model_name=${model_name} model_server=${model_server} + Fill Deploy Model Form model_name=${model_name} model_server=${model_server} ... serving_runtime=${serving_runtime} framework=${framework} ... data_connection_name=${data_connection_name} existing_data_connection=${existing_data_connection} ... model_path=${model_path} @@ -218,7 +217,6 @@ Get Model Route Via CLI Open ${section} Options Menu [Documentation] Opens the "Options menu" dropdown for different sections ... Valid sections are: "Name", "Model framework (name - version)", "Project" - ${optionsmenu}= Run Keyword And Return Status Page Should Contain Element ... xpath://span[.="${section}"]/../../..//button[@aria-label="Options menu"] IF ${optionsmenu} == ${TRUE} @@ -631,7 +629,7 @@ Deploy Model From Models Tab Open Data Science Project Details Page ${project_name} tab_id=model-server SeleniumLibrary.Click Button xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@class='pf-v5-c-table__td']//button SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] - Fill Deploy Model Form model_name=${model_name} model_server=${model_server} + Fill Deploy Model Form model_name=${model_name} model_server=${model_server} ... serving_runtime=${serving_runtime} framework=${framework} ... data_connection_name=${data_connection_name} existing_data_connection=${existing_data_connection} ... model_path=${model_path} From bcabc3c670b3933896d43f733709def7021e91ba Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 29 Sep 2024 09:50:08 +0300 Subject: [PATCH 16/32] resolve robocop comments Signed-off-by: rnetser rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ModelServer.resource | 16 ++++++++---- .../ODH/ODHDashboard/ODHModelServing.resource | 26 ++++++++++++++----- .../1008_model_serving_cross_auth.robot | 16 ++++++------ 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 73187ec91..edeef90da 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -31,9 +31,15 @@ ${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]// *** Keywords *** Create Model Server [Documentation] Keyword to create a Model Server in a Data Science Project - [Arguments] ${no_replicas}=1 ${server_size}=Small ${ext_route}=${TRUE} - ... ${token}=${TRUE} ${runtime}=OpenVINO Model Server ${server_name}=Model Serving Test - ... ${no_gpus}=0 ${existing_server}=${FALSE} ${service_account_name}=${NONE} + [Arguments] ${no_replicas}=1 + ... ${server_size}=Small + ... ${ext_route}=${TRUE} + ... ${token}=${TRUE} + ... ${runtime}=OpenVINO Model Server + ... ${server_name}=Model Serving Test + ... ${no_gpus}=0 + ... ${existing_server}=${FALSE} + ... ${service_account_name}=${NONE} Move To Tab Models IF ${existing_server} ${existing_server}= Run Keyword And Return Status Wait Until Page Contains Element //button[.="${server_name}"] @@ -64,7 +70,7 @@ Create Model Server SeleniumLibrary.Checkbox Should Be Selected ${TOKEN_AUTH_CHECKBOX_XP} END END - IF ${token}==${TRUE} + IF ${token} Enable Token Authentication service_account_name=${service_account_name} END SeleniumLibrary.Wait Until Element Is Enabled //button[contains(text(),"Add")] @@ -183,7 +189,7 @@ Enable Token Authentication [Arguments] ${service_account_name}=${NONE} SeleniumLibrary.Select Checkbox ${TOKEN_AUTH_CHECKBOX_XP} IF "${service_account_name}" == "${NONE}" - ${service_account_name} = Set Variable default-name + ${service_account_name}=default-name END Input Service Account Name ${service_account_name} diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 93a71b309..f2e02b3b2 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -623,11 +623,17 @@ Set Up Project Deploy Model From Models Tab [Documentation] Deploys a model from the project's Models tab - [Arguments] ${project_name} ${data_connection_name} ${model_name} - ... ${framework} ${existing_data_connection} ${model_path} ${model_server} + [Arguments] ${project_name} + ... ${data_connection_name} + ... ${model_name} + ... ${framework} + ... ${existing_data_connection} + ... ${model_path} + ... ${model_server} ... ${serving_runtime}=${NONE} Open Data Science Project Details Page ${project_name} tab_id=model-server - SeleniumLibrary.Click Button xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@class='pf-v5-c-table__td']//button + SeleniumLibrary.Click Button + ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@class='pf-v5-c-table__td']//button SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] Fill Deploy Model Form model_name=${model_name} model_server=${model_server} ... serving_runtime=${serving_runtime} framework=${framework} @@ -636,14 +642,20 @@ Deploy Model From Models Tab Fill Deploy Model Form [Documentation] Fills the deploy model form - [Arguments] ${model_name} ${model_server} ${serving_runtime} ${framework} - ... ${data_connection_name} ${existing_data_connection} ${model_path} + [Arguments] ${model_name} + ... ${model_server} + ... ${serving_runtime} + ... ${framework} + ... ${data_connection_name} + ... ${existing_data_connection} + ... ${model_path} Set Model Name ${model_name} Select Model Server ${model_server} IF "${serving_runtime}" != "${NONE}" Set Model Server Runtime ${serving_runtime} - SeleniumLibrary.Wait Until Page Contains Element xpath://span[.="Model framework (name - version)"] + SeleniumLibrary.Wait Until Page Contains Element + ... xpath://span[.="Model framework (name - version)"] Select Framework ${framework} - IF ${existing_data_connection}==${TRUE} + IF ${existing_data_connection} # Select Radio Button group_name=radiogroup value=existing-data-connection-radio # Selected by default, let's skip for now Select Existing Data Connection ${data_connection_name} diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 8a745b516..b8d85d913 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -44,6 +44,7 @@ Test Cross Model Authentication On ModelMesh Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} Template with embedded arguments + *** Keywords *** Template with embedded arguments [Documentation] Template for cross-auth test cases @@ -55,14 +56,14 @@ Template with embedded arguments Cross Auth Model Deployment single_model=${single_model} ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} - ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} single_model=${single_model} - ... model_name=${MODEL_NAME} project_name=${project_name} + ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} + ... single_model=${single_model} model_name=${MODEL_NAME} project_name=${project_name} Cross Auth Model Deployment single_model=${single_model} ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} - ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} single_model=${single_model} - ... model_name=${MODEL_NAME} project_name=${project_name} + ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} + ... single_model=${single_model} model_name=${MODEL_NAME} project_name=${project_name} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} @@ -89,18 +90,17 @@ Cross Auth Model Deployment ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} ... aws_bucket_name=ods-ci-s3 Open Data Science Project Details Page ${project_name} tab_id=model-server - IF ${single_model} == ${TRUE} + IF ${single_model} Deploy Kserve Model Via UI model_name=${model_name} serving_runtime=OpenVINO Model Server ... data_connection=${dc_name} path=test-dir model_framework=onnx ... service_account_name=${service_account_name} token=${TRUE} - Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} namespace=${project_name} + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} + ... namespace=${project_name} ELSE Create Model Server token=${TRUE} server_name=${model_name} service_account_name=${service_account_name} Deploy Model From Models Tab project_name=${project_name} model_name=${model_name} framework=onnx ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} ... model_path=openvino-example-model - ${runtime_pod_name}= Replace String Using Regexp string=${project_name} pattern=\\s replace_with=- - ${runtime_pod_name}= Convert To Lower Case ${runtime_pod_name} Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${model_name} ... project_name=${project_name} Wait Until Keyword Succeeds 5 min 10 sec Verify Serving Service project_name=${project_name} From dba77d17df2707945eea723104cfee55915b0689 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 29 Sep 2024 19:39:26 +0300 Subject: [PATCH 17/32] fix comments rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDashboard/ODHDataScienceProject/ModelServer.resource | 3 ++- .../ODH/ODHDashboard/ODHDataScienceProject/Projects.resource | 5 +---- .../1000__model_serving/1008_model_serving_cross_auth.robot | 5 +++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index edeef90da..4512f4189 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -226,7 +226,8 @@ Get Model Serving Access Token via UI ${token}= Get Single Model Token ${service_account_name} ELSE SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button - SeleniumLibrary.Click Element xpath://td[@data-label="Tokens"]/button + SeleniumLibrary.Click Button + ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button ${token}= SeleniumLibrary.Get Element Attribute ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value END diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource index d8085440c..a1b18a625 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource @@ -8,8 +8,7 @@ Resource ./Workbenches.resource ${DS_PROJECT_TITLE}= Data Science Projects ${TITLE_INPUT_XP}= xpath=//input[@id="manage-project-modal-name"] ${DESCR_INPUT_XP}= xpath=//textarea[@id="manage-project-modal-description"] -${RESOURCE_EDIT_BTN_XP}= xpath=//button[.="Edit resource name"] -${RESOURCE_INPUT_XP}= css:[data-testid="manage-project-modal-resourceName"] +${RESOURCE_INPUT_XP}= css:[data-testid="resource-manage-project-modal-name"] ${GENERIC_CREATE_BTN_XP}= xpath=//button[text()="Create"] ${GENERIC_CANCEL_BTN_XP}= xpath=//button[text()="Cancel"] # TODO: Update to latter option once the change is pulled from ODH into downstream! @@ -137,8 +136,6 @@ Create Data Science Project ELSE Wait Until Page Contains Element ${PROJECT_CREATE_BTN_XP} Click Button ${PROJECT_CREATE_BTN_XP} - Wait Until Page Contains Element ${RESOURCE_EDIT_BTN_XP} - Click Button ${RESOURCE_EDIT_BTN_XP} Wait Until Page Contains Element ${TITLE_INPUT_XP} Run Keyword And Warn On Failure Element Should Be Disabled ${GENERIC_CREATE_BTN_XP} Input Text ${TITLE_INPUT_XP} ${title} diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index b8d85d913..33f0786ab 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -21,7 +21,7 @@ ${PRJ_TITLE}= cross-auth-prj ${PRJ_DESCRIPTION}= project used for validating cross-auth CVE ${MODEL_CREATED}= ${FALSE} ${MODEL_NAME}= test-model -${SECOND_MODEL_NAME}= test-model-second +${SECOND_MODEL_NAME}= ${MODEL_NAME}-second ${RUNTIME_NAME}= Model Serving Test ${EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${SECOND_EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${SECOND_MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable @@ -63,7 +63,7 @@ Template with embedded arguments ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} - ... single_model=${single_model} model_name=${MODEL_NAME} project_name=${project_name} + ... single_model=${single_model} model_name=${SECOND_MODEL_NAME} project_name=${project_name} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} @@ -98,6 +98,7 @@ Cross Auth Model Deployment ... namespace=${project_name} ELSE Create Model Server token=${TRUE} server_name=${model_name} service_account_name=${service_account_name} + sleep 1m Deploy Model From Models Tab project_name=${project_name} model_name=${model_name} framework=onnx ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} ... model_path=openvino-example-model From aad7b586fbea25f92f660b6c8607e836857c63b2 Mon Sep 17 00:00:00 2001 From: rnetser Date: Sun, 29 Sep 2024 19:52:16 +0300 Subject: [PATCH 18/32] fix comments rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../1000__model_serving/1008_model_serving_cross_auth.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 33f0786ab..2b13211b0 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -101,7 +101,7 @@ Cross Auth Model Deployment sleep 1m Deploy Model From Models Tab project_name=${project_name} model_name=${model_name} framework=onnx ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} - ... model_path=openvino-example-model + ... model_path=mnist-8.onnx Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${model_name} ... project_name=${project_name} Wait Until Keyword Succeeds 5 min 10 sec Verify Serving Service project_name=${project_name} From a3b1727441c6511744d9c80266e528cfb10ef22a Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 08:06:54 +0300 Subject: [PATCH 19/32] fix assign rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 4512f4189..b4843467e 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -189,7 +189,7 @@ Enable Token Authentication [Arguments] ${service_account_name}=${NONE} SeleniumLibrary.Select Checkbox ${TOKEN_AUTH_CHECKBOX_XP} IF "${service_account_name}" == "${NONE}" - ${service_account_name}=default-name + ${service_account_name}= Set Variable default-name END Input Service Account Name ${service_account_name} From ad8950177262cbeb1c537fda7a9dc5095f8701bb Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 10:20:23 +0300 Subject: [PATCH 20/32] add runtime to token from ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ModelServer.resource | 15 +++++++++---- .../ODH/ODHDashboard/ODHModelServing.resource | 21 +++++++++++-------- .../1003__model_serving_customruntimes.robot | 1 + .../1008_model_serving_cross_auth.robot | 14 ++++++------- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index b4843467e..adaa4c84a 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -31,7 +31,7 @@ ${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]// *** Keywords *** Create Model Server [Documentation] Keyword to create a Model Server in a Data Science Project - [Arguments] ${no_replicas}=1 + [Arguments] ${no_replicas}=1 ... ${server_size}=Small ... ${ext_route}=${TRUE} ... ${token}=${TRUE} @@ -219,15 +219,22 @@ Input Service Account Name Get Model Serving Access Token via UI [Documentation] Returns the token used for authentication to the serving route ... TODO: There can be multiple tokens defined for each model server, handle this case as well - [Arguments] ${service_account_name}=default-name ${single_model}=${FALSE} ${model_name}=${NONE} + [Arguments] ${service_account_name}=default-name ${single_model}=${FALSE} + ... ${model_name}=${NONE} ${serving_runtime_name}=${NONE} IF ${single_model} # Expand the model SeleniumLibrary.Click Button xpath: //a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long ${token}= Get Single Model Token ${service_account_name} ELSE + IF ${serving_runtime_name} IS NOT NONE + ${model_server_name}= Replace String Using Regexp string=${serving_runtime_name} pattern=\\s replace_with=- + ${model_server_name}= Convert To Lower Case ${runtime_formattted_name} + ELSE + ${model_server_name}= Set Variable ${model_name} + END SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button - SeleniumLibrary.Click Button - ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button + SeleniumLibrary.Click Button + ... xpath://*[@id="expand-table-row-${model_server_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button ${token}= SeleniumLibrary.Get Element Attribute ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value END diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index f2e02b3b2..31df914b4 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -1,4 +1,4 @@ -# robocop: off=wrong-case-in-keyword-name,too-many-arguments,too-long-keyword +# robocop: off=wrong-case-in-keyword-name,too-many-arguments,too-long-keyword,line-too-long *** Settings *** Documentation Collection of keywords to interact with Model Serving Resource ../../../Page/Components/Components.resource @@ -229,9 +229,10 @@ Get Access Token Via UI [Documentation] Returns the access token for models deployed in a specific project ... by using the UI of DSP [Arguments] ${project_name} ${service_account_name}=default-name ${single_model}=${FALSE} - ... ${model_name}=${NONE} + ... ${model_name}=${NONE} ${serving_runtime_name}=${NONE} Open Data Science Project Details Page ${project_name} tab_id=model-server ${token}= Get Model Serving Access Token via UI ${service_account_name} ${single_model} ${model_name} + ... ${serving_runtime_name} RETURN ${token} Get Model Project @@ -250,6 +251,7 @@ Get Model Inference [Arguments] ${model_name} ${inference_input} ${token_auth}=${FALSE} ${project_title}=${NONE} ... ${kserve_mode}=Serverless ${deployment_mode}=UI ${service_port}=8888 ${end_point}=${NONE} ... ${service_account_name}=${NONE} ${token}=${NONE} + ... ${serving_runtime_name}=${NONE} ${curl_cmd}= Set Variable ${NONE} ${self_managed}= Is RHODS Self-Managed IF $deployment_mode == 'UI' @@ -263,7 +265,7 @@ Get Model Inference END IF $token == ${NONE} ${token}= Get Access Token via UI ${project_title} service_account_name=${service_account_name} - ... single_model=${kserve} model_name=${model_name} + ... single_model=${kserve} model_name=${model_name} serving_runtime_name=${serving_runtime_name} END ${curl_cmd}= Catenate ${curl_cmd} -H "Authorization: Bearer ${token}" END @@ -313,6 +315,7 @@ Verify Model Inference [Arguments] ${model_name} ${inference_input} ${expected_inference_output} ${token_auth}=${FALSE} ... ${project_title}=${NONE} ${deployment_mode}=UI ${kserve_mode}=Serverless ... ${service_port}=${NONE} ${end_point}=${NONE} ${token}=${NONE} + ... ${serving_runtime_name}=${NONE} IF $deployment_mode == 'UI' Open Model Serving Home Page Switch Model Serving Project ${project_title} @@ -320,7 +323,7 @@ Verify Model Inference ${inference_output}= Get Model Inference model_name=${model_name} inference_input=${inference_input} ... token_auth=${token_auth} kserve_mode=${kserve_mode} project_title=${project_title} ... deployment_mode=${deployment_mode} service_port=${service_port} end_point=${end_point} - ... token=${token} # robocop: disable + ... token=${token} serving_runtime_name=${serving_runtime_name} # robocop: disable Log ${inference_output} ${result} ${list}= Inference Comparison ${expected_inference_output} ${inference_output} Log ${result} @@ -623,7 +626,7 @@ Set Up Project Deploy Model From Models Tab [Documentation] Deploys a model from the project's Models tab - [Arguments] ${project_name} + [Arguments] ${project_name} ... ${data_connection_name} ... ${model_name} ... ${framework} @@ -632,7 +635,7 @@ Deploy Model From Models Tab ... ${model_server} ... ${serving_runtime}=${NONE} Open Data Science Project Details Page ${project_name} tab_id=model-server - SeleniumLibrary.Click Button + SeleniumLibrary.Click Button ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@class='pf-v5-c-table__td']//button SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] Fill Deploy Model Form model_name=${model_name} model_server=${model_server} @@ -640,9 +643,9 @@ Deploy Model From Models Tab ... data_connection_name=${data_connection_name} existing_data_connection=${existing_data_connection} ... model_path=${model_path} -Fill Deploy Model Form +Fill Deploy Model Form # robocop: off=too-many-calls-in-keyword [Documentation] Fills the deploy model form - [Arguments] ${model_name} + [Arguments] ${model_name} ... ${model_server} ... ${serving_runtime} ... ${framework} @@ -652,7 +655,7 @@ Fill Deploy Model Form Set Model Name ${model_name} Select Model Server ${model_server} IF "${serving_runtime}" != "${NONE}" Set Model Server Runtime ${serving_runtime} - SeleniumLibrary.Wait Until Page Contains Element + SeleniumLibrary.Wait Until Page Contains Element ... xpath://span[.="Model framework (name - version)"] Select Framework ${framework} IF ${existing_data_connection} diff --git a/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot b/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot index 52bae3d61..006731a06 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot @@ -86,6 +86,7 @@ Verify RHODS Users Can Deploy A Model Using A Custom Serving Runtime # roboco Verify Model Inference With Retries ${model_name} ${inference_input} ${exp_inference_output} ... token_auth=${TRUE} ... project_title=${PRJ_TITLE} + ... serving_runtime_name=${UPLOADED_OVMS_DISPLAYED_NAME} [Teardown] Run Keyword If Test Failed Get Events And Pod Logs namespace=${ns_name} ... label_selector=name=modelmesh-serving-${RUNTIME_POD_NAME} diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 2b13211b0..9e29c65f9 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -46,7 +46,7 @@ Test Cross Model Authentication On ModelMesh *** Keywords *** -Template with embedded arguments +Template with embedded arguments # robocop: off=too-many-calls-in-keyword [Documentation] Template for cross-auth test cases Cross Auth Test Setup ${single_model}= Set Variable If "${serving_mode}" == "kserve" ${True} ${False} @@ -56,14 +56,14 @@ Template with embedded arguments Cross Auth Model Deployment single_model=${single_model} ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} - ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} - ... single_model=${single_model} model_name=${MODEL_NAME} project_name=${project_name} + ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} + ... single_model=${single_model} serving_runtime_name=${MODEL_NAME} project_name=${project_name} Cross Auth Model Deployment single_model=${single_model} ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} - ... single_model=${single_model} model_name=${SECOND_MODEL_NAME} project_name=${project_name} + ... single_model=${single_model} serving_runtime_name=${SECOND_MODEL_NAME} project_name=${project_name} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} @@ -82,7 +82,7 @@ Template with embedded arguments ... token_auth=${TRUE} token=${first_token} Run Keyword And Warn On Failure Should Contain ${inf_out} Log in with OpenShift -Cross Auth Model Deployment +Cross Auth Model Deployment # robocop: off=too-many-calls-in-keyword [Documentation] Deploys a model with cross auth enabled [Arguments] ${single_model} ${model_name} ${service_account_name} ${dc_name}= Set Variable model-serving-connection-${serving_mode} @@ -105,7 +105,7 @@ Cross Auth Model Deployment Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${model_name} ... project_name=${project_name} Wait Until Keyword Succeeds 5 min 10 sec Verify Serving Service project_name=${project_name} - Verify Model Status ${MODEL_NAME} success + Verify Model Status model_name=${model_name} expected_status=success END Cross Auth Test Setup @@ -133,7 +133,7 @@ Cross Auth Test Teardown ELSE Log Model not deployed, skipping deletion step during teardown console=true END - ${projects}= Create List ${project_name} + VAR @{projects} ${project_name} Delete List Of Projects Via CLI ocp_projects=${projects} # Will only be present on SM cluster runs, but keyword passes # if file does not exist From d7bfab1945117d03f03ba3cfa5ed691b939e2412 Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 10:23:01 +0300 Subject: [PATCH 21/32] add runtime to token from ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../Page/ODH/ODHDashboard/ODHModelServing.resource | 6 ++---- .../1003__model_serving_customruntimes.robot | 1 - 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 31df914b4..de0525625 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -251,7 +251,6 @@ Get Model Inference [Arguments] ${model_name} ${inference_input} ${token_auth}=${FALSE} ${project_title}=${NONE} ... ${kserve_mode}=Serverless ${deployment_mode}=UI ${service_port}=8888 ${end_point}=${NONE} ... ${service_account_name}=${NONE} ${token}=${NONE} - ... ${serving_runtime_name}=${NONE} ${curl_cmd}= Set Variable ${NONE} ${self_managed}= Is RHODS Self-Managed IF $deployment_mode == 'UI' @@ -265,7 +264,7 @@ Get Model Inference END IF $token == ${NONE} ${token}= Get Access Token via UI ${project_title} service_account_name=${service_account_name} - ... single_model=${kserve} model_name=${model_name} serving_runtime_name=${serving_runtime_name} + ... single_model=${kserve} model_name=${model_name} END ${curl_cmd}= Catenate ${curl_cmd} -H "Authorization: Bearer ${token}" END @@ -315,7 +314,6 @@ Verify Model Inference [Arguments] ${model_name} ${inference_input} ${expected_inference_output} ${token_auth}=${FALSE} ... ${project_title}=${NONE} ${deployment_mode}=UI ${kserve_mode}=Serverless ... ${service_port}=${NONE} ${end_point}=${NONE} ${token}=${NONE} - ... ${serving_runtime_name}=${NONE} IF $deployment_mode == 'UI' Open Model Serving Home Page Switch Model Serving Project ${project_title} @@ -323,7 +321,7 @@ Verify Model Inference ${inference_output}= Get Model Inference model_name=${model_name} inference_input=${inference_input} ... token_auth=${token_auth} kserve_mode=${kserve_mode} project_title=${project_title} ... deployment_mode=${deployment_mode} service_port=${service_port} end_point=${end_point} - ... token=${token} serving_runtime_name=${serving_runtime_name} # robocop: disable + ... token=${token} # robocop: disable Log ${inference_output} ${result} ${list}= Inference Comparison ${expected_inference_output} ${inference_output} Log ${result} diff --git a/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot b/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot index 006731a06..52bae3d61 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1003__model_serving_customruntimes.robot @@ -86,7 +86,6 @@ Verify RHODS Users Can Deploy A Model Using A Custom Serving Runtime # roboco Verify Model Inference With Retries ${model_name} ${inference_input} ${exp_inference_output} ... token_auth=${TRUE} ... project_title=${PRJ_TITLE} - ... serving_runtime_name=${UPLOADED_OVMS_DISPLAYED_NAME} [Teardown] Run Keyword If Test Failed Get Events And Pod Logs namespace=${ns_name} ... label_selector=name=modelmesh-serving-${RUNTIME_POD_NAME} From d4a4a38c03c92a66aad1777f8a69261c2f82c231 Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 10:33:49 +0300 Subject: [PATCH 22/32] add runtime to token from ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDataScienceProject/ModelServer.resource | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index adaa4c84a..f2935d11c 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -226,15 +226,19 @@ Get Model Serving Access Token via UI SeleniumLibrary.Click Button xpath: //a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long ${token}= Get Single Model Token ${service_account_name} ELSE - IF ${serving_runtime_name} IS NOT NONE + SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button + # TODO: ModelMesh requires passing model server name and not model name; in particular when there are multiple model servers + # in the same project. + # This is a temporary workaround until the issue is fixed. With one model server, the model name is not required. + IF ${serving_runtime_name} != "${NONE}" ${model_server_name}= Replace String Using Regexp string=${serving_runtime_name} pattern=\\s replace_with=- ${model_server_name}= Convert To Lower Case ${runtime_formattted_name} + SeleniumLibrary.Click Button + ... xpath://*[@id="expand-table-row-${model_server_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button + ELSE - ${model_server_name}= Set Variable ${model_name} + SeleniumLibrary.Click Element xpath://td[@data-label="Tokens"]/button END - SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button - SeleniumLibrary.Click Button - ... xpath://*[@id="expand-table-row-${model_server_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button ${token}= SeleniumLibrary.Get Element Attribute ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value END From 08f2918bf6982c294ef4a8922f51d8d45bff3726 Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 11:01:48 +0300 Subject: [PATCH 23/32] add runtime to token from ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDataScienceProject/ModelServer.resource | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index f2935d11c..779c432eb 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -230,15 +230,19 @@ Get Model Serving Access Token via UI # TODO: ModelMesh requires passing model server name and not model name; in particular when there are multiple model servers # in the same project. # This is a temporary workaround until the issue is fixed. With one model server, the model name is not required. - IF ${serving_runtime_name} != "${NONE}" + IF "${serving_runtime_name}" != "${NONE}" ${model_server_name}= Replace String Using Regexp string=${serving_runtime_name} pattern=\\s replace_with=- ${model_server_name}= Convert To Lower Case ${runtime_formattted_name} SeleniumLibrary.Click Button ... xpath://*[@id="expand-table-row-${model_server_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button - ELSE SeleniumLibrary.Click Element xpath://td[@data-label="Tokens"]/button END + # TODO: service account name is required; remove from all places that pass `None` + IF "${service_account_name}" == "${NONE}" + ${service_account_name}= Set Variable default-name + END + ${token}= SeleniumLibrary.Get Element Attribute ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value END From 7891842377f88fb5c87436e744288ba5e697d79c Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 11:13:16 +0300 Subject: [PATCH 24/32] add runtime to token from ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../1008_model_serving_cross_auth.robot | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 9e29c65f9..7145aa9c5 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -23,6 +23,7 @@ ${MODEL_CREATED}= ${FALSE} ${MODEL_NAME}= test-model ${SECOND_MODEL_NAME}= ${MODEL_NAME}-second ${RUNTIME_NAME}= Model Serving Test +${SECOND_RUNTIME_NAME}= Second Model Serving Test ${EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${SECOND_EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${SECOND_MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${FIRST_SERVICE_ACCOUNT}= first_account @@ -54,16 +55,18 @@ Template with embedded arguments # robocop: off=too-many-calls-in-keyword Create Data Science Project title=${project_name} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} Cross Auth Model Deployment single_model=${single_model} - ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} + ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} + ... runtime=${RUNTIME_NAME} ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} - ... single_model=${single_model} serving_runtime_name=${MODEL_NAME} project_name=${project_name} + ... single_model=${single_model} serving_runtime_name=${RUNTIME_NAME} project_name=${project_name} Cross Auth Model Deployment single_model=${single_model} - ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} + ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} + ... runtime=${SECOND_RUNTIME_NAME} ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} - ... single_model=${single_model} serving_runtime_name=${SECOND_MODEL_NAME} project_name=${project_name} + ... single_model=${single_model} serving_runtime_name=${SECOND_RUNTIME_NAME} project_name=${project_name} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} @@ -84,7 +87,7 @@ Template with embedded arguments # robocop: off=too-many-calls-in-keyword Cross Auth Model Deployment # robocop: off=too-many-calls-in-keyword [Documentation] Deploys a model with cross auth enabled - [Arguments] ${single_model} ${model_name} ${service_account_name} + [Arguments] ${single_model} ${model_name} ${service_account_name} ${runtime} ${dc_name}= Set Variable model-serving-connection-${serving_mode} Recreate S3 Data Connection project_title=${project_name} dc_name=${dc_name} ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} @@ -97,12 +100,12 @@ Cross Auth Model Deployment # robocop: off=too-many-calls-in-keyword Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} ... namespace=${project_name} ELSE - Create Model Server token=${TRUE} server_name=${model_name} service_account_name=${service_account_name} + Create Model Server token=${TRUE} server_name=${runtime} service_account_name=${service_account_name} sleep 1m Deploy Model From Models Tab project_name=${project_name} model_name=${model_name} framework=onnx - ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} + ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${runtime} ... model_path=mnist-8.onnx - Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${model_name} + Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${runtime} ... project_name=${project_name} Wait Until Keyword Succeeds 5 min 10 sec Verify Serving Service project_name=${project_name} Verify Model Status model_name=${model_name} expected_status=success @@ -133,7 +136,7 @@ Cross Auth Test Teardown ELSE Log Model not deployed, skipping deletion step during teardown console=true END - VAR @{projects} ${project_name} + ${projects}= Create List ${project_name} Delete List Of Projects Via CLI ocp_projects=${projects} # Will only be present on SM cluster runs, but keyword passes # if file does not exist From 4f57e4661974d20cd8eafb3280e7378c4ed83d14 Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 12:41:27 +0300 Subject: [PATCH 25/32] add runtime to token from ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ModelServer.resource | 10 ++++---- .../ODH/ODHDashboard/ODHModelServing.resource | 4 ++-- .../1008_model_serving_cross_auth.robot | 24 +++++++++---------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 779c432eb..f99be61cc 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -220,7 +220,7 @@ Get Model Serving Access Token via UI [Documentation] Returns the token used for authentication to the serving route ... TODO: There can be multiple tokens defined for each model server, handle this case as well [Arguments] ${service_account_name}=default-name ${single_model}=${FALSE} - ... ${model_name}=${NONE} ${serving_runtime_name}=${NONE} + ... ${model_name}=${NONE} ${multi_model_servers}=${FALSE} IF ${single_model} # Expand the model SeleniumLibrary.Click Button xpath: //a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long @@ -230,11 +230,9 @@ Get Model Serving Access Token via UI # TODO: ModelMesh requires passing model server name and not model name; in particular when there are multiple model servers # in the same project. # This is a temporary workaround until the issue is fixed. With one model server, the model name is not required. - IF "${serving_runtime_name}" != "${NONE}" - ${model_server_name}= Replace String Using Regexp string=${serving_runtime_name} pattern=\\s replace_with=- - ${model_server_name}= Convert To Lower Case ${runtime_formattted_name} + IF ${multi_model_servers} SeleniumLibrary.Click Button - ... xpath://*[@id="expand-table-row-${model_server_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button + ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button ELSE SeleniumLibrary.Click Element xpath://td[@data-label="Tokens"]/button END @@ -242,7 +240,7 @@ Get Model Serving Access Token via UI IF "${service_account_name}" == "${NONE}" ${service_account_name}= Set Variable default-name END - + ${token}= SeleniumLibrary.Get Element Attribute ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value END diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index de0525625..f2a2bdf04 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -229,10 +229,10 @@ Get Access Token Via UI [Documentation] Returns the access token for models deployed in a specific project ... by using the UI of DSP [Arguments] ${project_name} ${service_account_name}=default-name ${single_model}=${FALSE} - ... ${model_name}=${NONE} ${serving_runtime_name}=${NONE} + ... ${model_name}=${NONE} ${multi_model_servers}=${FALSE} Open Data Science Project Details Page ${project_name} tab_id=model-server ${token}= Get Model Serving Access Token via UI ${service_account_name} ${single_model} ${model_name} - ... ${serving_runtime_name} + ... ${multi_model_servers} RETURN ${token} Get Model Project diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 7145aa9c5..9e8af561a 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -22,8 +22,6 @@ ${PRJ_DESCRIPTION}= project used for validating cross-auth CVE ${MODEL_CREATED}= ${FALSE} ${MODEL_NAME}= test-model ${SECOND_MODEL_NAME}= ${MODEL_NAME}-second -${RUNTIME_NAME}= Model Serving Test -${SECOND_RUNTIME_NAME}= Second Model Serving Test ${EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${SECOND_EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${SECOND_MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${FIRST_SERVICE_ACCOUNT}= first_account @@ -55,18 +53,18 @@ Template with embedded arguments # robocop: off=too-many-calls-in-keyword Create Data Science Project title=${project_name} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} Cross Auth Model Deployment single_model=${single_model} - ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} - ... runtime=${RUNTIME_NAME} + ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} - ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} - ... single_model=${single_model} serving_runtime_name=${RUNTIME_NAME} project_name=${project_name} + ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} + ... single_model=${single_model} model_name=${MODEL_NAME} project_name=${project_name} + ... multi_model_servers=not ${single_model} Cross Auth Model Deployment single_model=${single_model} - ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} - ... runtime=${SECOND_RUNTIME_NAME} + ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} - ... single_model=${single_model} serving_runtime_name=${SECOND_RUNTIME_NAME} project_name=${project_name} + ... single_model=${single_model} model_name=${SECOND_MODEL_NAME} project_name=${project_name} + ... multi_model_servers=not ${single_model} Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} @@ -87,7 +85,7 @@ Template with embedded arguments # robocop: off=too-many-calls-in-keyword Cross Auth Model Deployment # robocop: off=too-many-calls-in-keyword [Documentation] Deploys a model with cross auth enabled - [Arguments] ${single_model} ${model_name} ${service_account_name} ${runtime} + [Arguments] ${single_model} ${model_name} ${service_account_name} ${dc_name}= Set Variable model-serving-connection-${serving_mode} Recreate S3 Data Connection project_title=${project_name} dc_name=${dc_name} ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} @@ -100,12 +98,12 @@ Cross Auth Model Deployment # robocop: off=too-many-calls-in-keyword Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} ... namespace=${project_name} ELSE - Create Model Server token=${TRUE} server_name=${runtime} service_account_name=${service_account_name} + Create Model Server token=${TRUE} server_name=${model_name} service_account_name=${service_account_name} sleep 1m Deploy Model From Models Tab project_name=${project_name} model_name=${model_name} framework=onnx - ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${runtime} + ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} ... model_path=mnist-8.onnx - Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${runtime} + Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${model_name} ... project_name=${project_name} Wait Until Keyword Succeeds 5 min 10 sec Verify Serving Service project_name=${project_name} Verify Model Status model_name=${model_name} expected_status=success From c3a830df8444b4cf8f4934c489ecf6431944de97 Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 18:32:35 +0300 Subject: [PATCH 26/32] update paths in project rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODH/ODHDashboard/ODHDataScienceProject/Projects.resource | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource index a1b18a625..d8085440c 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource @@ -8,7 +8,8 @@ Resource ./Workbenches.resource ${DS_PROJECT_TITLE}= Data Science Projects ${TITLE_INPUT_XP}= xpath=//input[@id="manage-project-modal-name"] ${DESCR_INPUT_XP}= xpath=//textarea[@id="manage-project-modal-description"] -${RESOURCE_INPUT_XP}= css:[data-testid="resource-manage-project-modal-name"] +${RESOURCE_EDIT_BTN_XP}= xpath=//button[.="Edit resource name"] +${RESOURCE_INPUT_XP}= css:[data-testid="manage-project-modal-resourceName"] ${GENERIC_CREATE_BTN_XP}= xpath=//button[text()="Create"] ${GENERIC_CANCEL_BTN_XP}= xpath=//button[text()="Cancel"] # TODO: Update to latter option once the change is pulled from ODH into downstream! @@ -136,6 +137,8 @@ Create Data Science Project ELSE Wait Until Page Contains Element ${PROJECT_CREATE_BTN_XP} Click Button ${PROJECT_CREATE_BTN_XP} + Wait Until Page Contains Element ${RESOURCE_EDIT_BTN_XP} + Click Button ${RESOURCE_EDIT_BTN_XP} Wait Until Page Contains Element ${TITLE_INPUT_XP} Run Keyword And Warn On Failure Element Should Be Disabled ${GENERIC_CREATE_BTN_XP} Input Text ${TITLE_INPUT_XP} ${title} From 2db9d6252a18249a30ab861dce2def82ec2e65ce Mon Sep 17 00:00:00 2001 From: rnetser Date: Mon, 30 Sep 2024 18:46:42 +0300 Subject: [PATCH 27/32] address robocop comments rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDashboard/ODHDataScienceProject/ModelServer.resource | 6 ++++-- .../Page/ODH/ODHDashboard/ODHModelServing.resource | 5 +++-- .../1000__model_serving/1008_model_serving_cross_auth.robot | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index f99be61cc..6c476da98 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -227,9 +227,11 @@ Get Model Serving Access Token via UI ${token}= Get Single Model Token ${service_account_name} ELSE SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button - # TODO: ModelMesh requires passing model server name and not model name; in particular when there are multiple model servers + # TODO: ModelMesh requires passing model server name and not model name; + # in particular when there are multiple model servers # in the same project. - # This is a temporary workaround until the issue is fixed. With one model server, the model name is not required. + # This is a temporary workaround until the issue is fixed. + # With one model server, the model name is not required. IF ${multi_model_servers} SeleniumLibrary.Click Button ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index f2a2bdf04..7d767972b 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -228,8 +228,9 @@ Open ${section} Options Menu Get Access Token Via UI [Documentation] Returns the access token for models deployed in a specific project ... by using the UI of DSP - [Arguments] ${project_name} ${service_account_name}=default-name ${single_model}=${FALSE} - ... ${model_name}=${NONE} ${multi_model_servers}=${FALSE} + [Arguments] ${project_name} ${service_account_name}=default-name + ... ${single_model}=${FALSE} ${model_name}=${NONE} + ... ${multi_model_servers}=${FALSE} Open Data Science Project Details Page ${project_name} tab_id=model-server ${token}= Get Model Serving Access Token via UI ${service_account_name} ${single_model} ${model_name} ... ${multi_model_servers} diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 9e8af561a..91eda1500 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -55,7 +55,7 @@ Template with embedded arguments # robocop: off=too-many-calls-in-keyword Cross Auth Model Deployment single_model=${single_model} ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} - ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} + ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} ... single_model=${single_model} model_name=${MODEL_NAME} project_name=${project_name} ... multi_model_servers=not ${single_model} From 5827368203d7e7a19eec1b59db91108be1c3cfe5 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 1 Oct 2024 08:23:14 +0300 Subject: [PATCH 28/32] fix ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDataScienceProject/ModelServer.resource | 5 +++-- .../ODH/ODHDashboard/ODHModelServing.resource | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 6c476da98..780404b0a 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -80,7 +80,8 @@ Create Model Server Set Replicas Number With Buttons [Documentation] Sets the number of replicas for the model serving pod [Arguments] ${number} - ${current}= Get Element Attribute ${REPLICAS_COUNT_XP} value + ${RESOURCE_INPUT_XP}= css:[id="model-server-replicas"] + ${current}= Get Element Attribute css:[id="model-server-replicas"] value ${difference}= Evaluate int(${number})-int(${current}) ${op}= Set Variable plus IF ${difference}<${0} @@ -94,7 +95,7 @@ Set Replicas Number With Buttons Click Minus Button END END - ${current}= Get Element Attribute ${REPLICAS_COUNT_XP} value + ${current}= Get Element Attribute css:[id="model-server-replicas"] value Should Be Equal As Integers ${current} ${number} Set Server Size diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 7d767972b..71369e98f 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -373,11 +373,18 @@ Clean Up Model Serving Page [Documentation] Deletes all currently deployed models, if any are present. # Returns an empty list if no matching elements found Switch Model Serving Project project_name=All projects - ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a - FOR ${model} IN @{models} - ${model}= Get Text ${model} - Delete Model Via UI ${model} - Sleep 1s + ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] + FOR ${project} IN @{projects} + ${project}= Get Text ${project} + @{project description}= Split String ${project} + Switch Model Serving Project ${project description}[0] + ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a + FOR ${model} IN @{models} + ${model}= Get Text ${model} + Delete Model Via UI ${model} + Sleep 1s + END + Switch Model Serving Project project_name=All projects END Add Namespace To ServiceMeshMemberRoll From 846f459fe7db8ef0d572d082b8ad777a0f5f417e Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 1 Oct 2024 12:43:07 +0300 Subject: [PATCH 29/32] fix ui rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- .../ODHDataScienceProject/ModelServer.resource | 4 ++-- .../Page/ODH/ODHDashboard/ODHModelServing.resource | 4 +++- .../1008_model_serving_cross_auth.robot | 12 ++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 780404b0a..e17da7866 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -80,7 +80,6 @@ Create Model Server Set Replicas Number With Buttons [Documentation] Sets the number of replicas for the model serving pod [Arguments] ${number} - ${RESOURCE_INPUT_XP}= css:[id="model-server-replicas"] ${current}= Get Element Attribute css:[id="model-server-replicas"] value ${difference}= Evaluate int(${number})-int(${current}) ${op}= Set Variable plus @@ -224,7 +223,8 @@ Get Model Serving Access Token via UI ... ${model_name}=${NONE} ${multi_model_servers}=${FALSE} IF ${single_model} # Expand the model - SeleniumLibrary.Click Button xpath: //a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long + #SeleniumLibrary.Click Button xpath://a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long + SeleniumLibrary.Click Element xpath://a[text()='${model_name}']/../../../../td/button[@class="pf-v5-c-button pf-m-plain pf-m-expanded"] ${token}= Get Single Model Token ${service_account_name} ELSE SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 71369e98f..ddd215823 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -473,7 +473,9 @@ Set Model Server Runtime [Arguments] ${runtime}=Caikit TGIS Page Should Contain Element ${KSERVE_RUNTIME_DROPDOWN} Click Element ${KSERVE_RUNTIME_DROPDOWN} - Click Element ${KSERVE_RUNTIME_DROPDOWN}/..//span[contains(text(),"${runtime}")] + ${runtime_element} Set Variable ${KSERVE_RUNTIME_DROPDOWN}/..//span[contains(text(),"${runtime}")] + Page Should Contain Element ${runtime_element} + Click Element ${runtime_element} Get Kserve Inference Host Via UI [Documentation] Fetches the host of the model's URL from the Data Science Project UI diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 91eda1500..01f4eb1c0 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -119,7 +119,7 @@ Cross Auth Test Setup Launch Dashboard ${TEST_USER.USERNAME} ${TEST_USER.PASSWORD} ${TEST_USER.AUTH_TYPE} ... ${ODH_DASHBOARD_URL} ${BROWSER.NAME} ${BROWSER.OPTIONS} Fetch Knative CA Certificate filename=openshift_ca_istio_knative.crt - Clean All Models Of Current User + # Clean All Models Of Current User Cross Auth Test Teardown [Documentation] Test teardown steps after testing DSG. It Deletes @@ -129,11 +129,11 @@ Cross Auth Test Teardown # Even if kw fails, deleting the whole project will also delete the model # Failure will be shown in the logs of the run nonetheless - IF ${MODEL_CREATED} - Clean All Models Of Current User - ELSE - Log Model not deployed, skipping deletion step during teardown console=true - END + # IF ${MODEL_CREATED} + # Clean All Models Of Current User + # ELSE + # Log Model not deployed, skipping deletion step during teardown console=true + # END ${projects}= Create List ${project_name} Delete List Of Projects Via CLI ocp_projects=${projects} # Will only be present on SM cluster runs, but keyword passes From 78fe13fd61e96993194a71ca5bab6bf834fb4651 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 1 Oct 2024 12:46:06 +0300 Subject: [PATCH 30/32] update 2.13 rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- ods_ci/tests/Resources/Common.robot | 9 ++++---- ods_ci/tests/Resources/OCP.resource | 10 --------- .../ModelServer.resource | 11 +++++----- .../ODH/ODHDashboard/ODHModelServing.resource | 21 ++++++------------- .../1008_model_serving_cross_auth.robot | 12 +++++------ 5 files changed, 21 insertions(+), 42 deletions(-) diff --git a/ods_ci/tests/Resources/Common.robot b/ods_ci/tests/Resources/Common.robot index 6aeccc8db..cba7f048f 100644 --- a/ods_ci/tests/Resources/Common.robot +++ b/ods_ci/tests/Resources/Common.robot @@ -437,12 +437,11 @@ Extract URLs From Text RETURN ${urls} Run And Verify Command - [Documentation] Run and verify shell command - [Arguments] ${command} ${print_to_log}=${TRUE} - ${result}= Run Process ${command} shell=yes stderr=STDOUT - IF ${print_to_log} Log ${result.stdout} console=True + [Documentation] Run an oc delete command and log the output and errors + [Arguments] ${command} + ${result}= Run Process ${command} shell=yes + Log ${result.stdout}\n${result.stderr} console=True Should Be True ${result.rc} == 0 - RETURN ${result.stdout} Run And Watch Command [Documentation] Run any shell command (including args) with optional: diff --git a/ods_ci/tests/Resources/OCP.resource b/ods_ci/tests/Resources/OCP.resource index 87aa8004e..464635b19 100644 --- a/ods_ci/tests/Resources/OCP.resource +++ b/ods_ci/tests/Resources/OCP.resource @@ -124,16 +124,6 @@ Wait For Pods To Be Ready Length Should Be ${replicas} ${exp_replicas} END -Wait For Deployment Replica To Be Ready - [Documentation] Wait for Deployment of ${label_selector} in ${namespace} to have the Replica-Set Ready - [Arguments] ${namespace} ${label_selector} ${timeout}=600s - Log To Console Waiting for Deployment with label "${label_selector}" in Namespace "${namespace}", to have desired Replica-Set - ${output} = Wait Until Keyword Succeeds ${timeout} 3s Run And Verify Command - ... oc wait -n ${namespace} --for condition\=ready pod -l "${label_selector}" --timeout\=${timeout} - ... print_to_log=${FALSE} - Run And Verify Command - ... oc get deployment -l ${label_selector} -n ${namespace} -o json | jq -e '.status | .replicas \=\= .readyReplicas' - Wait For Pods To Succeed [Arguments] ${label_selector} ${namespace} ${timeout}=300s ${exp_replicas}=${NONE} Wait Until Keyword Succeeds ${timeout} 3s diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index e17da7866..c4a50976f 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -80,7 +80,7 @@ Create Model Server Set Replicas Number With Buttons [Documentation] Sets the number of replicas for the model serving pod [Arguments] ${number} - ${current}= Get Element Attribute css:[id="model-server-replicas"] value + ${current}= Get Element Attribute ${REPLICAS_COUNT_XP} value ${difference}= Evaluate int(${number})-int(${current}) ${op}= Set Variable plus IF ${difference}<${0} @@ -94,7 +94,7 @@ Set Replicas Number With Buttons Click Minus Button END END - ${current}= Get Element Attribute css:[id="model-server-replicas"] value + ${current}= Get Element Attribute ${REPLICAS_COUNT_XP} value Should Be Equal As Integers ${current} ${number} Set Server Size @@ -223,15 +223,14 @@ Get Model Serving Access Token via UI ... ${model_name}=${NONE} ${multi_model_servers}=${FALSE} IF ${single_model} # Expand the model - #SeleniumLibrary.Click Button xpath://a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long - SeleniumLibrary.Click Element xpath://a[text()='${model_name}']/../../../../td/button[@class="pf-v5-c-button pf-m-plain pf-m-expanded"] + SeleniumLibrary.Click Button xpath: //a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long ${token}= Get Single Model Token ${service_account_name} ELSE SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button - # TODO: ModelMesh requires passing model server name and not model name; + # TODO: ModelMesh requires passing model server name and not model name; # in particular when there are multiple model servers # in the same project. - # This is a temporary workaround until the issue is fixed. + # This is a temporary workaround until the issue is fixed. # With one model server, the model name is not required. IF ${multi_model_servers} SeleniumLibrary.Click Button diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index ddd215823..7d767972b 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -373,18 +373,11 @@ Clean Up Model Serving Page [Documentation] Deletes all currently deployed models, if any are present. # Returns an empty list if no matching elements found Switch Model Serving Project project_name=All projects - ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] - FOR ${project} IN @{projects} - ${project}= Get Text ${project} - @{project description}= Split String ${project} - Switch Model Serving Project ${project description}[0] - ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a - FOR ${model} IN @{models} - ${model}= Get Text ${model} - Delete Model Via UI ${model} - Sleep 1s - END - Switch Model Serving Project project_name=All projects + ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a + FOR ${model} IN @{models} + ${model}= Get Text ${model} + Delete Model Via UI ${model} + Sleep 1s END Add Namespace To ServiceMeshMemberRoll @@ -473,9 +466,7 @@ Set Model Server Runtime [Arguments] ${runtime}=Caikit TGIS Page Should Contain Element ${KSERVE_RUNTIME_DROPDOWN} Click Element ${KSERVE_RUNTIME_DROPDOWN} - ${runtime_element} Set Variable ${KSERVE_RUNTIME_DROPDOWN}/..//span[contains(text(),"${runtime}")] - Page Should Contain Element ${runtime_element} - Click Element ${runtime_element} + Click Element ${KSERVE_RUNTIME_DROPDOWN}/..//span[contains(text(),"${runtime}")] Get Kserve Inference Host Via UI [Documentation] Fetches the host of the model's URL from the Data Science Project UI diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 01f4eb1c0..91eda1500 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -119,7 +119,7 @@ Cross Auth Test Setup Launch Dashboard ${TEST_USER.USERNAME} ${TEST_USER.PASSWORD} ${TEST_USER.AUTH_TYPE} ... ${ODH_DASHBOARD_URL} ${BROWSER.NAME} ${BROWSER.OPTIONS} Fetch Knative CA Certificate filename=openshift_ca_istio_knative.crt - # Clean All Models Of Current User + Clean All Models Of Current User Cross Auth Test Teardown [Documentation] Test teardown steps after testing DSG. It Deletes @@ -129,11 +129,11 @@ Cross Auth Test Teardown # Even if kw fails, deleting the whole project will also delete the model # Failure will be shown in the logs of the run nonetheless - # IF ${MODEL_CREATED} - # Clean All Models Of Current User - # ELSE - # Log Model not deployed, skipping deletion step during teardown console=true - # END + IF ${MODEL_CREATED} + Clean All Models Of Current User + ELSE + Log Model not deployed, skipping deletion step during teardown console=true + END ${projects}= Create List ${project_name} Delete List Of Projects Via CLI ocp_projects=${projects} # Will only be present on SM cluster runs, but keyword passes From 2cfb81d376c33c741a00b32486ff8f0344c55fb9 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 1 Oct 2024 12:47:24 +0300 Subject: [PATCH 31/32] update 2.13 rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- ods_ci/tests/Resources/Common.robot | 9 +++++---- ods_ci/tests/Resources/OCP.resource | 10 ++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ods_ci/tests/Resources/Common.robot b/ods_ci/tests/Resources/Common.robot index cba7f048f..4e67e9eca 100644 --- a/ods_ci/tests/Resources/Common.robot +++ b/ods_ci/tests/Resources/Common.robot @@ -437,11 +437,12 @@ Extract URLs From Text RETURN ${urls} Run And Verify Command - [Documentation] Run an oc delete command and log the output and errors - [Arguments] ${command} - ${result}= Run Process ${command} shell=yes - Log ${result.stdout}\n${result.stderr} console=True + [Documentation] Run and verify shell command + [Arguments] ${command} ${print_to_log}=${TRUE} + ${result}= Run Process ${command} shell=yes stderr=STDOUT + IF ${print_to_log} Log ${result.stdout} console=True Should Be True ${result.rc} == 0 + RETURN ${result.stdout} Run And Watch Command [Documentation] Run any shell command (including args) with optional: diff --git a/ods_ci/tests/Resources/OCP.resource b/ods_ci/tests/Resources/OCP.resource index 464635b19..87aa8004e 100644 --- a/ods_ci/tests/Resources/OCP.resource +++ b/ods_ci/tests/Resources/OCP.resource @@ -124,6 +124,16 @@ Wait For Pods To Be Ready Length Should Be ${replicas} ${exp_replicas} END +Wait For Deployment Replica To Be Ready + [Documentation] Wait for Deployment of ${label_selector} in ${namespace} to have the Replica-Set Ready + [Arguments] ${namespace} ${label_selector} ${timeout}=600s + Log To Console Waiting for Deployment with label "${label_selector}" in Namespace "${namespace}", to have desired Replica-Set + ${output} = Wait Until Keyword Succeeds ${timeout} 3s Run And Verify Command + ... oc wait -n ${namespace} --for condition\=ready pod -l "${label_selector}" --timeout\=${timeout} + ... print_to_log=${FALSE} + Run And Verify Command + ... oc get deployment -l ${label_selector} -n ${namespace} -o json | jq -e '.status | .replicas \=\= .readyReplicas' + Wait For Pods To Succeed [Arguments] ${label_selector} ${namespace} ${timeout}=300s ${exp_replicas}=${NONE} Wait Until Keyword Succeeds ${timeout} 3s From 6d29aefb4307d4a40661d7b1ddeab7d9a8009484 Mon Sep 17 00:00:00 2001 From: rnetser Date: Tue, 1 Oct 2024 12:54:08 +0300 Subject: [PATCH 32/32] update 2.13 rh-pre-commit.version: 2.3.1 rh-pre-commit.check-secrets: ENABLED --- ods_ci/tests/Resources/Common.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Common.robot b/ods_ci/tests/Resources/Common.robot index 4e67e9eca..6aeccc8db 100644 --- a/ods_ci/tests/Resources/Common.robot +++ b/ods_ci/tests/Resources/Common.robot @@ -437,7 +437,7 @@ Extract URLs From Text RETURN ${urls} Run And Verify Command - [Documentation] Run and verify shell command + [Documentation] Run and verify shell command [Arguments] ${command} ${print_to_log}=${TRUE} ${result}= Run Process ${command} shell=yes stderr=STDOUT IF ${print_to_log} Log ${result.stdout} console=True