From 81c2549b1d009e48a37816d12bf472dd91666115 Mon Sep 17 00:00:00 2001 From: rpancham <rpancham@redhat.com> Date: Fri, 15 Nov 2024 20:18:18 +0530 Subject: [PATCH 1/4] Added FIL model REST protocol test on triton for Modelmesh --- .../triton/modelmesh-triton-fil-input.json | 1 + .../triton/modelmesh-triton-fil-output.json | 1 + ...0__model_serving_triton_on_modelmesh.robot | 39 ++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-input.json create mode 100644 ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-output.json diff --git a/ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-input.json b/ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-input.json new file mode 100644 index 000000000..d9fc08f2d --- /dev/null +++ b/ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-input.json @@ -0,0 +1 @@ +{"inputs":[{"name":"input__0","shape":[1,393],"datatype":"FP32","data":[0.3284832835197449,0.8268185257911682,0.78487628698349,0.9519302845001221,0.12701496481895447,0.5136651992797852,0.6298874020576477,0.45734643936157227,0.9838705658912659,0.20956359803676605,0.8379985690116882,0.5718668103218079,0.019412444904446602,0.38271811604499817,0.6325977444648743,0.03927919268608093,0.48838457465171814,0.4702889323234558,0.4560658633708954,0.6781136989593506,0.5536667704582214,0.4988061487674713,0.12478730827569962,0.1059209406375885,0.20966315269470215,0.4273585081100464,0.653721034526825,0.45579633116722107,0.14691787958145142,0.33020544052124023,0.05173210799694061,0.11969181895256042,0.16853050887584686,0.4049186110496521,0.755652666091919,0.7312496900558472,0.10803086310625076,0.457566499710083,0.3736112415790558,0.1936161369085312,0.8652274012565613,0.4038611948490143,0.5566637516021729,0.06435327231884003,0.566157877445221,0.6998145580291748,0.15337392687797546,0.6230511665344238,0.7362670302391052,0.5723153352737427,0.6682037115097046,0.3011724650859833,0.042016711086034775,0.6602086424827576,0.5835669040679932,0.0284038707613945,0.1020161584019661,0.1951567679643631,0.4169561266899109,0.6155021786689758,0.2703559100627899,0.07744283229112625,0.051553454250097275,0.46595120429992676,0.38126346468925476,0.8140078186988831,0.6293876767158508,0.9310184717178345,0.7771000266075134,0.2593490481376648,0.2669752538204193,0.5493547916412354,0.017035549506545067,0.6309475898742676,0.7316464185714722,0.6175021529197693,0.7201271057128906,0.08383309096097946,0.9262048006057739,0.43735557794570923,0.3239584267139435,0.047336988151073456,0.36462053656578064,0.7607018947601318,0.8412635326385498,0.9601210951805115,0.6327416896820068,0.8613588213920593,0.24567070603370667,0.5343116521835327,0.828964114189148,0.8069473505020142,0.40954720973968506,0.4407307207584381,0.8217458128929138,0.8649827241897583,0.555292010307312,0.8169907927513123,0.9104942083358765,0.3510514795780182,0.313764363527298,0.706859827041626,0.2965836524963379,0.40725183486938477,0.5099840760231018,0.132075235247612,0.43063321709632874,0.3924606144428253,0.28096944093704224,0.32640188932418823,0.7225360870361328,0.5635764598846436,0.5840679407119751,0.7885806560516357,0.21381767094135284,0.8454336524009705,0.17979982495307922,0.5565366148948669,0.23402656614780426,0.7756942510604858,0.03492642194032669,0.5121867060661316,0.557293713092804,0.6212375164031982,0.6052588224411011,0.9269223809242249,0.305753618478775,0.45190030336380005,0.6436784863471985,0.7278525233268738,0.5478901863098145,0.9916322231292725,0.9576631188392639,0.07307152450084686,0.19526472687721252,0.949763834476471,0.6788548827171326,0.33683764934539795,0.6835283041000366,0.6119109988212585,0.25176212191581726,0.220492884516716,0.7287231087684631,0.9773213267326355,0.6861571669578552,0.7782228589057922,0.06823815405368805,0.9881873726844788,0.6924068331718445,0.1236552745103836,0.40915706753730774,0.6072324514389038,0.1805514097213745,0.4088282883167267,0.7606160044670105,0.27846649289131165,0.4318022131919861,0.10575707256793976,0.4982239305973053,0.14627672731876373,0.14424724876880646,0.7654518485069275,0.4624658226966858,0.027059316635131836,0.48425596952438354,0.5849552750587463,0.531979501247406,0.33964803814888,0.6425493359565735,0.10979773104190826,0.5976448655128479,0.9113814234733582,0.6277787089347839,0.43358638882637024,0.023739399388432503,0.6573179960250854,0.8587101101875305,0.2997402548789978,0.6855675578117371,0.2859164774417877,0.38119083642959595,0.8460462689399719,0.7710570096969604,0.3703351318836212,0.5643764138221741,0.5831498503684998,0.8303821086883545,0.8836601376533508,0.8476974964141846,0.9472933411598206,0.25329846143722534,0.11601176112890244,0.9747433066368103,0.34176814556121826,0.8251526355743408,0.17527931928634644,0.5483436584472656,0.8419793248176575,0.9838578701019287,0.07968159765005112,0.09145355969667435,0.826462984085083,0.23544777929782867,0.3038272559642792,0.8828607201576233,0.344532310962677,0.2766166627407074,0.057731930166482925,0.7391823530197144,0.47562375664711,0.7556608319282532,0.2841605246067047,0.6486907601356506,0.2636808753013611,0.023107757791876793,0.6709638833999634,0.6070297956466675,0.2201913446187973,0.8700276613235474,0.12464558333158493,0.8486520648002625,0.235423743724823,0.9058780670166016,0.03418419882655144,0.7535243630409241,0.5563464760780334,0.8332407474517822,0.5922387838363647,0.40195754170417786,0.25363990664482117,0.9972585439682007,0.9446473717689514,0.44357454776763916,0.5888155102729797,0.3688054382801056,0.2110026776790619,0.16790752112865448,0.010274024680256844,0.5046085715293884,0.7927681803703308,0.07285496592521667,0.8019525408744812,0.5883767604827881,0.0405036099255085,0.4820171296596527,0.188790425658226,0.22243718802928925,0.6020722389221191,0.7702606320381165,0.07360490411520004,0.7017942070960999,0.018562905490398407,0.6460157632827759,0.29041850566864014,0.5751222968101501,0.40912067890167236,0.7265976071357727,0.6026481986045837,0.26270273327827454,0.2948898375034332,0.665690541267395,0.47435808181762695,0.36917752027511597,0.06358526647090912,0.565690815448761,0.04643912985920906,0.6920033693313599,0.6904599666595459,0.7231625914573669,0.5998018383979797,0.5368053913116455,0.9239375591278076,0.3611655533313751,0.8831526041030884,0.8676901459693909,0.7948328852653503,0.4128318130970001,0.4721153676509857,0.39066994190216064,0.2203458845615387,0.7242655158042908,0.8013365268707275,0.552757978439331,0.6674852967262268,0.9250481128692627,0.7821402549743652,0.13559220731258392,0.7416204810142517,0.9529541730880737,0.5500898361206055,0.3610351085662842,0.6199477314949036,0.7445921897888184,0.5828084945678711,0.22443918883800507,0.10699205845594406,0.6624839901924133,0.2649187445640564,0.9852810502052307,0.6515886187553406,0.5583882331848145,0.3834531009197235,0.9359939098358154,0.8240406513214111,0.631750226020813,0.03376126289367676,0.8696344494819641,0.17219533026218414,0.0512053407728672,0.1792028546333313,0.1741051971912384,0.1618472784757614,0.4589572846889496,0.5846876502037048,0.7165510654449463,0.6666379570960999,0.8034464716911316,0.8408963084220886,0.33244580030441284,0.9043223261833191,0.4367749094963074,0.5260583162307739,0.17052626609802246,0.7733922004699707,0.34092459082603455,0.7339818477630615,0.9219359159469604,0.236061692237854,0.05947384610772133,0.7501789927482605,0.9122900366783142,0.22693008184432983,0.4718751907348633,0.4210229218006134,0.11501990258693695,0.4955475628376007,0.7880254983901978,0.1566125601530075,0.379436194896698,0.28470107913017273,0.8385181427001953,0.669925332069397,0.44045940041542053,0.21030208468437195,0.43408194184303284,0.6881758570671082,0.999151349067688,0.8823217153549194,0.6883155107498169,0.41020721197128296,0.8052054643630981,0.5057960152626038,0.340267539024353,0.43431583046913147,0.5931105017662048,0.017704544588923454,0.7937710285186768,0.7223883271217346,0.41894444823265076,0.5870479941368103,0.5931740999221802,0.377535879611969,0.568333625793457,0.7279545664787292,0.9516356587409973,0.7008804082870483,0.5602816343307495,0.9451645016670227,0.04009874165058136,0.5847292542457581,0.04684118181467056,0.8882943391799927,0.12886029481887817,0.26304569840431213,0.6296922564506531,0.4929397702217102,0.26196181774139404,0.6638117432594299,0.33967679738998413,0.8484995365142822,0.634166955947876,0.38546454906463623,0.05189669504761696,0.2459883987903595,0.25851762294769287,0.789595901966095,0.3542202413082123,0.09381032735109329,0.06850702315568924,0.5028350949287415,0.6331533193588257,0.8120883107185364,0.027338240295648575]}]} \ No newline at end of file diff --git a/ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-output.json b/ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-output.json new file mode 100644 index 000000000..828a827f1 --- /dev/null +++ b/ods_ci/tests/Resources/Files/triton/modelmesh-triton-fil-output.json @@ -0,0 +1 @@ +{"model_name":"fil__isvc-7a1acc070b","model_version":"1","outputs":[{"name":"output__0","datatype":"FP32","shape":[1,2],"data":[0.99887556,0.0011244703]}]} \ No newline at end of file diff --git a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot index 7f2c35953..aa34b8690 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot @@ -36,7 +36,9 @@ ${EXPECTED_INFERENCE_REST_OUTPUT_FILE_PYTORCH}= tests/Resources/Files/trit ${INFERENCE_REST_INPUT_TENSORFLOW_FILE}= @${RESOURCES_DIRPATH}/modelmesh-triton-tensorflow-input.json ${EXPECTED_INFERENCE_TENSORFLOW_OUTPUT_FILE}= ${RESOURCES_DIRPATH}/modelmesh-triton-tensorflow-output.json ${TENSORFLOW_RUNTIME_FILEPATH}= ${RESOURCES_DIRPATH}/triton_tensorflow_modelmesh_runtime.yaml - +${FIL_MODEL_NAME}= fil +${INFERENCE_REST_INPUT_FIL_FILE}= @${RESOURCES_DIRPATH}/modelmesh-triton-fil-input.json +${EXPECTED_INFERENCE_FIL_OUTPUT_FILE}= ${RESOURCES_DIRPATH}/modelmesh-triton-fil-output.json *** Test Cases *** Test Onnx Model Rest Inference Via UI (Triton on Modelmesh) @@ -150,6 +152,41 @@ Test Tensorflow Model Rest Inference Via UI (Triton on Modelmesh) ... AND ... Clean All Models Of Current User +Test Tensorflow Model Rest Inference Via UI (Triton on Modelmesh) + [Documentation] Test the deployment of an onnx model in Kserve using Triton + [Tags] Sanity RHOAIENG-9069 + + Open Data Science Projects Home Page + Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} + ... existing_project=${FALSE} + Open Dashboard Settings settings_page=Serving runtimes + Upload Serving Runtime Template runtime_filepath=${TENSORFLOW_RUNTIME_FILEPATH} + ... serving_platform=multi runtime_protocol=REST + Serving Runtime Template Should Be Listed displayed_name=${ONNX_RUNTIME_NAME} + ... serving_platform=multi + Recreate S3 Data Connection project_title=${PRJ_TITLE} dc_name=model-serving-connection + ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} + ... aws_bucket_name=ods-ci-s3 + Create Model Server token=${TRUE} runtime=${ONNX_RUNTIME_NAME} server_name=${ONNX_RUNTIME_NAME} existing_server=${TRUE} + Sleep 10s + Serve Model project_name=${PRJ_TITLE} model_name=${FIL_MODEL_NAME} framework=xgboost - 1 + ... existing_data_connection=${TRUE} data_connection_name=model-serving-connection + ... model_path=triton/model_repository/fil/ model_server=${ONNX_RUNTIME_NAME} + Wait Until Runtime Pod Is Running server_name=${ONNX_RUNTIME_NAME} + ... project_title=${PRJ_TITLE} timeout=5m + Verify Model Status ${FIL_MODEL_NAME} success + ${EXPECTED_INFERENCE_REST_OUTPUT_FIL}= Load Json File file_path=${EXPECTED_INFERENCE_FIL_OUTPUT_FILE} + ... as_string=${TRUE} + Log ${EXPECTED_INFERENCE_REST_OUTPUT_FIL} + Verify Model Inference With Retries ${FIL_MODEL_NAME} ${INFERENCE_REST_INPUT_FIL_FILE} + ... ${EXPECTED_INFERENCE_REST_OUTPUT_FIL} + ... token_auth=${TRUE} + ... project_title=${PRJ_TITLE} + [Teardown] Run Keywords Get Modelmesh Events And Logs model_name=${FIL_MODEL_NAME} + ... project_title=${PRJ_TITLE} + ... AND + ... Clean All Models Of Current User + *** Keywords *** From b65f167283082603dbc529dcd63a96991f7c3fc7 Mon Sep 17 00:00:00 2001 From: rpancham <rpancham@redhat.com> Date: Tue, 19 Nov 2024 18:53:01 +0530 Subject: [PATCH 2/4] Modified test case name --- .../1010__model_serving_triton_on_modelmesh.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot index aa34b8690..b6f1e8c47 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot @@ -152,7 +152,7 @@ Test Tensorflow Model Rest Inference Via UI (Triton on Modelmesh) ... AND ... Clean All Models Of Current User -Test Tensorflow Model Rest Inference Via UI (Triton on Modelmesh) +Test FIL Model Rest Inference Via UI (Triton on Modelmesh) [Documentation] Test the deployment of an onnx model in Kserve using Triton [Tags] Sanity RHOAIENG-9069 From 90fe951b4c367001dd586d2a72d197fed5e853ba Mon Sep 17 00:00:00 2001 From: rpancham <rpancham@redhat.com> Date: Wed, 4 Dec 2024 19:23:19 +0530 Subject: [PATCH 3/4] comment line --- .../1010__model_serving_triton_on_modelmesh.robot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot index 75904c58e..a84c00985 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot @@ -164,8 +164,8 @@ Test FIL Model Rest Inference Via UI (Triton on Modelmesh) Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} Open Dashboard Settings settings_page=Serving runtimes - Upload Serving Runtime Template runtime_filepath=${TENSORFLOW_RUNTIME_FILEPATH} - ... serving_platform=multi runtime_protocol=REST + # Upload Serving Runtime Template runtime_filepath=${TENSORFLOW_RUNTIME_FILEPATH} + # ... serving_platform=multi runtime_protocol=REST Serving Runtime Template Should Be Listed displayed_name=${ONNX_RUNTIME_NAME} ... serving_platform=multi Recreate S3 Data Connection project_title=${PRJ_TITLE} dc_name=model-serving-connection From 8e43baabde3b803e31869ca35097dbffbad05e61 Mon Sep 17 00:00:00 2001 From: rpancham <rpancham@redhat.com> Date: Wed, 4 Dec 2024 20:00:42 +0530 Subject: [PATCH 4/4] fix --- .../1010__model_serving_triton_on_modelmesh.robot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot index a84c00985..75904c58e 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1010__model_serving_triton_on_modelmesh/1010__model_serving_triton_on_modelmesh.robot @@ -164,8 +164,8 @@ Test FIL Model Rest Inference Via UI (Triton on Modelmesh) Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} Open Dashboard Settings settings_page=Serving runtimes - # Upload Serving Runtime Template runtime_filepath=${TENSORFLOW_RUNTIME_FILEPATH} - # ... serving_platform=multi runtime_protocol=REST + Upload Serving Runtime Template runtime_filepath=${TENSORFLOW_RUNTIME_FILEPATH} + ... serving_platform=multi runtime_protocol=REST Serving Runtime Template Should Be Listed displayed_name=${ONNX_RUNTIME_NAME} ... serving_platform=multi Recreate S3 Data Connection project_title=${PRJ_TITLE} dc_name=model-serving-connection