diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 2333463d4..c4a50976f 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -31,9 +31,15 @@ ${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]// *** Keywords *** Create Model Server [Documentation] Keyword to create a Model Server in a Data Science Project - [Arguments] ${no_replicas}=1 ${server_size}=Small ${ext_route}=${TRUE} - ... ${token}=${TRUE} ${runtime}=OpenVINO Model Server ${server_name}=Model Serving Test - ... ${no_gpus}=0 ${existing_server}=${FALSE} + [Arguments] ${no_replicas}=1 + ... ${server_size}=Small + ... ${ext_route}=${TRUE} + ... ${token}=${TRUE} + ... ${runtime}=OpenVINO Model Server + ... ${server_name}=Model Serving Test + ... ${no_gpus}=0 + ... ${existing_server}=${FALSE} + ... ${service_account_name}=${NONE} Move To Tab Models IF ${existing_server} ${existing_server}= Run Keyword And Return Status Wait Until Page Contains Element //button[.="${server_name}"] @@ -63,8 +69,9 @@ Create Model Server Log message=Token Auth should be enabled by default..(from v2.5) SeleniumLibrary.Checkbox Should Be Selected ${TOKEN_AUTH_CHECKBOX_XP} END - ELSE IF ${token}==${TRUE} - Enable Token Authentication + END + IF ${token} + Enable Token Authentication service_account_name=${service_account_name} END SeleniumLibrary.Wait Until Element Is Enabled //button[contains(text(),"Add")] SeleniumLibrary.Click Button Add @@ -181,9 +188,10 @@ Enable Token Authentication [Documentation] Enables Token authentication to serving route [Arguments] ${service_account_name}=${NONE} SeleniumLibrary.Select Checkbox ${TOKEN_AUTH_CHECKBOX_XP} - IF "${service_account_name}" != "${NONE}" - Input Service Account Name ${service_account_name} + IF "${service_account_name}" == "${NONE}" + ${service_account_name}= Set Variable default-name END + Input Service Account Name ${service_account_name} Disable Token Authentication [Documentation] Disable Token authentication to serving route @@ -211,14 +219,30 @@ Input Service Account Name Get Model Serving Access Token via UI [Documentation] Returns the token used for authentication to the serving route ... TODO: There can be multiple tokens defined for each model server, handle this case as well - [Arguments] ${service_account_name}=default-name ${single_model}=${FALSE} ${model_name}=${NONE} + [Arguments] ${service_account_name}=default-name ${single_model}=${FALSE} + ... ${model_name}=${NONE} ${multi_model_servers}=${FALSE} IF ${single_model} # Expand the model SeleniumLibrary.Click Button xpath: //a[text()='${model_name}']/parent::div/parent::td[@data-label='Name']/preceding-sibling::td/button # robocop: off=line-too-long ${token}= Get Single Model Token ${service_account_name} ELSE SeleniumLibrary.Wait Until Page Contains Element xpath://td[@data-label="Tokens"]/button - SeleniumLibrary.Click Element xpath://td[@data-label="Tokens"]/button + # TODO: ModelMesh requires passing model server name and not model name; + # in particular when there are multiple model servers + # in the same project. + # This is a temporary workaround until the issue is fixed. + # With one model server, the model name is not required. + IF ${multi_model_servers} + SeleniumLibrary.Click Button + ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@data-label='Tokens']//button + ELSE + SeleniumLibrary.Click Element xpath://td[@data-label="Tokens"]/button + END + # TODO: service account name is required; remove from all places that pass `None` + IF "${service_account_name}" == "${NONE}" + ${service_account_name}= Set Variable default-name + END + ${token}= SeleniumLibrary.Get Element Attribute ... xpath://div[.="${service_account_name} "]/../../td[@data-label="Token Secret"]//input value END diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 66a86b725..7d767972b 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -1,4 +1,4 @@ -# robocop: off=wrong-case-in-keyword-name,too-many-arguments,too-long-keyword +# robocop: off=wrong-case-in-keyword-name,too-many-arguments,too-long-keyword,line-too-long *** Settings *** Documentation Collection of keywords to interact with Model Serving Resource ../../../Page/Components/Components.resource @@ -75,26 +75,10 @@ Serve Model SeleniumLibrary.Click Button ${DEPLOY_MODEL_BTN} END SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] - Set Model Name ${model_name} - Select Model Server ${model_server} - IF "${serving_runtime}" != "${NONE}" Set Model Server Runtime ${serving_runtime} - SeleniumLibrary.Wait Until Page Contains Element xpath://span[.="Model framework (name - version)"] - Select Framework ${framework} - IF ${existing_data_connection}==${TRUE} - # Select Radio Button group_name=radiogroup value=existing-data-connection-radio - # Selected by default, let's skip for now - Select Existing Data Connection ${data_connection_name} - Set Folder Path ${model_path} - ELSE - # Existing connection radio is selected by default; for now blindly click on new connection radio - SeleniumLibrary.Click Element //input[@id="new-data-connection-radio"] - # Select Radio Button group_name=radiogroup value=new-data-connection-radio - Set Up New Data Connection dc_name=${data_connection_name} - Set Folder Path ${model_path} - END - SeleniumLibrary.Wait Until Element Is Enabled //button[contains(text(),"Deploy")] - SeleniumLibrary.Click Button Deploy - SeleniumLibrary.Wait Until Page Does Not Contain xpath://h1[.="Deploy model"] + Fill Deploy Model Form model_name=${model_name} model_server=${model_server} + ... serving_runtime=${serving_runtime} framework=${framework} + ... data_connection_name=${data_connection_name} existing_data_connection=${existing_data_connection} + ... model_path=${model_path} Select Project [Documentation] Selects a project in the "deploy model" modal. @@ -244,10 +228,12 @@ Open ${section} Options Menu Get Access Token Via UI [Documentation] Returns the access token for models deployed in a specific project ... by using the UI of DSP - [Arguments] ${project_name} ${service_account_name}=default-name ${single_model}=${FALSE} - ... ${model_name}=${NONE} + [Arguments] ${project_name} ${service_account_name}=default-name + ... ${single_model}=${FALSE} ${model_name}=${NONE} + ... ${multi_model_servers}=${FALSE} Open Data Science Project Details Page ${project_name} tab_id=model-server ${token}= Get Model Serving Access Token via UI ${service_account_name} ${single_model} ${model_name} + ... ${multi_model_servers} RETURN ${token} Get Model Project @@ -387,18 +373,11 @@ Clean Up Model Serving Page [Documentation] Deletes all currently deployed models, if any are present. # Returns an empty list if no matching elements found Switch Model Serving Project project_name=All projects - ${projects}= Get WebElements xpath://table/tbody/tr/td[@data-label="Project"] - FOR ${project} IN @{projects} - ${project}= Get Text ${project} - @{project description}= Split String ${project} - Switch Model Serving Project ${project description}[0] - ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a - FOR ${model} IN @{models} - ${model}= Get Text ${model} - Delete Model Via UI ${model} - Sleep 1s - END - Switch Model Serving Project project_name=All projects + ${models}= Get WebElements xpath://table/tbody/tr/td[@data-label="Name"]/div/a + FOR ${model} IN @{models} + ${model}= Get Text ${model} + Delete Model Via UI ${model} + Sleep 1s END Add Namespace To ServiceMeshMemberRoll @@ -643,3 +622,53 @@ Set Up Project ELSE Log message=Skipping UserWorkloadMonitoring enablement. END + +Deploy Model From Models Tab + [Documentation] Deploys a model from the project's Models tab + [Arguments] ${project_name} + ... ${data_connection_name} + ... ${model_name} + ... ${framework} + ... ${existing_data_connection} + ... ${model_path} + ... ${model_server} + ... ${serving_runtime}=${NONE} + Open Data Science Project Details Page ${project_name} tab_id=model-server + SeleniumLibrary.Click Button + ... xpath://*[@id="expand-table-row-${model_name}-1-undefined-1"]/../../td[@class='pf-v5-c-table__td']//button + SeleniumLibrary.Wait Until Page Contains Element //div[@role="dialog"] + Fill Deploy Model Form model_name=${model_name} model_server=${model_server} + ... serving_runtime=${serving_runtime} framework=${framework} + ... data_connection_name=${data_connection_name} existing_data_connection=${existing_data_connection} + ... model_path=${model_path} + +Fill Deploy Model Form # robocop: off=too-many-calls-in-keyword + [Documentation] Fills the deploy model form + [Arguments] ${model_name} + ... ${model_server} + ... ${serving_runtime} + ... ${framework} + ... ${data_connection_name} + ... ${existing_data_connection} + ... ${model_path} + Set Model Name ${model_name} + Select Model Server ${model_server} + IF "${serving_runtime}" != "${NONE}" Set Model Server Runtime ${serving_runtime} + SeleniumLibrary.Wait Until Page Contains Element + ... xpath://span[.="Model framework (name - version)"] + Select Framework ${framework} + IF ${existing_data_connection} + # Select Radio Button group_name=radiogroup value=existing-data-connection-radio + # Selected by default, let's skip for now + Select Existing Data Connection ${data_connection_name} + Set Folder Path ${model_path} + ELSE + # Existing connection radio is selected by default; for now blindly click on new connection radio + SeleniumLibrary.Click Element //input[@id="new-data-connection-radio"] + # Select Radio Button group_name=radiogroup value=new-data-connection-radio + Set Up New Data Connection dc_name=${data_connection_name} + Set Folder Path ${model_path} + END + SeleniumLibrary.Wait Until Element Is Enabled //button[contains(text(),"Deploy")] + SeleniumLibrary.Click Button Deploy + SeleniumLibrary.Wait Until Page Does Not Contain xpath://h1[.="Deploy model"] diff --git a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot index 91dbabb40..91eda1500 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1008_model_serving_cross_auth.robot @@ -1,6 +1,6 @@ # robocop: off=too-long-test-case,too-many-calls-in-test-case,wrong-case-in-keyword-name *** Settings *** -Documentation Suite of test cases for OVMS in Kserve +Documentation Suite of test cases for OVMS in Kserve and ModelMesh Library OperatingSystem Library ../../../libs/Helpers.py Resource ../../Resources/Page/ODH/JupyterHub/HighAvailability.robot @@ -11,9 +11,8 @@ Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Mo Resource ../../Resources/Page/ODH/Monitoring/Monitoring.resource Resource ../../Resources/OCP.resource Resource ../../Resources/CLI/ModelServing/modelmesh.resource -Suite Setup Cross Auth On Kserve Suite Setup -Suite Teardown Cross Auth On Kserve Suite Teardown -Test Tags Kserve Modelmesh +Test Teardown Cross Auth Test Teardown +Test Tags Sanity ProductBug *** Variables *** @@ -22,43 +21,59 @@ ${PRJ_TITLE}= cross-auth-prj ${PRJ_DESCRIPTION}= project used for validating cross-auth CVE ${MODEL_CREATED}= ${FALSE} ${MODEL_NAME}= test-model -${SECOND_MODEL_NAME}= test-model-second -${RUNTIME_NAME}= Model Serving Test +${SECOND_MODEL_NAME}= ${MODEL_NAME}-second ${EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable ${SECOND_EXPECTED_INFERENCE_OUTPUT}= {"model_name":"${SECOND_MODEL_NAME}__isvc-83d6fab7bd","model_version":"1","outputs":[{"name":"Plus214_Output_0","datatype":"FP32","shape":[1,10],"data":[-8.233053,-7.7497034,-3.4236815,12.3630295,-12.079103,17.266596,-10.570976,0.7130762,3.321715,1.3621228]}]} #robocop: disable +${FIRST_SERVICE_ACCOUNT}= first_account +${SECOND_SERVICE_ACCOUNT}= second_account *** Test Cases *** Test Cross Model Authentication On Kserve [Documentation] Tests for the presence of CVE-2024-7557 when using Kserve - [Tags] Sanity ProductBug - ... RHOAIENG-11007 RHOAIENG-12048 + [Tags] Kserve RHOAIENG-11007 RHOAIENG-12048 + Set Test Variable $serving_mode kserve + Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} + Template with embedded arguments + +Test Cross Model Authentication On ModelMesh + [Documentation] Tests for the presence of CVE-2024-7557 when using ModelMesh + [Tags] ModelMesh RHOAIENG-11007 RHOAIENG-12853 + Set Test Variable $serving_mode modelmeshserving + Set Test Variable $project_name ${PRJ_TITLE}-${serving_mode} + Template with embedded arguments + + +*** Keywords *** +Template with embedded arguments # robocop: off=too-many-calls-in-keyword + [Documentation] Template for cross-auth test cases + Cross Auth Test Setup + ${single_model}= Set Variable If "${serving_mode}" == "kserve" ${True} ${False} Open Data Science Projects Home Page - Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} + Create Data Science Project title=${project_name} description=${PRJ_DESCRIPTION} ... existing_project=${FALSE} - Recreate S3 Data Connection project_title=${PRJ_TITLE} dc_name=model-serving-connection - ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} - ... aws_bucket_name=ods-ci-s3 - Deploy Kserve Model Via UI model_name=${MODEL_NAME} serving_runtime=OpenVINO Model Server - ... data_connection=model-serving-connection path=test-dir model_framework=onnx - ... service_account_name=first_account token=${TRUE} - Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${MODEL_NAME} - ... namespace=${PRJ_TITLE} - ${first_token}= Get Model Serving Access Token via UI service_account_name=first_account single_model=${TRUE} - ... model_name=${MODEL_NAME} - Deploy Kserve Model Via UI model_name=${SECOND_MODEL_NAME} serving_runtime=OpenVINO Model Server - ... data_connection=model-serving-connection path=test-dir model_framework=onnx - ... service_account_name=second_account token=${TRUE} - Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${SECOND_MODEL_NAME} - ... namespace=${PRJ_TITLE} - ${second_token}= Get Model Serving Access Token via UI service_account_name=second_account - ... single_model=${TRUE} model_name=${SECOND_MODEL_NAME} + Cross Auth Model Deployment single_model=${single_model} + ... model_name=${MODEL_NAME} service_account_name=${FIRST_SERVICE_ACCOUNT} + + ${first_token}= Get Access Token Via UI service_account_name=${FIRST_SERVICE_ACCOUNT} + ... single_model=${single_model} model_name=${MODEL_NAME} project_name=${project_name} + ... multi_model_servers=not ${single_model} + + Cross Auth Model Deployment single_model=${single_model} + ... model_name=${SECOND_MODEL_NAME} service_account_name=${SECOND_SERVICE_ACCOUNT} + + ${second_token}= Get Access Token Via UI service_account_name=${SECOND_SERVICE_ACCOUNT} + ... single_model=${single_model} model_name=${SECOND_MODEL_NAME} project_name=${project_name} + ... multi_model_servers=not ${single_model} + Verify Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${first_token} - ... project_title=${PRJ_TITLE} + ... project_title=${project_name} + Verify Model Inference model_name=${SECOND_MODEL_NAME} inference_input=${INFERENCE_INPUT} ... expected_inference_output=${SECOND_EXPECTED_INFERENCE_OUTPUT} token_auth=${TRUE} token=${second_token} - ... project_title=${PRJ_TITLE} + ... project_title=${project_name} + # Should not be able to query first model with second token # Will fail at this step until CVE is fixed from dashboard side ${inf_out}= Get Model Inference model_name=${MODEL_NAME} inference_input=${INFERENCE_INPUT} @@ -67,25 +82,51 @@ Test Cross Model Authentication On Kserve ${inf_out}= Get Model Inference model_name=${SECOND_MODEL_NAME} inference_input=${INFERENCE_INPUT} ... token_auth=${TRUE} token=${first_token} Run Keyword And Warn On Failure Should Contain ${inf_out} Log in with OpenShift - [Teardown] Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs - ... model_name=${MODEL_NAME} project_title=${PRJ_TITLE} AND Clean All Models Of Current User +Cross Auth Model Deployment # robocop: off=too-many-calls-in-keyword + [Documentation] Deploys a model with cross auth enabled + [Arguments] ${single_model} ${model_name} ${service_account_name} + ${dc_name}= Set Variable model-serving-connection-${serving_mode} + Recreate S3 Data Connection project_title=${project_name} dc_name=${dc_name} + ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} + ... aws_bucket_name=ods-ci-s3 + Open Data Science Project Details Page ${project_name} tab_id=model-server + IF ${single_model} + Deploy Kserve Model Via UI model_name=${model_name} serving_runtime=OpenVINO Model Server + ... data_connection=${dc_name} path=test-dir model_framework=onnx + ... service_account_name=${service_account_name} token=${TRUE} + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} + ... namespace=${project_name} + ELSE + Create Model Server token=${TRUE} server_name=${model_name} service_account_name=${service_account_name} + sleep 1m + Deploy Model From Models Tab project_name=${project_name} model_name=${model_name} framework=onnx + ... existing_data_connection=${TRUE} data_connection_name=${dc_name} model_server=${model_name} + ... model_path=mnist-8.onnx + Wait Until Keyword Succeeds 5 min 10 sec Verify Openvino Deployment runtime_name=${model_name} + ... project_name=${project_name} + Wait Until Keyword Succeeds 5 min 10 sec Verify Serving Service project_name=${project_name} + Verify Model Status model_name=${model_name} expected_status=success + END -*** Keywords *** -Cross Auth On Kserve Suite Setup - [Documentation] Suite setup steps for testing DSG. It creates some test variables +Cross Auth Test Setup + [Documentation] Test setup steps for testing DSG. It creates some test variables ... and runs RHOSi setup + Set Library Search Order SeleniumLibrary - Skip If Component Is Not Enabled kserve + Skip If Component Is Not Enabled ${serving_mode} RHOSi Setup Launch Dashboard ${TEST_USER.USERNAME} ${TEST_USER.PASSWORD} ${TEST_USER.AUTH_TYPE} ... ${ODH_DASHBOARD_URL} ${BROWSER.NAME} ${BROWSER.OPTIONS} Fetch Knative CA Certificate filename=openshift_ca_istio_knative.crt Clean All Models Of Current User -Cross Auth On Kserve Suite Teardown - [Documentation] Suite teardown steps after testing DSG. It Deletes +Cross Auth Test Teardown + [Documentation] Test teardown steps after testing DSG. It Deletes ... all the DS projects created by the tests and run RHOSi teardown + Run Keywords Run Keyword If Test Failed Get Kserve Events And Logs + ... model_name=${MODEL_NAME} project_title=${project_name} AND Clean All Models Of Current User + # Even if kw fails, deleting the whole project will also delete the model # Failure will be shown in the logs of the run nonetheless IF ${MODEL_CREATED} @@ -93,7 +134,7 @@ Cross Auth On Kserve Suite Teardown ELSE Log Model not deployed, skipping deletion step during teardown console=true END - ${projects}= Create List ${PRJ_TITLE} + ${projects}= Create List ${project_name} Delete List Of Projects Via CLI ocp_projects=${projects} # Will only be present on SM cluster runs, but keyword passes # if file does not exist