From fe92110b786c4d8e2d407aa8bc727cd6250b1df8 Mon Sep 17 00:00:00 2001 From: saimachi Date: Wed, 26 Jun 2024 18:18:19 -0400 Subject: [PATCH 1/7] Parameterize load test resource allocations. --- .github/workflows/load-testing.yml | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index 770fefec09..c3329c5a0f 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -44,6 +44,54 @@ on: - sandbox - e2e default: e2e + core_api_replica_count: + description: Fixed number of replicas for Core API (enter 0 for the default auto-scaling policy) + type: number + default: 0 + core_api_cpu: + description: Core API CPU allocation + type: number + default: 1.0 + core_api_mem: + description: Core API memory allocation + type: string + default: 2.0Gi + orchestration_api_replica_count: + description: Fixed number of replicas for Orchestration API (enter 0 for the default auto-scaling policy) + type: number + default: 0 + orchestration_api_cpu: + description: Orchestration API CPU allocation + type: number + default: 1.0 + orchestration_api_mem: + description: Orchestration API memory allocation + type: string + default: 2.0Gi + langchain_api_replica_count: + description: Fixed number of replicas for LangChain API (enter 0 for the default auto-scaling policy) + type: number + default: 0 + langchain_api_cpu: + description: LangChain API CPU allocation + type: number + default: 1.0 + langchain_api_mem: + description: LangChain API memory allocation + type: string + default: 2.0Gi + semantickernel_api_replica_count: + description: Fixed number of replicas for Semantic Kernel API (enter 0 for the default auto-scaling policy) + type: number + default: 0 + semantickernel_api_cpu: + description: Semantic Kernel API CPU allocation + type: number + default: 1.0 + semantickernel_api_mem: + description: Semantic Kernel API memory allocation + type: string + default: 2.0Gi jobs: generate_matrix: @@ -323,6 +371,35 @@ jobs: $env:AZURE_SUBSCRIPTION_ID=$($info.subscriptionId) Push-Location ./deploy/quick-start + + if (${{ inputs.core_api_replica_count }}) { + azd env set SERVICE_COREAPI_REPLICAS ${{ inputs.core_api_replica_count }} + } + + if (${{ inputs.orchestration_api_replica_count }}) { + azd env set SERVICE_ORCHESTRATIONAPI_REPLICAS ${{ inputs.orchestration_api_replica_count }} + } + + if (${{ inputs.langchain_api_replica_count }}) { + azd env set SERVICE_LANGCHAINAPI_REPLICAS ${{ inputs.langchain_api_replica_count }} + } + + if (${{ inputs.semantickernel_api_replica_count }}) { + azd env set SERVICE_SEMANTICKERNELAPI_REPLICAS ${{ inputs.semantickernel_api_replica_count }} + } + + azd env set SERVICE_COREAPI_CPU ${{ inputs.core_api_cpu }} + azd env set SERVICE_COREAPI_MEMORY ${{ inputs.core_api_mem }} + + azd env set SERVICE_ORCHESTRATIONAPI_CPU ${{ inputs.orchestration_api_cpu }} + azd env set SERVICE_ORCHESTRATIONAPI_MEMORY ${{ inputs.orchestration_api_mem }} + + azd env set SERVICE_LANGCHAINAPI_CPU ${{ inputs.langchain_api_cpu }} + azd env set SERVICE_LANGCHAINAPI_MEMORY ${{ inputs.langchain_api_mem }} + + azd env set SERVICE_SEMANTICKERNELAPI_CPU ${{ inputs.semantickernel_api_cpu }} + azd env set SERVICE_SEMANTICKERNELAPI_MEMORY ${{ inputs.semantickernel_api_mem }} + azd provision --no-prompt Pop-Location shell: pwsh From f8e2c74f535e1a1349394edf674f90407428a45a Mon Sep 17 00:00:00 2001 From: saimachi Date: Wed, 26 Jun 2024 18:33:02 -0400 Subject: [PATCH 2/7] Reuse existing App Registrations --- .github/workflows/load-testing.yml | 46 +++++++++--------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index c3329c5a0f..b6a37d271a 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -251,20 +251,6 @@ jobs: --tenant-id "$($info.tenantId)" shell: pwsh - - name: Provision App Registrations - id: provision-app-registrations - run: | - Write-Host "Provision App Registrations" - Push-Location ./tests/scripts - ./Create-FllmEntraIdApps.ps1 ` - -authAppName FoundationaLLM-Authorization-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -coreAppName FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -coreClientAppName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` - -mgmtAppName FoundationaLLM-Management-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -mgmtClientAppName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} - Pop-Location - shell: pwsh - - name: Provision Admin Group id: provision-admin-group run: | @@ -354,11 +340,11 @@ jobs: -tenantId $($info.tenantId) ` -principalType ServicePrincipal ` -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 ` - -fllmApiName FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -fllmClientName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` - -fllmMgmtApiName FoundationaLLM-Management-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -fllmMgmtClientName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` - -fllmAuthApiName FoundationaLLM-Authorization-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` + -fllmApiName FoundationaLLM-Core-API ` + -fllmClientName FoundationaLLM-Core-Portal ` + -fllmMgmtApiName FoundationaLLM-Management-API ` + -fllmMgmtClientName FoundationaLLM-Management-Portal ` + -fllmAuthApiName FoundationaLLM-Authorization-API ` -fllmAdminGroupName FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} Pop-Location shell: pwsh @@ -422,8 +408,8 @@ jobs: Write-Host "Update App Registration OAuth Callbacks" Push-Location ./deploy/quick-start pwsh -File ../../tests/scripts/Update-OAuthCallbackUris.ps1 ` - -fllmChatUiName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` - -fllmMgmtUiName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} + -fllmChatUiName FoundationaLLM-Core-Portal ` + -fllmMgmtUiName FoundationaLLM-Management-Portal Pop-Location shell: pwsh @@ -682,7 +668,7 @@ jobs: echo "AZURE_APP_CONFIG_NAME=$($configurationParameters['AZURE_APP_CONFIG_NAME'])" >> $env:GITHUB_ENV echo "AZURE_ENV_NAME=$($configurationParameters['AZURE_ENV_NAME'])" >> $env:GITHUB_ENV - $token = (az account get-access-token --scope "api://FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }}/.default" --query accessToken | ConvertFrom-Json) + $token = (az account get-access-token --scope "api://FoundationaLLM-Core/.default" --query accessToken | ConvertFrom-Json) $kvUrl = (az keyvault secret set --vault-name "fllme2emkeyvault7f393d64" --name "bearer-token-${{ needs.generate_matrix.outputs.unique_id }}" --value $token --query '{id: id}' | ConvertFrom-Json).id # Replace Placeholders @@ -821,21 +807,15 @@ jobs: -principalType ServicePrincipal ` -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 + Write-Host "Removing App Registration OAuth Callbacks" + pwsh -File ../../tests/scripts/Remove-OAuthCallbackUris.ps1 ` + -fllmChatUiName FoundationaLLM-Core-Portal ` + -fllmMgmtUiName FoundationaLLM-Management-Portal + azd env refresh --no-prompt azd down --force --purge --no-prompt Pop-Location Write-Host "Remove Admin Group" az ad group delete --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} --output json - - Write-Host "Tear down App Registrations" - Push-Location ./tests/scripts - ./Remove-EntraIdApps.ps1 ` - -authAppName FoundationaLLM-Authorization-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -coreAppName FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -coreClientAppName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` - -mgmtAppName FoundationaLLM-Management-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` - -mgmtClientAppName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` - -interactiveMode $false - Pop-Location shell: pwsh From 7864931e2773c037c1f50d6d513793ebd43dea2c Mon Sep 17 00:00:00 2001 From: saimachi Date: Wed, 26 Jun 2024 19:57:26 -0400 Subject: [PATCH 3/7] Get Load Testing JSON Data --- .github/workflows/load-testing.yml | 92 +++++++---------------- tests/config/load-testing-parameters.json | 16 +++- 2 files changed, 43 insertions(+), 65 deletions(-) diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index b6a37d271a..cdc9d6f0f7 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -44,54 +44,6 @@ on: - sandbox - e2e default: e2e - core_api_replica_count: - description: Fixed number of replicas for Core API (enter 0 for the default auto-scaling policy) - type: number - default: 0 - core_api_cpu: - description: Core API CPU allocation - type: number - default: 1.0 - core_api_mem: - description: Core API memory allocation - type: string - default: 2.0Gi - orchestration_api_replica_count: - description: Fixed number of replicas for Orchestration API (enter 0 for the default auto-scaling policy) - type: number - default: 0 - orchestration_api_cpu: - description: Orchestration API CPU allocation - type: number - default: 1.0 - orchestration_api_mem: - description: Orchestration API memory allocation - type: string - default: 2.0Gi - langchain_api_replica_count: - description: Fixed number of replicas for LangChain API (enter 0 for the default auto-scaling policy) - type: number - default: 0 - langchain_api_cpu: - description: LangChain API CPU allocation - type: number - default: 1.0 - langchain_api_mem: - description: LangChain API memory allocation - type: string - default: 2.0Gi - semantickernel_api_replica_count: - description: Fixed number of replicas for Semantic Kernel API (enter 0 for the default auto-scaling policy) - type: number - default: 0 - semantickernel_api_cpu: - description: Semantic Kernel API CPU allocation - type: number - default: 1.0 - semantickernel_api_mem: - description: Semantic Kernel API memory allocation - type: string - default: 2.0Gi jobs: generate_matrix: @@ -103,6 +55,7 @@ jobs: test_matrix: ${{ steps.set-test-matrix.outputs.result }} docker_tag: ${{ steps.docker-tag.outputs.docker_tag }} unique_id: ${{ steps.set-unique-id.outputs.digest }} + qs_resources: ${{ steps.qs-resources.outputs.digest }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -161,6 +114,17 @@ jobs: echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV echo "docker_tag: Latest Tag is : $DOCKER_TAG" echo "docker_tag=$DOCKER_TAG" >> $GITHUB_OUTPUT + + - name: Resource Parameterization + id: qs-resources + uses: actions/github-script@v7 + with: + result-encoding: string + script: | + const fs = require('fs') + const jsonString = fs.readFileSync('./tests/config/load-testing-parameters.json') + var data = JSON.parse(jsonString) + return JSON.stringify(data.resources) helm_build: name: Package Helm Charts @@ -358,33 +322,33 @@ jobs: Push-Location ./deploy/quick-start - if (${{ inputs.core_api_replica_count }}) { - azd env set SERVICE_COREAPI_REPLICAS ${{ inputs.core_api_replica_count }} + if (${{ fromJson(needs.generate_matrix.outputs.qs_resources).core_api_replica_count }}) { + azd env set SERVICE_COREAPI_REPLICAS ${{ fromJson(needs.generate_matrix.outputs.qs_resources).core_api_replica_count }} } - if (${{ inputs.orchestration_api_replica_count }}) { - azd env set SERVICE_ORCHESTRATIONAPI_REPLICAS ${{ inputs.orchestration_api_replica_count }} + if (${{ fromJson(needs.generate_matrix.outputs.qs_resources).orchestration_api_replica_count }}) { + azd env set SERVICE_ORCHESTRATIONAPI_REPLICAS ${{ fromJson(needs.generate_matrix.outputs.qs_resources).orchestration_api_replica_count }} } - if (${{ inputs.langchain_api_replica_count }}) { - azd env set SERVICE_LANGCHAINAPI_REPLICAS ${{ inputs.langchain_api_replica_count }} + if (${{ fromJson(needs.generate_matrix.outputs.qs_resources).langchain_api_replica_count }}) { + azd env set SERVICE_LANGCHAINAPI_REPLICAS ${{ fromJson(needs.generate_matrix.outputs.qs_resources).langchain_api_replica_count }} } - if (${{ inputs.semantickernel_api_replica_count }}) { - azd env set SERVICE_SEMANTICKERNELAPI_REPLICAS ${{ inputs.semantickernel_api_replica_count }} + if (${{ fromJson(needs.generate_matrix.outputs.qs_resources).semantickernel_api_replica_count }}) { + azd env set SERVICE_SEMANTICKERNELAPI_REPLICAS ${{ fromJson(needs.generate_matrix.outputs.qs_resources).semantickernel_api_replica_count }} } - azd env set SERVICE_COREAPI_CPU ${{ inputs.core_api_cpu }} - azd env set SERVICE_COREAPI_MEMORY ${{ inputs.core_api_mem }} + azd env set SERVICE_COREAPI_CPU ${{ fromJson(needs.generate_matrix.outputs.qs_resources).core_api_cpu }} + azd env set SERVICE_COREAPI_MEMORY ${{ fromJson(needs.generate_matrix.outputs.qs_resources).core_api_mem }} - azd env set SERVICE_ORCHESTRATIONAPI_CPU ${{ inputs.orchestration_api_cpu }} - azd env set SERVICE_ORCHESTRATIONAPI_MEMORY ${{ inputs.orchestration_api_mem }} + azd env set SERVICE_ORCHESTRATIONAPI_CPU ${{ fromJson(needs.generate_matrix.outputs.qs_resources).orchestration_api_cpu }} + azd env set SERVICE_ORCHESTRATIONAPI_MEMORY ${{ fromJson(needs.generate_matrix.outputs.qs_resources).orchestration_api_mem }} - azd env set SERVICE_LANGCHAINAPI_CPU ${{ inputs.langchain_api_cpu }} - azd env set SERVICE_LANGCHAINAPI_MEMORY ${{ inputs.langchain_api_mem }} + azd env set SERVICE_LANGCHAINAPI_CPU ${{ fromJson(needs.generate_matrix.outputs.qs_resources).langchain_api_cpu }} + azd env set SERVICE_LANGCHAINAPI_MEMORY ${{ fromJson(needs.generate_matrix.outputs.qs_resources).langchain_api_mem }} - azd env set SERVICE_SEMANTICKERNELAPI_CPU ${{ inputs.semantickernel_api_cpu }} - azd env set SERVICE_SEMANTICKERNELAPI_MEMORY ${{ inputs.semantickernel_api_mem }} + azd env set SERVICE_SEMANTICKERNELAPI_CPU ${{ fromJson(needs.generate_matrix.outputs.qs_resources).semantickernel_api_cpu }} + azd env set SERVICE_SEMANTICKERNELAPI_MEMORY ${{ fromJson(needs.generate_matrix.outputs.qs_resources).semantickernel_api_mem }} azd provision --no-prompt Pop-Location diff --git a/tests/config/load-testing-parameters.json b/tests/config/load-testing-parameters.json index 6933d6c944..77536cc4db 100644 --- a/tests/config/load-testing-parameters.json +++ b/tests/config/load-testing-parameters.json @@ -4,5 +4,19 @@ "question_bank_file": "QuestionBank.csv", "jmeter_file": "Load-Testing.jmx", "resource_group_name": "E2E-Testing-Resources", - "load_test_resource": "fllm-loadtest" + "load_test_resource": "fllm-loadtest", + "resources": { + "core_api_replica_count": 2, + "core_api_cpu": 1.0, + "core_api_mem": "2.0Gi", + "orchestration_api_replica_count": 0, + "orchestration_api_cpu": 1.0, + "orchestration_api_mem": "2.0Gi", + "langchain_api_replica_count": 0, + "langchain_api_cpu": 2.0, + "langchain_api_mem": "4.0Gi", + "semantickernel_api_replica_count": 0, + "semantickernel_api_cpu": 1.0, + "semantickernel_api_mem": "2.0Gi" + } } \ No newline at end of file From 1bc7ac5cb49d1981020113bce1ff9347fba74609 Mon Sep 17 00:00:00 2001 From: saimachi Date: Wed, 26 Jun 2024 20:22:30 -0400 Subject: [PATCH 4/7] Fix invalid JSON for parameterization. --- .github/workflows/load-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index cdc9d6f0f7..2255d11f62 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -55,7 +55,7 @@ jobs: test_matrix: ${{ steps.set-test-matrix.outputs.result }} docker_tag: ${{ steps.docker-tag.outputs.docker_tag }} unique_id: ${{ steps.set-unique-id.outputs.digest }} - qs_resources: ${{ steps.qs-resources.outputs.digest }} + qs_resources: ${{ steps.qs-resources.outputs.result }} steps: - name: Checkout code uses: actions/checkout@v4 From c7e3bfe06ed15f7150e237a62fb6018d730e23e1 Mon Sep 17 00:00:00 2001 From: saimachi Date: Wed, 26 Jun 2024 20:55:59 -0400 Subject: [PATCH 5/7] Do not deploy new OpenAI resource. --- .github/workflows/load-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index 2255d11f62..d31203604e 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -273,7 +273,7 @@ jobs: azd env set PIPELINE_DEPLOY "$true" azd env set FLLM_E2E_SP_OBJECT_ID $spObjectId - if (${{ inputs.deployOpenAi }}) { + if ('${{ inputs.deployOpenAi }}' -eq 'false') { azd env set OPENAI_NAME ${{ inputs.openAiName }} azd env set OPENAI_RESOURCE_GROUP ${{ inputs.openAiResourceGroup }} azd env set OPENAI_SUBSCRIPTION_ID $($info.subscriptionId) From 0fbd2072230ffeb269f5b1578b5c04187eae20de Mon Sep 17 00:00:00 2001 From: saimachi Date: Wed, 26 Jun 2024 21:42:53 -0400 Subject: [PATCH 6/7] Moved environment variable refresh step. --- .github/workflows/load-testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index d31203604e..e095d2f96d 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -771,12 +771,13 @@ jobs: -principalType ServicePrincipal ` -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 + azd env refresh --no-prompt + Write-Host "Removing App Registration OAuth Callbacks" pwsh -File ../../tests/scripts/Remove-OAuthCallbackUris.ps1 ` -fllmChatUiName FoundationaLLM-Core-Portal ` -fllmMgmtUiName FoundationaLLM-Management-Portal - azd env refresh --no-prompt azd down --force --purge --no-prompt Pop-Location From 4599c8957bcffc5568ad91a84dcd4e0b9db03bfa Mon Sep 17 00:00:00 2001 From: saimachi Date: Mon, 22 Jul 2024 17:59:02 -0400 Subject: [PATCH 7/7] Update Entra script paths. --- .github/workflows/load-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index e095d2f96d..b8173958c6 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -371,7 +371,7 @@ jobs: run: | Write-Host "Update App Registration OAuth Callbacks" Push-Location ./deploy/quick-start - pwsh -File ../../tests/scripts/Update-OAuthCallbackUris.ps1 ` + pwsh -File ../common/scripts/Update-OAuthCallbackUris.ps1 ` -fllmChatUiName FoundationaLLM-Core-Portal ` -fllmMgmtUiName FoundationaLLM-Management-Portal Pop-Location @@ -774,7 +774,7 @@ jobs: azd env refresh --no-prompt Write-Host "Removing App Registration OAuth Callbacks" - pwsh -File ../../tests/scripts/Remove-OAuthCallbackUris.ps1 ` + pwsh -File ../common/scripts/Remove-OAuthCallbackUris.ps1 ` -fllmChatUiName FoundationaLLM-Core-Portal ` -fllmMgmtUiName FoundationaLLM-Management-Portal