Load Testing #27
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Load Testing | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: AZD Environment Name | |
| type: string | |
| default: fllm-e2e-aca | |
| deployOpenAi: | |
| description: Deploy a new OpenAI Instance | |
| type: boolean | |
| default: false | |
| openAiName: | |
| description: Shared OpenAI Instance Name | |
| type: string | |
| default: fllm-01 | |
| openAiResourceGroup: | |
| description: Shared OpenAI Resource Group | |
| type: string | |
| default: fllm-shared-01 | |
| location: | |
| description: AZD Target Location | |
| type: choice | |
| options: | |
| - AustraliaEast | |
| - CanadaEast | |
| - EastUS | |
| - EastUS2 | |
| - SwedenCentral | |
| default: EastUS2 | |
| enableTeardown: | |
| description: Enable teardown of environment | |
| type: boolean | |
| default: true | |
| bypassAndTeardown: | |
| description: Bypass Tests and Force Teardown | |
| type: boolean | |
| default: false | |
| target: | |
| description: Select target tenant/subscription | |
| type: choice | |
| options: | |
| - sandbox | |
| - e2e | |
| default: e2e | |
| jobs: | |
| generate_matrix: | |
| name: Generate Configuration Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| registry: ${{ steps.set-registry.outputs.result }} | |
| service_matrix: ${{ steps.set-service-matrix.outputs.result }} | |
| test_matrix: ${{ steps.set-test-matrix.outputs.result }} | |
| docker_tag: ${{ steps.docker-tag.outputs.docker_tag }} | |
| unique_id: ${{ steps.set-unique-id.outputs.digest }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Generate Unique ID | |
| uses: pplanel/[email protected] | |
| id: set-unique-id | |
| with: | |
| input: ${{ inputs.environment }} | |
| method: SHA3 | |
| output_len: 32 | |
| - name: Set Service Matrix | |
| uses: actions/github-script@v7 | |
| id: set-service-matrix | |
| with: | |
| result-encoding: string | |
| script: | | |
| const fs = require('fs') | |
| const jsonString = fs.readFileSync('./tests/config/e2e-test-manifest.json') | |
| var data = JSON.parse(jsonString) | |
| return JSON.stringify(data.service_matrix) | |
| - name: Set Test Matrix | |
| uses: actions/github-script@v7 | |
| id: set-test-matrix | |
| with: | |
| result-encoding: string | |
| script: | | |
| const fs = require('fs') | |
| const jsonString = fs.readFileSync('./tests/config/e2e-test-manifest.json') | |
| var data = JSON.parse(jsonString) | |
| return JSON.stringify(data.test_matrix) | |
| - name: Set Container registry | |
| uses: actions/github-script@v7 | |
| id: set-registry | |
| with: | |
| result-encoding: string | |
| script: | | |
| var registries = new Map(); | |
| registries['e2e'] = 'crfllme2eops' | |
| registries['sandbox'] = 'cropseastus2svinternal' | |
| return registries['${{ inputs.target }}'] | |
| - name: Set Git and Helm Tag | |
| id: docker-tag | |
| run: | | |
| if [ '${{ (startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/poc/')) }}' == 'true' ]; then | |
| DOCKER_TAG=$(echo ${{ github.ref_name }} | cut -d'/' -f 2) | |
| else | |
| DOCKER_TAG=0.0.0-${GITHUB_REF#refs/heads/} | |
| fi | |
| DOCKER_TAG=$DOCKER_TAG-$(date +'%Y%m%dT%H%M%SZ') | |
| echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV | |
| echo "docker_tag: Latest Tag is : $DOCKER_TAG" | |
| echo "docker_tag=$DOCKER_TAG" >> $GITHUB_OUTPUT | |
| helm_build: | |
| name: Package Helm Charts | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| needs: generate_matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| object: ${{ fromJson(needs.generate_matrix.outputs.service_matrix) }} | |
| uses: ./.github/workflows/build-helm-chart.yml | |
| with: | |
| chart: ${{ matrix.object.name }} | |
| registry: ${{ needs.generate_matrix.outputs.registry }}.azurecr.io | |
| tag: ${{ needs.generate_matrix.outputs.docker_tag }} | |
| username: e2e-token | |
| target: ${{ inputs.target }} | |
| secrets: inherit | |
| docker_build: | |
| name: Build Docker Images | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| needs: generate_matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| object: ${{ fromJson(needs.generate_matrix.outputs.service_matrix) }} | |
| uses: ./.github/workflows/build-docker-image.yml | |
| with: | |
| image: ${{ matrix.object.name }} | |
| context: ${{ matrix.object.context }} | |
| dockerfile: ${{ matrix.object.dockerfile }} | |
| registry: ${{ needs.generate_matrix.outputs.registry }}.azurecr.io | |
| tag: ${{ needs.generate_matrix.outputs.docker_tag }} | |
| username: e2e-token | |
| target: ${{ inputs.target }} | |
| secrets: inherit | |
| deploy_quickstart: | |
| name: Deploy Quick Start | |
| runs-on: ubuntu-latest | |
| needs: [generate_matrix, docker_build] | |
| outputs: | |
| azd-deployment-data: ${{ steps.azd-deployment-data.outputs.AZD_DEPLOYMENT_DATA }} | |
| env: | |
| AZURE_ENV_NAME: ${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| steps: | |
| - name: Sets Credentials for Sandbox | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| with: | |
| version: "1.9.3" | |
| - name: Log in with Azure (Client Credentials) | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| cd ./deploy/quick-start | |
| azd auth login ` | |
| --client-id "$($info.clientId)" ` | |
| --client-secret "$($info.clientSecret)" ` | |
| --tenant-id "$($info.tenantId)" | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" | |
| $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)" | |
| azcopy login ` | |
| --service-principal ` | |
| --application-id "$($info.clientId)" ` | |
| --tenant-id "$($info.tenantId)" | |
| shell: pwsh | |
| - name: Provision App Registrations | |
| id: provision-app-registrations | |
| run: | | |
| Write-Host "Provision App Registrations" | |
| Push-Location ./tests/scripts | |
| ./Create-FllmEntraIdApps.ps1 ` | |
| -authAppName FoundationaLLM-Authorization-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -coreAppName FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -coreClientAppName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -mgmtAppName FoundationaLLM-Management-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -mgmtClientAppName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} | |
| Pop-Location | |
| shell: pwsh | |
| - name: Provision Admin Group | |
| id: provision-admin-group | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Write-Host "Provision Admin Group and add ObjectId to AZD Environment" | |
| Push-Location ./tests/scripts | |
| $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv) | |
| Write-Host "SP Object Id: $spObjectId" | |
| ./Create-FllmAdminGroup.ps1 ` | |
| -groupName FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} | |
| $spAssignmentExists = $(az ad group member check ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id $spObjectId ` | |
| --output json | ConvertFrom-Json).value | |
| if (-not $spAssignmentExists) | |
| { | |
| az ad group member add ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id $spObjectId ` | |
| --output json | |
| } | |
| if ('${{ inputs.target }}' -eq 'sandbox') { | |
| $assignmentExists = $(az ad group member check ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id d3bd4e8e-d413-477d-a420-0792b0504adf ` | |
| --output json | ConvertFrom-Json).value | |
| if (-not $assignmentExists) | |
| { | |
| az ad group member add ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id d3bd4e8e-d413-477d-a420-0792b0504adf ` | |
| --output json | |
| } | |
| } | |
| Pop-Location | |
| shell: pwsh | |
| - name: Update AZD Environment | |
| id: update-azd-environment | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv) | |
| Write-Host "Create new AZD Environment" | |
| Push-Location ./deploy/quick-start | |
| azd env new ${{ inputs.environment }} | |
| azd env set PIPELINE_DEPLOY "$true" | |
| azd env set FLLM_E2E_SP_OBJECT_ID $spObjectId | |
| if (${{ inputs.deployOpenAi }}) { | |
| azd env set OPENAI_NAME ${{ inputs.openAiName }} | |
| azd env set OPENAI_RESOURCE_GROUP ${{ inputs.openAiResourceGroup }} | |
| azd env set OPENAI_SUBSCRIPTION_ID $($info.subscriptionId) | |
| } | |
| Pop-Location | |
| Write-Host "Update AZD Environment with Service Docker Image Names" | |
| $configPath = "./tests/config/e2e-test-manifest.json" | |
| $config = Get-content $configPath | ConvertFrom-Json | |
| Push-Location ./deploy/quick-start | |
| foreach ($service in $($config.service_matrix)) { | |
| $imageName = "${{ needs.generate_matrix.outputs.registry }}.azurecr.io/$($service.name):${{ needs.generate_matrix.outputs.docker_tag }}" | |
| Write-Host "Setting $($service.azd_env_key) to $($imageName)..." | |
| azd env set $($service.azd_env_key) "$($imageName)" | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Error("Failed to set $($value.Key).") | |
| exit 1 | |
| } | |
| } | |
| Pop-Location | |
| Write-Host "Update AZD Environment with App Registration Metadata" | |
| Push-Location ./deploy/quick-start | |
| ../../tests/scripts/Set-AzdEnv.ps1 ` | |
| -tenantId $($info.tenantId) ` | |
| -principalType ServicePrincipal ` | |
| -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 ` | |
| -fllmApiName FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -fllmClientName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -fllmMgmtApiName FoundationaLLM-Management-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -fllmMgmtClientName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -fllmAuthApiName FoundationaLLM-Authorization-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -fllmAdminGroupName FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} | |
| Pop-Location | |
| shell: pwsh | |
| - name: Provision Infrastructure | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| $env:AZURE_SUBSCRIPTION_ID=$($info.subscriptionId) | |
| Push-Location ./deploy/quick-start | |
| azd provision --no-prompt | |
| Pop-Location | |
| shell: pwsh | |
| - name: Deploy Application | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| $env:AZURE_SUBSCRIPTION_ID=$($info.subscriptionId) | |
| Push-Location ./deploy/quick-start | |
| azd deploy --no-prompt | |
| Pop-Location | |
| shell: pwsh | |
| - name: Update App Registration OAuth Callbacks | |
| id: update-app-reg-oauth | |
| run: | | |
| Write-Host "Update App Registration OAuth Callbacks" | |
| Push-Location ./deploy/quick-start | |
| pwsh -File ../../tests/scripts/Update-OAuthCallbackUris.ps1 ` | |
| -fllmChatUiName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -fllmMgmtUiName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} | |
| Pop-Location | |
| shell: pwsh | |
| - name: Apply MS Graph API Role Assignments | |
| id: apply-ms-graph-api-roles | |
| run: | | |
| Write-Host "Apply MS Graph API Role Assignments" | |
| Push-Location ./deploy/quick-start | |
| pwsh -File ../../tests/scripts/Apply-MSGraphAPIRoles.ps1 | |
| Pop-Location | |
| shell: pwsh | |
| - name: Store AZD Deployment Information | |
| id: azd-deployment-data | |
| run: | | |
| cd ./deploy/quick-start | |
| echo 'AZD_DEPLOYMENT_DATA<<EOF' >> $GITHUB_OUTPUT | |
| cat './.azure/${{ inputs.environment }}/.env' >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| - name: Apply Cosmos DB RBAC to Service Principal | |
| id: apply-cosmos-db-rbac | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| (azd env get-values) | foreach { | |
| $name, $value = $_.split('=') | |
| set-content env:\$name $value | |
| } | |
| Pop-Location | |
| Write-Host "Apply Cosmos DB Role to Service Principal" | |
| Push-Location ./tests/scripts | |
| $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv) | |
| Write-Host "SP Object Id: $spObjectId" | |
| $cosmosDbName = $env:AZURE_COSMOS_DB_NAME | ConvertFrom-Json | |
| $cosmosDbObjectId = $(az cosmosdb show --name $cosmosDbName --resource-group rg-${{ inputs.environment }} --query 'id' --output tsv) | |
| Write-Host "CosmosDB Object Id: $cosmosDbObjectId" | |
| az cosmosdb sql role assignment create ` | |
| --account-name $cosmosDbName ` | |
| --resource-group rg-${{ inputs.environment }} ` | |
| --role-definition-id 00000000-0000-0000-0000-000000000002 ` | |
| --scope $cosmosDbObjectId ` | |
| --principal-id $spObjectId | |
| Pop-Location | |
| shell: pwsh | |
| add_test_data: | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| name: Add Test Data | |
| needs: [deploy_quickstart] | |
| runs-on: ubuntu-latest # this may need to change to windows | |
| env: | |
| AZURE_ENV_NAME: ${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| steps: | |
| - name: Sets Credentials for Source (SBX) | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS_SRC<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Target (E2E) | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_E2E_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'e2e' | |
| env: | |
| FLLM_E2E_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Target (Sandbox) | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'sandbox' | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| with: | |
| version: "1.9.3" | |
| - name: Log in with Azure (Client Credentials) | |
| shell: pwsh | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| azd auth login ` | |
| --client-id "$($info.clientId)" ` | |
| --client-secret "$($info.clientSecret)" ` | |
| --tenant-id "$($info.tenantId)" | |
| Pop-Location | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" ` | |
| - name: Copy Test Data | |
| shell: pwsh | |
| run: | | |
| $src = $Env:AZURE_CREDENTIALS_SRC | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($src.clientSecret)" | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Write-Host "Create new AZD Environment" | |
| Push-Location ./deploy/quick-start | |
| azd env new ${{ inputs.environment }} | |
| azd env set AZURE_SUBSCRIPTION_ID $($info.subscriptionId) | |
| azd env set PIPELINE_DEPLOY "$true" | |
| Pop-Location | |
| Push-Location ./deploy/quick-start | |
| ../../tests/scripts/Set-AzdEnv.ps1 ` | |
| -tenantId $($info.tenantId) ` | |
| -principalType ServicePrincipal ` | |
| -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 | |
| azd env refresh --no-prompt | |
| (azd env get-values) | foreach { | |
| $name, $value = $_.split('=') | |
| $value = $value.Trim('"') | |
| set-content env:\$name $value | |
| } | |
| Pop-Location | |
| # Assign “Search Index Data Contributor” on the target resource group | |
| $objectId = (az ad sp list --all --query "[?appId=='$($info.clientId)'].id" --output tsv) | |
| az role assignment create ` | |
| --role "Search Index Data Contributor" ` | |
| --assignee-object-id $objectId ` | |
| --assignee-principal-type ServicePrincipal ` | |
| --scope "/subscriptions/${env:AZURE_SUBSCRIPTION_ID}/resourceGroups/${env:RESOURCE_GROUP_NAME_DEFAULT}" | |
| $env:AZCOPY_SPA_CLIENT_SECRET="$($src.clientSecret)" | |
| azcopy login ` | |
| --login-type=SPN ` | |
| --application-id "$($src.clientId)" ` | |
| --tenant-id "$($src.tenantId)" | |
| ./deploy/common/scripts/Get-TestData.ps1 -storageAccountName foundationallmdata | |
| $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)" | |
| azcopy login ` | |
| --login-type=SPN ` | |
| --application-id "$($info.clientId)" ` | |
| --tenant-id "$($info.tenantId)" | |
| ./deploy/common/scripts/Set-TestData.ps1 ` | |
| -storageAccountName $env:AZURE_STORAGE_ACCOUNT_NAME ` | |
| -aiSearchName $env:AZURE_COGNITIVE_SEARCH_NAME | |
| run_load_test: | |
| name: Run Load Test | |
| runs-on: ubuntu-latest | |
| needs: [deploy_quickstart, generate_matrix, add_test_data] | |
| steps: | |
| - name: Sets Credentials for Sandbox | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Load Config File | |
| uses: actions/github-script@v7 | |
| id: set-config | |
| with: | |
| result-encoding: string | |
| script: | | |
| const fs = require('fs') | |
| return fs.readFileSync('./tests/config/load-testing-parameters.json') | |
| - name: Log in with Azure (Client Credentials) | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" | |
| shell: pwsh | |
| - name: Download Question Bank & Load Testing File | |
| run: | | |
| az storage blob download ` | |
| --account-name ${{ fromJson(steps.set-config.outputs.result).storage_account_name }} ` | |
| --container-name ${{ fromJson(steps.set-config.outputs.result).storage_account_container }} ` | |
| --name ${{ fromJson(steps.set-config.outputs.result).question_bank_file }} ` | |
| --file QuestionBank.csv ` | |
| --auth-mode login | |
| az storage blob download ` | |
| --account-name ${{ fromJson(steps.set-config.outputs.result).storage_account_name }} ` | |
| --container-name ${{ fromJson(steps.set-config.outputs.result).storage_account_container }} ` | |
| --name ${{ fromJson(steps.set-config.outputs.result).jmeter_file }} ` | |
| --file fllm-test-plan.jmx ` | |
| --auth-mode login | |
| shell: pwsh | |
| - name: Populate Load Testing Config | |
| id: populate-load-testing-config | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| $load_test_name = "FLLM-Load-Test-$([int64](Get-Date -UFormat %s))" | |
| $load_test_file_name = "Load-Test-Questions.csv" | |
| $load_test_id = (New-Guid).Guid | |
| # Load .env | |
| $configurationParameters = @{} | |
| echo @" | |
| ${{ needs.deploy_quickstart.outputs.azd-deployment-data }} | |
| "@ >> .env | |
| $fileData = Get-Content ".env" -Raw | |
| $fileData.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) | ForEach-Object { | |
| $envVar = $_ -split "=" | |
| Write-Host $envVar | |
| $configurationParameters[$envVar[0].Trim()] = $envVar[1].Replace('"', '') | |
| } | |
| echo "load_test_name=$load_test_name" >> $env:GITHUB_ENV | |
| echo "load_test_id=$load_test_id" >> $env:GITHUB_ENV | |
| echo "AZURE_APP_CONFIG_NAME=$($configurationParameters['AZURE_APP_CONFIG_NAME'])" >> $env:GITHUB_ENV | |
| echo "AZURE_ENV_NAME=$($configurationParameters['AZURE_ENV_NAME'])" >> $env:GITHUB_ENV | |
| $token = (az account get-access-token --scope "api://FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }}/.default" --query accessToken | ConvertFrom-Json) | |
| $kvUrl = (az keyvault secret set --vault-name "fllme2emkeyvault7f393d64" --name "bearer-token-${{ needs.generate_matrix.outputs.unique_id }}" --value $token --query '{id: id}' | ConvertFrom-Json).id | |
| # Replace Placeholders | |
| $config = @{ | |
| load_test_name = $load_test_name | |
| load_test_id = $load_test_id | |
| load_test_file_name = $load_test_file_name | |
| token = $kvUrl | |
| core_api_endpoint = $configurationParameters["SERVICE_CORE_API_ENDPOINT_URL"] | |
| } | |
| $load_testing_config = Get-Content "./tests/config/load-testing-template.yml" -Raw | |
| foreach ($h in $config.GetEnumerator()) | |
| { | |
| $load_testing_config = $load_testing_config.replace("`${$($h.Name)}", $h.Value) | |
| } | |
| Out-File -FilePath "load-testing.yml" -InputObject $load_testing_config | |
| # Randomly Sample 100 Lines for Testing | |
| $file_content = Get-Content "QuestionBank.csv" | |
| $lines = New-Object Collections.Generic.List[String] | |
| 1..100 | %{ | |
| $line = Get-Random -InputObject $file_content | |
| while(1) { | |
| # It's possible for the returned line to be empty | |
| if ([string]::IsNullOrEmpty($line)) { | |
| $line = Get-Random -InputObject $file_content | |
| } | |
| else { | |
| break | |
| } | |
| } | |
| $lines.Add($line) | |
| } | |
| Out-File -FilePath $load_test_file_name -InputObject $lines | |
| shell: pwsh | |
| - name: Run Load Test | |
| run: | | |
| # Create Agents | |
| $connectionString = $(az appconfig credential list ` | |
| --name $env:AZURE_APP_CONFIG_NAME ` | |
| --resource-group rg-$($env:AZURE_ENV_NAME) ` | |
| --query "[].{name:name,connectionString:connectionString}[?name=='Primary Read Only'].connectionString" ` | |
| --output tsv) | |
| $env:FoundationaLLM_AppConfig_ConnectionString=$connectionString | |
| dotnet test --filter "FullyQualifiedName=FoundationaLLM.Core.Examples.CreateAgentsForLoadTest.RunAsync" ./tests/dotnet/Core.Examples/Core.Examples.csproj | |
| az load test create ` | |
| --load-test-resource ${{ fromJson(steps.set-config.outputs.result).load_test_resource }} ` | |
| --test-id $env:load_test_id ` | |
| --load-test-config-file "load-testing.yml" ` | |
| -g ${{ fromJson(steps.set-config.outputs.result).resource_group_name }} | |
| az load test-run create ` | |
| --load-test-resource ${{ fromJson(steps.set-config.outputs.result).load_test_resource }} ` | |
| --test-id $env:load_test_id ` | |
| --test-run-id $(New-Guid).Guid ` | |
| --no-wait ` | |
| -g ${{ fromJson(steps.set-config.outputs.result).resource_group_name }} | |
| shell: pwsh | |
| teardown_quickstart: | |
| name: Tear Down Quick Start | |
| if: ${{ (cancelled() && inputs.bypassAndTeardown) || inputs.enableTeardown }} | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_ENV_NAME: ${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| needs: [generate_matrix, run_load_test] | |
| steps: | |
| - name: Sets Credentials for E2E | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_E2E_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'e2e' | |
| env: | |
| FLLM_E2E_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Sandbox | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'sandbox' | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| with: | |
| version: "1.9.3" | |
| - name: Log in with Azure (Client Credentials) | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| cd ./deploy/quick-start | |
| azd auth login ` | |
| --client-id "$($info.clientId)" ` | |
| --client-secret "$($info.clientSecret)" ` | |
| --tenant-id "$($info.tenantId)" | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" | |
| shell: pwsh | |
| - name: Tearing Down AZD Deployment | |
| id: azd-down | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Write-Host "Create new AZD Environment" | |
| Push-Location ./deploy/quick-start | |
| azd env new ${{ inputs.environment }} | |
| azd env set AZURE_SUBSCRIPTION_ID $($info.subscriptionId) | |
| azd env set PIPELINE_DEPLOY "$true" | |
| Pop-Location | |
| Push-Location ./deploy/quick-start | |
| ../../tests/scripts/Set-AzdEnv.ps1 ` | |
| -tenantId $($info.tenantId) ` | |
| -principalType ServicePrincipal ` | |
| -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 | |
| azd env refresh --no-prompt | |
| azd down --force --purge --no-prompt | |
| Pop-Location | |
| Write-Host "Remove Admin Group" | |
| az ad group delete --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} --output json | |
| Write-Host "Tear down App Registrations" | |
| Push-Location ./tests/scripts | |
| ./Remove-EntraIdApps.ps1 ` | |
| -authAppName FoundationaLLM-Authorization-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -coreAppName FoundationaLLM-Core-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -coreClientAppName FoundationaLLM-Core-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -mgmtAppName FoundationaLLM-Management-E2E-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -mgmtClientAppName FoundationaLLM-Management-E2E-Client-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| -interactiveMode $false | |
| Pop-Location | |
| shell: pwsh |