End to End Testing #388
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End to End Testing | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: AZD Environment Name | |
| type: string | |
| default: fllm-e2e-aca | |
| deployOpenAi: | |
| description: Deploy a new OpenAI Instance | |
| type: boolean | |
| default: false | |
| openAiName: | |
| description: Shared OpenAI Instance Name | |
| type: string | |
| default: fllm-01 | |
| openAiResourceGroup: | |
| description: Shared OpenAI Resource Group | |
| type: string | |
| default: fllm-shared-01 | |
| location: | |
| description: AZD Target Location | |
| type: choice | |
| options: | |
| - AustraliaEast | |
| - CanadaEast | |
| - EastUS | |
| - EastUS2 | |
| - SwedenCentral | |
| default: EastUS2 | |
| notificationsEnabled: | |
| description: Enable Slack Notifications | |
| type: boolean | |
| default: false | |
| enableTeardown: | |
| description: Enable teardown of environment | |
| type: boolean | |
| default: true | |
| bypassAndTeardown: | |
| description: Bypass Tests and Force Teardown | |
| type: boolean | |
| default: false | |
| target: | |
| description: Select target tenant/subscription | |
| type: choice | |
| options: | |
| - sandbox | |
| - e2e | |
| default: e2e | |
| deployAMLResources: | |
| description: Deploy Azure Machine Learning resources & run tests that depend on them | |
| type: boolean | |
| default: false | |
| workflow_call: | |
| inputs: | |
| environment: | |
| description: AZD Environment Name | |
| type: string | |
| deployOpenAi: | |
| description: Deploy a new OpenAI Instance | |
| type: boolean | |
| openAiName: | |
| description: Shared OpenAI Instance Name | |
| type: string | |
| openAiResourceGroup: | |
| description: Shared OpenAI Resource Group | |
| type: string | |
| location: | |
| description: AZD Target Location | |
| type: string | |
| notificationsEnabled: | |
| description: Enable Slack Notifications | |
| type: boolean | |
| enableTeardown: | |
| description: Enable teardown of environment | |
| type: boolean | |
| bypassAndTeardown: | |
| description: Bypass Tests and Force Teardown | |
| type: boolean | |
| target: | |
| description: Select target tenant/subscription | |
| type: string | |
| deployAMLResources: | |
| description: Deploy Azure Machine Learning resources & run tests that depend on them | |
| type: boolean | |
| jobs: | |
| generate_matrix: | |
| name: Generate Configuration Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| registry: ${{ steps.set-registry.outputs.result }} | |
| service_matrix: ${{ steps.set-service-matrix.outputs.result }} | |
| test_matrix: ${{ steps.set-test-matrix.outputs.result }} | |
| docker_tag: ${{ steps.docker-tag.outputs.docker_tag }} | |
| unique_id: ${{ steps.set-unique-id.outputs.digest }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Generate Unique ID | |
| uses: pplanel/[email protected] | |
| id: set-unique-id | |
| with: | |
| input: ${{ inputs.environment }} | |
| method: SHA3 | |
| output_len: 32 | |
| - name: Set Service Matrix | |
| uses: actions/github-script@v7 | |
| id: set-service-matrix | |
| with: | |
| result-encoding: string | |
| script: | | |
| const fs = require('fs') | |
| const jsonString = fs.readFileSync('./tests/config/e2e-test-manifest.json') | |
| var data = JSON.parse(jsonString) | |
| return JSON.stringify(data.service_matrix) | |
| - name: Set Test Matrix | |
| uses: actions/github-script@v7 | |
| id: set-test-matrix | |
| with: | |
| result-encoding: string | |
| script: | | |
| const fs = require('fs') | |
| const jsonString = fs.readFileSync('./tests/config/e2e-test-manifest.json') | |
| var data = JSON.parse(jsonString) | |
| return JSON.stringify(data.test_matrix) | |
| - name: Set Container registry | |
| uses: actions/github-script@v7 | |
| id: set-registry | |
| with: | |
| result-encoding: string | |
| script: | | |
| var registries = new Map(); | |
| registries['e2e'] = 'crfllme2eops' | |
| registries['sandbox'] = 'cropseastus2svinternal' | |
| return registries['${{ inputs.target }}'] | |
| - name: Set Git and Helm Tag | |
| id: docker-tag | |
| run: | | |
| if [ '${{ (startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/poc/')) }}' == 'true' ]; then | |
| DOCKER_TAG=$(echo ${{ github.ref_name }} | cut -d'/' -f 2) | |
| else | |
| DOCKER_TAG=0.0.0-${GITHUB_REF#refs/heads/} | |
| fi | |
| DOCKER_TAG=$DOCKER_TAG-$(date +'%Y%m%dT%H%M%SZ') | |
| echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV | |
| echo "docker_tag: Latest Tag is : $DOCKER_TAG" | |
| echo "docker_tag=$DOCKER_TAG" >> $GITHUB_OUTPUT | |
| helm_build: | |
| name: Package Helm Charts | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| needs: generate_matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| object: ${{ fromJson(needs.generate_matrix.outputs.service_matrix) }} | |
| uses: ./.github/workflows/build-helm-chart.yml | |
| with: | |
| chart: ${{ matrix.object.name }} | |
| registry: ${{ needs.generate_matrix.outputs.registry }}.azurecr.io | |
| tag: ${{ needs.generate_matrix.outputs.docker_tag }} | |
| username: e2e-token | |
| target: ${{ inputs.target }} | |
| secrets: inherit | |
| docker_build: | |
| name: Build Docker Images | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| needs: generate_matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| object: ${{ fromJson(needs.generate_matrix.outputs.service_matrix) }} | |
| uses: ./.github/workflows/build-docker-image.yml | |
| with: | |
| image: ${{ matrix.object.name }} | |
| context: ${{ matrix.object.context }} | |
| dockerfile: ${{ matrix.object.dockerfile }} | |
| registry: ${{ needs.generate_matrix.outputs.registry }}.azurecr.io | |
| tag: ${{ needs.generate_matrix.outputs.docker_tag }} | |
| username: e2e-token | |
| target: ${{ inputs.target }} | |
| secrets: inherit | |
| deploy_quickstart: | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| name: Deploy Quick Start | |
| needs: [generate_matrix, docker_build] | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_ENV_NAME: ${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| steps: | |
| - name: Sets Credentials for E2E | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_E2E_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'e2e' | |
| env: | |
| FLLM_E2E_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Sandbox | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'sandbox' | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| with: | |
| version: "1.9.3" | |
| - name: Log in with Azure (Client Credentials) | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| cd ./deploy/quick-start | |
| azd auth login ` | |
| --client-id "$($info.clientId)" ` | |
| --client-secret "$($info.clientSecret)" ` | |
| --tenant-id "$($info.tenantId)" | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" | |
| $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)" | |
| azcopy login ` | |
| --login-type=SPN ` | |
| --application-id "$($info.clientId)" ` | |
| --tenant-id "$($info.tenantId)" | |
| shell: pwsh | |
| - name: Deploy Azure Machine Learning | |
| if: ${{ inputs.deployAMLResources }} | |
| env: | |
| AZURE_ENV_NAME: ext-${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Push-Location ./tests/extended-test-env | |
| azd env new ext-${{ inputs.environment }} | |
| azd env set AZURE_SUBSCRIPTION_ID $($info.subscriptionId) | |
| azd up --no-prompt | |
| Pop-Location | |
| shell: pwsh | |
| - name: Provision Admin Group | |
| id: provision-admin-group | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Write-Host "Provision Admin Group and add ObjectId to AZD Environment" | |
| Push-Location ./tests/scripts | |
| $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv) | |
| Write-Host "SP Object Id: $spObjectId" | |
| ./Create-FllmAdminGroup.ps1 ` | |
| -groupName FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} | |
| $spAssignmentExists = $(az ad group member check ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id $spObjectId ` | |
| --output json | ConvertFrom-Json).value | |
| if (-not $spAssignmentExists) | |
| { | |
| az ad group member add ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id $spObjectId ` | |
| --output json | |
| } | |
| if ('${{ inputs.target }}' -eq 'sandbox') { | |
| $assignmentExists = $(az ad group member check ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id d3bd4e8e-d413-477d-a420-0792b0504adf ` | |
| --output json | ConvertFrom-Json).value | |
| if (-not $assignmentExists) | |
| { | |
| az ad group member add ` | |
| --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} ` | |
| --member-id d3bd4e8e-d413-477d-a420-0792b0504adf ` | |
| --output json | |
| } | |
| } | |
| Pop-Location | |
| shell: pwsh | |
| - name: Update AZD Environment | |
| id: update-azd-environment | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv) | |
| Write-Host "Create new AZD Environment" | |
| Push-Location ./deploy/quick-start | |
| azd env new ${{ inputs.environment }} | |
| azd env set PIPELINE_DEPLOY "$true" | |
| azd env set FLLM_E2E_SP_OBJECT_ID $spObjectId | |
| if ('${{ inputs.deployOpenAi }}' -eq 'false') { | |
| azd env set OPENAI_NAME ${{ inputs.openAiName }} | |
| azd env set OPENAI_RESOURCE_GROUP ${{ inputs.openAiResourceGroup }} | |
| azd env set OPENAI_SUBSCRIPTION_ID $($info.subscriptionId) | |
| } | |
| Pop-Location | |
| Write-Host "Update AZD Environment with Service Docker Image Names" | |
| $configPath = "./tests/config/e2e-test-manifest.json" | |
| $config = Get-content $configPath | ConvertFrom-Json | |
| Push-Location ./deploy/quick-start | |
| foreach ($service in $($config.service_matrix)) { | |
| $imageName = "${{ needs.generate_matrix.outputs.registry }}.azurecr.io/$($service.name):${{ needs.generate_matrix.outputs.docker_tag }}" | |
| Write-Host "Setting $($service.azd_env_key) to $($imageName)..." | |
| azd env set $($service.azd_env_key) "$($imageName)" | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Error("Failed to set $($value.Key).") | |
| exit 1 | |
| } | |
| } | |
| Pop-Location | |
| Write-Host "Update AZD Environment with App Registration Metadata" | |
| Push-Location ./deploy/quick-start | |
| ../../tests/scripts/Set-AzdEnv.ps1 ` | |
| -tenantId $($info.tenantId) ` | |
| -principalType ServicePrincipal ` | |
| -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 ` | |
| -fllmApiName FoundationaLLM-Core-API ` | |
| -fllmClientName FoundationaLLM-Core-Portal ` | |
| -fllmMgmtApiName FoundationaLLM-Management-API ` | |
| -fllmMgmtClientName FoundationaLLM-Management-Portal ` | |
| -fllmAuthApiName FoundationaLLM-Authorization-API ` | |
| -fllmAdminGroupName FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} | |
| Pop-Location | |
| shell: pwsh | |
| - name: Provision Infrastructure | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| $env:AZURE_SUBSCRIPTION_ID=$($info.subscriptionId) | |
| Push-Location ./deploy/quick-start | |
| azd provision --no-prompt | |
| Pop-Location | |
| shell: pwsh | |
| - name: Deploy Application | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| $env:AZURE_SUBSCRIPTION_ID=$($info.subscriptionId) | |
| Push-Location ./deploy/quick-start | |
| azd deploy --no-prompt | |
| Pop-Location | |
| shell: pwsh | |
| - name: Publish AZD Environment | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: azd-env | |
| path: ./deploy/quick-start/.azure | |
| - name: Update App Registration OAuth Callbacks | |
| id: update-app-reg-oauth | |
| run: | | |
| Write-Host "Update App Registration OAuth Callbacks" | |
| Push-Location ./deploy/quick-start | |
| pwsh -File ../common/scripts/Update-OAuthCallbackUris.ps1 ` | |
| -fllmChatUiName FoundationaLLM-Core-Portal ` | |
| -fllmMgmtUiName FoundationaLLM-Management-Portal | |
| Pop-Location | |
| shell: pwsh | |
| - name: Apply MS Graph API Role Assignments | |
| id: apply-ms-graph-api-roles | |
| run: | | |
| Write-Host "Apply MS Graph API Role Assignments" | |
| Push-Location ./deploy/quick-start | |
| pwsh -File ../../tests/scripts/Apply-MSGraphAPIRoles.ps1 | |
| Pop-Location | |
| shell: pwsh | |
| - name: Apply Cosmos DB RBAC to Service Principal | |
| id: apply-cosmos-db-rbac | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| (azd env get-values) | foreach { | |
| $name, $value = $_.split('=') | |
| set-content env:\$name $value | |
| } | |
| Pop-Location | |
| Write-Host "Apply Cosmos DB Role to Service Principal" | |
| Push-Location ./tests/scripts | |
| $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv) | |
| Write-Host "SP Object Id: $spObjectId" | |
| $cosmosDbName = $env:AZURE_COSMOS_DB_NAME | ConvertFrom-Json | |
| $cosmosDbObjectId = $(az cosmosdb show --name $cosmosDbName --resource-group rg-${{ inputs.environment }} --query 'id' --output tsv) | |
| Write-Host "CosmosDB Object Id: $cosmosDbObjectId" | |
| az cosmosdb sql role assignment create ` | |
| --account-name $cosmosDbName ` | |
| --resource-group rg-${{ inputs.environment }} ` | |
| --role-definition-id 00000000-0000-0000-0000-000000000002 ` | |
| --scope $cosmosDbObjectId ` | |
| --principal-id $spObjectId | |
| Pop-Location | |
| shell: pwsh | |
| add_test_data: | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| name: Add Test Data | |
| needs: [deploy_quickstart] | |
| runs-on: ubuntu-latest # this may need to change to windows | |
| env: | |
| AZURE_ENV_NAME: ${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| steps: | |
| - name: Sets Credentials for Source (SBX) | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS_SRC<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Target (E2E) | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_E2E_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'e2e' | |
| env: | |
| FLLM_E2E_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Target (Sandbox) | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'sandbox' | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download AZD Environment | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: azd-env | |
| path: ./deploy/quick-start/.azure | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| with: | |
| version: "1.9.3" | |
| - name: Log in with Azure (Client Credentials) | |
| shell: pwsh | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| azd auth login ` | |
| --client-id "$($info.clientId)" ` | |
| --client-secret "$($info.clientSecret)" ` | |
| --tenant-id "$($info.tenantId)" | |
| Pop-Location | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" ` | |
| - name: Copy Test Data | |
| shell: pwsh | |
| run: | | |
| $src = $Env:AZURE_CREDENTIALS_SRC | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($src.clientSecret)" | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| (azd env get-values) | foreach { | |
| $name, $value = $_.split('=') | |
| $value = $value.Trim('"') | |
| set-content env:\$name $value | |
| } | |
| Pop-Location | |
| # Assign “Search Index Data Contributor” on the target resource group | |
| $objectId = (az ad sp list --all --query "[?appId=='$($info.clientId)'].id" --output tsv) | |
| az role assignment create ` | |
| --role "Search Index Data Contributor" ` | |
| --assignee-object-id $objectId ` | |
| --assignee-principal-type ServicePrincipal ` | |
| --scope "/subscriptions/${env:AZURE_SUBSCRIPTION_ID}/resourceGroups/${env:RESOURCE_GROUP_NAME_DEFAULT}" | |
| $env:AZCOPY_SPA_CLIENT_SECRET="$($src.clientSecret)" | |
| azcopy login ` | |
| --login-type=SPN ` | |
| --application-id "$($src.clientId)" ` | |
| --tenant-id "$($src.tenantId)" | |
| ./deploy/common/scripts/Get-TestData.ps1 -storageAccountName foundationallmdata | |
| $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)" | |
| azcopy login ` | |
| --login-type=SPN ` | |
| --application-id "$($info.clientId)" ` | |
| --tenant-id "$($info.tenantId)" | |
| ./deploy/common/scripts/Set-TestData.ps1 ` | |
| -storageAccountName $env:AZURE_STORAGE_ACCOUNT_NAME ` | |
| -aiSearchName $env:AZURE_COGNITIVE_SEARCH_NAME | |
| build_e2e_tests: | |
| name: Build End to End Tests | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v4 | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v3 | |
| with: | |
| dotnet-version: 8.x | |
| - name: Sets Credentials for Source (SBX) | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Pull Build Artifacts | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Push-Location ./deploy/common/scripts | |
| $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)" | |
| azcopy login ` | |
| --login-type=SPN ` | |
| --application-id "$($info.clientId)" ` | |
| --tenant-id "$($info.tenantId)" | |
| ./Get-TestSettings.ps1 -storageAccountName foundationallmdata | |
| Pop-Location | |
| shell: pwsh | |
| - name: Build Tests | |
| shell: pwsh | |
| run: | | |
| dotnet restore ./src/FoundationaLLM.sln --output ./dist | |
| dotnet build ./tests/dotnet/Common.Tests/Common.Tests.csproj --output ./dist | |
| dotnet build ./tests/dotnet/Core.Examples/Core.Examples.csproj --output ./dist | |
| dotnet build ./tests/dotnet/Core.Test/Core.Tests.csproj --output ./dist | |
| dotnet build ./tests/dotnet/Gatekeeper.Tests/Gatekeeper.Tests.csproj --output ./dist | |
| dotnet build ./tests/dotnet/Orchestration.Tests/Orchestration.Tests.csproj --output ./dist | |
| dotnet build ./tests/dotnet/SemanticKernel.Tests/SemanticKernel.Tests.csproj --output ./dist | |
| dotnet build ./tests/dotnet/Vectorization.Tests/Vectorization.Tests.csproj --output ./dist | |
| - name: Publish Build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build | |
| path: ./dist | |
| run_e2e_tests: | |
| name: Run End to End Tests | |
| if: ${{ !inputs.bypassAndTeardown }} | |
| continue-on-error: true | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_ENV_NAME: ${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| needs: [generate_matrix, deploy_quickstart, add_test_data, build_e2e_tests] | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| object: ${{ fromJson(needs.generate_matrix.outputs.test_matrix) }} | |
| steps: | |
| - name: Sets Credentials for E2E | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_E2E_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'e2e' | |
| env: | |
| FLLM_E2E_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Sandbox | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'sandbox' | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download Build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: build | |
| path: ./dist | |
| - name: Download AZD Environment | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: azd-env | |
| path: ./deploy/quick-start/.azure | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| with: | |
| version: "1.9.3" | |
| - name: Log in with Azure (Client Credentials) | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| azd auth login ` | |
| --client-id "$($info.clientId)" ` | |
| --client-secret "$($info.clientSecret)" ` | |
| --tenant-id "$($info.tenantId)" | |
| Pop-Location | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" ` | |
| --scope api://FoundationaLLM-Core/.default | |
| shell: pwsh | |
| - name: Running End to End Tests | |
| id: e2e-test-exec | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| (azd env get-values) | foreach { | |
| $name, $value = $_.split('=') | |
| set-content env:\$name $value | |
| } | |
| Pop-Location | |
| $connectionString = $(az appconfig credential list ` | |
| --name $($env:AZURE_APP_CONFIG_NAME | ConvertFrom-Json) ` | |
| --resource-group rg-$($env:AZURE_ENV_NAME) ` | |
| --query "[].{name:name,connectionString:connectionString}[?name=='Primary Read Only'].connectionString" ` | |
| --output tsv) | |
| $env:FoundationaLLM_AppConfig_ConnectionString=$connectionString | |
| echo "Running End to End Test: ${{ matrix.object.name }}" | |
| dotnet test ` | |
| --output ./dist ` | |
| --no-build ` | |
| --filter "FullyQualifiedName=${{ matrix.object.namespace }}.${{ matrix.object.task_name }}" ` | |
| --logger:"html;verbosity=detailed;LogFileName=${{ github.workspace }}/${{ matrix.object.name }}.html" ` | |
| ${{ matrix.object.target }} | |
| shell: pwsh | |
| - name: Publish Test Results | |
| if: (success() || failure()) | |
| id: publish-results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.object.name }}-results | |
| path: ${{ github.workspace }}/${{ matrix.object.name }}.html | |
| - name: Send Test Failure Notification | |
| if: (inputs.notificationsEnabled && failure()) | |
| uses: slackapi/[email protected] | |
| id: notify-failed-test | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.FLLM_SLACK_WEBHOOK_URL }} | |
| SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| with: | |
| payload: | | |
| { | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "*End to End Test Failure: ${{ matrix.object.name }}*" | |
| } | |
| }, | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "End to End Test ${{ matrix.object.name }} failed in branch `${{ github.head_ref || github.ref_name }}`" | |
| } | |
| }, | |
| { | |
| "type": "actions", | |
| "elements": [ | |
| { | |
| "type": "button", | |
| "text": { | |
| "type": "plain_text", | |
| "text": "Download Test Results" | |
| }, | |
| "url": "${{ steps.publish-results.outputs.artifact-url }}" | |
| } | |
| ] | |
| } | |
| ] | |
| } | |
| - name: Output Job Status | |
| if: (success() || failure()) | |
| uses: cloudposse/github-action-matrix-outputs-write@v1 | |
| id: job-status | |
| with: | |
| matrix-step-name: ${{ github.job }} | |
| matrix-key: ${{ matrix.object.name }} | |
| outputs: | | |
| passed: ${{ steps.e2e-test-exec.outcome }} | |
| teardown_quickstart: | |
| name: Tear Down Quick Start | |
| if: ${{ (cancelled() && inputs.bypassAndTeardown) || inputs.enableTeardown }} | |
| runs-on: ubuntu-latest | |
| env: | |
| AZURE_ENV_NAME: ${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| needs: [generate_matrix, run_e2e_tests] | |
| steps: | |
| - name: Sets Credentials for E2E | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_E2E_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'e2e' | |
| env: | |
| FLLM_E2E_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }} | |
| - name: Sets Credentials for Sandbox | |
| run: | | |
| { | |
| echo 'AZURE_CREDENTIALS<<EOF' | |
| echo "$FLLM_SBX_AZURE_CREDENTIALS" | |
| echo EOF | |
| } >> "$GITHUB_ENV" | |
| if: inputs.target == 'sandbox' | |
| env: | |
| FLLM_SBX_AZURE_CREDENTIALS: | | |
| ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }} | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Download AZD Environment | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: azd-env | |
| path: ./deploy/quick-start/.azure | |
| - name: Install azd | |
| uses: Azure/[email protected] | |
| with: | |
| version: "1.9.3" | |
| - name: Log in with Azure (Client Credentials) | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| cd ./deploy/quick-start | |
| azd auth login ` | |
| --client-id "$($info.clientId)" ` | |
| --client-secret "$($info.clientSecret)" ` | |
| --tenant-id "$($info.tenantId)" | |
| az login ` | |
| --service-principal ` | |
| --username "$($info.clientId)" ` | |
| --password "$($info.clientSecret)" ` | |
| --tenant "$($info.tenantId)" | |
| shell: pwsh | |
| - name: Tearing Down Azure Machine Learning | |
| if: ${{ inputs.deployAMLResources }} | |
| env: | |
| AZURE_ENV_NAME: ext-${{ inputs.environment }} | |
| AZURE_LOCATION: ${{ inputs.location }} | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Push-Location ./tests/extended-test-env | |
| azd env new ext-${{ inputs.environment }} | |
| $env:AZURE_SUBSCRIPTION_ID = $($info.subscriptionId) | |
| azd env refresh --no-prompt | |
| azd down --force --purge --no-prompt | |
| Pop-Location | |
| shell: pwsh | |
| - name: Tearing Down AZD Deployment | |
| id: azd-down | |
| run: | | |
| $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable; | |
| Write-Host "::add-mask::$($info.clientSecret)" | |
| Push-Location ./deploy/quick-start | |
| Write-Host "Removing App Registration OAuth Callbacks" | |
| pwsh -File ../common/scripts/Remove-OAuthCallbackUris.ps1 ` | |
| -fllmChatUiName FoundationaLLM-Core-Portal ` | |
| -fllmMgmtUiName FoundationaLLM-Management-Portal | |
| azd down --force --purge --no-prompt | |
| Pop-Location | |
| Write-Host "Remove Admin Group" | |
| az ad group delete --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} --output json | |
| shell: pwsh | |
| - name: Get E2E Test Results | |
| uses: cloudposse/github-action-matrix-outputs-read@v1 | |
| id: get-e2e-results | |
| with: | |
| matrix-step-name: run_e2e_tests | |
| - name: Check E2E Test Status | |
| uses: actions/github-script@v7 | |
| env: | |
| TEST_RESULTS: ${{ steps.get-e2e-results.outputs.result }} | |
| with: | |
| script: | | |
| const jsonString = process.env.TEST_RESULTS | |
| var data = JSON.parse(jsonString) | |
| var tests = data.passed | |
| for (const testName of Object.keys(tests)) { | |
| if (tests[testName] !== 'success') { | |
| process.exit(1); | |
| } | |
| } |