e2e-testing.yml

name: End to End Testing

on:
  workflow_dispatch:
    inputs:
      environment:
        description: AZD Environment Name
        type: string
        default: fllm-e2e-aca
      deployOpenAi:
        description: Deploy a new OpenAI Instance
        type: boolean
        default: false
      openAiName:
        description: Shared OpenAI Instance Name
        type: string
        default: fllm-01
      openAiResourceGroup:
        description: Shared OpenAI Resource Group
        type: string
        default: fllm-shared-01
      location:
        description: AZD Target Location
        type: choice
        options:
          - AustraliaEast
          - CanadaEast
          - EastUS
          - EastUS2
          - SwedenCentral
        default: EastUS2
      notificationsEnabled:
        description: Enable Slack Notifications
        type: boolean
        default: false
      enableTeardown:
        description: Enable teardown of environment
        type: boolean
        default: true
      bypassAndTeardown:
        description: Bypass Tests and Force Teardown
        type: boolean
        default: false
      target:
        description: Select target tenant/subscription
        type: choice
        options:
          - sandbox
          - e2e
        default: e2e
      deployAMLResources:
        description: Deploy Azure Machine Learning resources & run tests that depend on them
        type: boolean
        default: false
  workflow_call:
    inputs:
      environment:
        description: AZD Environment Name
        type: string
      deployOpenAi:
        description: Deploy a new OpenAI Instance
        type: boolean
      openAiName:
        description: Shared OpenAI Instance Name
        type: string
      openAiResourceGroup:
        description: Shared OpenAI Resource Group
        type: string
      location:
        description: AZD Target Location
        type: string
      notificationsEnabled:
        description: Enable Slack Notifications
        type: boolean
      enableTeardown:
        description: Enable teardown of environment
        type: boolean
      bypassAndTeardown:
        description: Bypass Tests and Force Teardown
        type: boolean
      target:
        description: Select target tenant/subscription
        type: string
      deployAMLResources:
        description: Deploy Azure Machine Learning resources & run tests that depend on them
        type: boolean

jobs:
  generate_matrix:
    name: Generate Configuration Matrix
    runs-on: ubuntu-latest
    outputs:
      registry: ${{ steps.set-registry.outputs.result }}
      service_matrix: ${{ steps.set-service-matrix.outputs.result }}
      test_matrix: ${{ steps.set-test-matrix.outputs.result }}
      docker_tag: ${{ steps.docker-tag.outputs.docker_tag }}
      unique_id: ${{ steps.set-unique-id.outputs.digest }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Generate Unique ID
        uses: pplanel/hash-calculator-action@v1.3.1
        id: set-unique-id
        with:
          input: ${{ inputs.environment }}
          method: SHA3
          output_len: 32

      - name: Set Service Matrix
        uses: actions/github-script@v7
        id: set-service-matrix
        with:
          result-encoding: string
          script: |
            const fs = require('fs')
            const jsonString = fs.readFileSync('./tests/config/e2e-test-manifest.json')
            var data = JSON.parse(jsonString)
            return JSON.stringify(data.service_matrix)

      - name: Set Test Matrix
        uses: actions/github-script@v7
        id: set-test-matrix
        with:
          result-encoding: string
          script: |
            const fs = require('fs')
            const jsonString = fs.readFileSync('./tests/config/e2e-test-manifest.json')
            var data = JSON.parse(jsonString)
            return JSON.stringify(data.test_matrix)

      - name: Set Container registry
        uses: actions/github-script@v7
        id: set-registry
        with:
          result-encoding: string
          script: |
            var registries = new Map();
            registries['e2e'] = 'crfllme2eops'
            registries['sandbox'] = 'cropseastus2svinternal'
            return registries['${{ inputs.target }}']

      - name: Set Git and Helm Tag
        id: docker-tag
        run: |
          if [ '${{ (startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/heads/poc/')) }}' == 'true' ]; then
            DOCKER_TAG=$(echo ${{ github.ref_name }} |  cut -d'/' -f 2)
          else
            DOCKER_TAG=0.0.0-${GITHUB_REF#refs/heads/}
          fi

          DOCKER_TAG=$DOCKER_TAG-$(date +'%Y%m%dT%H%M%SZ')
          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
          echo "docker_tag: Latest Tag is : $DOCKER_TAG"
          echo "docker_tag=$DOCKER_TAG" >> $GITHUB_OUTPUT

  helm_build:
    name: Package Helm Charts
    if: ${{ !inputs.bypassAndTeardown }}
    needs: generate_matrix
    strategy:
      fail-fast: false
      matrix:
        object: ${{ fromJson(needs.generate_matrix.outputs.service_matrix) }}
    uses: ./.github/workflows/build-helm-chart.yml
    with:
      chart: ${{ matrix.object.name }}
      registry: ${{ needs.generate_matrix.outputs.registry }}.azurecr.io
      tag: ${{ needs.generate_matrix.outputs.docker_tag }}
      username: e2e-token
      target: ${{ inputs.target }}
    secrets: inherit

  docker_build:
    name: Build Docker Images
    if: ${{ !inputs.bypassAndTeardown }}
    needs: generate_matrix
    strategy:
      fail-fast: false
      matrix:
        object: ${{ fromJson(needs.generate_matrix.outputs.service_matrix) }}
    uses: ./.github/workflows/build-docker-image.yml
    with:
      image: ${{ matrix.object.name }}
      context: ${{ matrix.object.context }}
      dockerfile: ${{ matrix.object.dockerfile }}
      registry: ${{ needs.generate_matrix.outputs.registry }}.azurecr.io
      tag: ${{ needs.generate_matrix.outputs.docker_tag }}
      username: e2e-token
      target: ${{ inputs.target }}
    secrets: inherit

  deploy_quickstart:
    if: ${{ !inputs.bypassAndTeardown }}
    name: Deploy Quick Start
    needs: [generate_matrix, docker_build]
    runs-on: ubuntu-latest
    env:
      AZURE_ENV_NAME: ${{ inputs.environment }}
      AZURE_LOCATION: ${{ inputs.location }}
    steps:
      - name: Sets Credentials for E2E
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_E2E_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'e2e'
        env:
          FLLM_E2E_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }}

      - name: Sets Credentials for Sandbox
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_SBX_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'sandbox'
        env:
          FLLM_SBX_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }}

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install azd
        uses: Azure/setup-azd@v1.0.0
        with:
          version: "1.9.3"

      - name: Log in with Azure (Client Credentials)
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          cd ./deploy/quick-start
          azd auth login `
            --client-id "$($info.clientId)" `
            --client-secret "$($info.clientSecret)" `
            --tenant-id "$($info.tenantId)"

          az login `
            --service-principal `
            --username "$($info.clientId)" `
            --password "$($info.clientSecret)" `
            --tenant "$($info.tenantId)"

          $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)"
          azcopy login `
            --login-type=SPN `
            --application-id "$($info.clientId)" `
            --tenant-id "$($info.tenantId)"
        shell: pwsh

      - name: Deploy Azure Machine Learning
        if: ${{ inputs.deployAMLResources }}
        env:
          AZURE_ENV_NAME: ext-${{ inputs.environment }}
          AZURE_LOCATION: ${{ inputs.location }}
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Push-Location ./tests/extended-test-env
          azd env new ext-${{ inputs.environment }}
          azd env set AZURE_SUBSCRIPTION_ID $($info.subscriptionId)
          azd up --no-prompt
          Pop-Location
        shell: pwsh

      - name: Provision Admin Group
        id: provision-admin-group
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          Write-Host "Provision Admin Group and add ObjectId to AZD Environment"
          Push-Location ./tests/scripts
          $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv)
          Write-Host "SP Object Id: $spObjectId"
          ./Create-FllmAdminGroup.ps1 `
            -groupName FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }}

          $spAssignmentExists = $(az ad group member check `
            --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} `
            --member-id $spObjectId `
            --output json | ConvertFrom-Json).value

          if (-not $spAssignmentExists)
          {
            az ad group member add `
            --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} `
            --member-id $spObjectId `
            --output json
          }


          if ('${{ inputs.target }}' -eq 'sandbox') {
            $assignmentExists = $(az ad group member check `
              --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} `
              --member-id d3bd4e8e-d413-477d-a420-0792b0504adf `
              --output json | ConvertFrom-Json).value

            if (-not $assignmentExists)
            {
              az ad group member add `
              --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} `
              --member-id d3bd4e8e-d413-477d-a420-0792b0504adf `
              --output json
            }
          }
          Pop-Location
        shell: pwsh

      - name: Update AZD Environment
        id: update-azd-environment
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv)

          Write-Host "Create new AZD Environment"
          Push-Location ./deploy/quick-start
          azd env new ${{ inputs.environment }}
          azd env set PIPELINE_DEPLOY "$true"
          azd env set FLLM_E2E_SP_OBJECT_ID $spObjectId

          if ('${{ inputs.deployOpenAi }}' -eq 'false') {
            azd env set OPENAI_NAME ${{ inputs.openAiName }}
            azd env set OPENAI_RESOURCE_GROUP ${{ inputs.openAiResourceGroup }}
            azd env set OPENAI_SUBSCRIPTION_ID $($info.subscriptionId)
          }

          Pop-Location

          Write-Host "Update AZD Environment with Service Docker Image Names"
          $configPath = "./tests/config/e2e-test-manifest.json"
          $config = Get-content $configPath | ConvertFrom-Json

          Push-Location ./deploy/quick-start
          foreach ($service in $($config.service_matrix)) {
            $imageName = "${{ needs.generate_matrix.outputs.registry }}.azurecr.io/$($service.name):${{ needs.generate_matrix.outputs.docker_tag }}"
            Write-Host "Setting $($service.azd_env_key) to $($imageName)..."
            azd env set $($service.azd_env_key) "$($imageName)"

            if ($LASTEXITCODE -ne 0) {
                Write-Error("Failed to set $($value.Key).")
                exit 1
            }
          }
          Pop-Location

          Write-Host "Update AZD Environment with App Registration Metadata"
          Push-Location ./deploy/quick-start
          ../../tests/scripts/Set-AzdEnv.ps1 `
            -tenantId $($info.tenantId) `
            -principalType ServicePrincipal `
            -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176 `
            -fllmApiName FoundationaLLM-Core-API `
            -fllmClientName FoundationaLLM-Core-Portal `
            -fllmMgmtApiName FoundationaLLM-Management-API `
            -fllmMgmtClientName FoundationaLLM-Management-Portal `
            -fllmAuthApiName FoundationaLLM-Authorization-API `
            -fllmAdminGroupName FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }}
          Pop-Location
        shell: pwsh

      - name: Provision Infrastructure
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          $env:AZURE_SUBSCRIPTION_ID=$($info.subscriptionId)

          Push-Location ./deploy/quick-start
          azd provision --no-prompt
          Pop-Location
        shell: pwsh

      - name: Deploy Application
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          $env:AZURE_SUBSCRIPTION_ID=$($info.subscriptionId)

          Push-Location ./deploy/quick-start
          azd deploy --no-prompt
          Pop-Location
        shell: pwsh

      - name: Publish AZD Environment
        uses: actions/upload-artifact@v4
        with:
          name: azd-env
          path: ./deploy/quick-start/.azure

      - name: Update App Registration OAuth Callbacks
        id: update-app-reg-oauth
        run: |
          Write-Host "Update App Registration OAuth Callbacks"
          Push-Location ./deploy/quick-start
          pwsh -File ../common/scripts/Update-OAuthCallbackUris.ps1 `
            -fllmChatUiName FoundationaLLM-Core-Portal `
            -fllmMgmtUiName FoundationaLLM-Management-Portal
          Pop-Location
        shell: pwsh

      - name: Apply MS Graph API Role Assignments
        id: apply-ms-graph-api-roles
        run: |
          Write-Host "Apply MS Graph API Role Assignments"
          Push-Location ./deploy/quick-start
          pwsh -File ../../tests/scripts/Apply-MSGraphAPIRoles.ps1
          Pop-Location
        shell: pwsh

      - name: Apply Cosmos DB RBAC to Service Principal
        id: apply-cosmos-db-rbac
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          Push-Location ./deploy/quick-start
          (azd env get-values) | foreach {
              $name, $value = $_.split('=')
              set-content env:\$name $value
          }
          Pop-Location

          Write-Host "Apply Cosmos DB Role to Service Principal"
          Push-Location ./tests/scripts
          $spObjectId = $(az ad sp list --filter "appId eq '$($info.clientId)'" --query '[].id' --output tsv)
          Write-Host "SP Object Id: $spObjectId"
          $cosmosDbName = $env:AZURE_COSMOS_DB_NAME | ConvertFrom-Json
          $cosmosDbObjectId = $(az cosmosdb show --name $cosmosDbName --resource-group rg-${{ inputs.environment }} --query 'id' --output tsv)
          Write-Host "CosmosDB Object Id: $cosmosDbObjectId"
          az cosmosdb sql role assignment create `
            --account-name $cosmosDbName `
            --resource-group rg-${{ inputs.environment }} `
            --role-definition-id 00000000-0000-0000-0000-000000000002 `
            --scope $cosmosDbObjectId `
            --principal-id $spObjectId
          Pop-Location
        shell: pwsh

  add_test_data:
    if: ${{ !inputs.bypassAndTeardown }}
    name: Add Test Data
    needs: [deploy_quickstart]
    runs-on: ubuntu-latest # this may need to change to windows
    env:
      AZURE_ENV_NAME: ${{ inputs.environment }}
      AZURE_LOCATION: ${{ inputs.location }}
    steps:
      - name: Sets Credentials for Source (SBX)
        run: |
          {
            echo 'AZURE_CREDENTIALS_SRC<<EOF'
            echo "$FLLM_SBX_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        env:
          FLLM_SBX_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }}

      - name: Sets Credentials for Target (E2E)
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_E2E_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'e2e'
        env:
          FLLM_E2E_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }}

      - name: Sets Credentials for Target (Sandbox)
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_SBX_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'sandbox'
        env:
          FLLM_SBX_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }}

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download AZD Environment
        uses: actions/download-artifact@v4
        with:
          name: azd-env
          path: ./deploy/quick-start/.azure

      - name: Install azd
        uses: Azure/setup-azd@v1.0.0
        with:
          version: "1.9.3"

      - name: Log in with Azure (Client Credentials)
        shell: pwsh
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          Push-Location ./deploy/quick-start
          azd auth login `
            --client-id "$($info.clientId)" `
            --client-secret "$($info.clientSecret)" `
            --tenant-id "$($info.tenantId)"
          Pop-Location

          az login `
            --service-principal `
            --username "$($info.clientId)" `
            --password "$($info.clientSecret)" `
            --tenant "$($info.tenantId)" `

      - name: Copy Test Data
        shell: pwsh
        run: |
          $src = $Env:AZURE_CREDENTIALS_SRC | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($src.clientSecret)"

          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          Push-Location ./deploy/quick-start
          (azd env get-values) | foreach {
              $name, $value = $_.split('=')
              $value = $value.Trim('"')
              set-content env:\$name $value
          }
          Pop-Location

          # Assign “Search Index Data Contributor” on the target resource group
          $objectId = (az ad sp list --all --query "[?appId=='$($info.clientId)'].id" --output tsv)
          az role assignment create `
            --role "Search Index Data Contributor" `
            --assignee-object-id $objectId `
            --assignee-principal-type ServicePrincipal `
            --scope "/subscriptions/${env:AZURE_SUBSCRIPTION_ID}/resourceGroups/${env:RESOURCE_GROUP_NAME_DEFAULT}"

          $env:AZCOPY_SPA_CLIENT_SECRET="$($src.clientSecret)"
          azcopy login `
            --login-type=SPN `
            --application-id "$($src.clientId)" `
            --tenant-id "$($src.tenantId)"
          ./deploy/common/scripts/Get-TestData.ps1 -storageAccountName foundationallmdata

          $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)"
          azcopy login `
            --login-type=SPN `
            --application-id "$($info.clientId)" `
            --tenant-id "$($info.tenantId)"
          ./deploy/common/scripts/Set-TestData.ps1 `
            -storageAccountName $env:AZURE_STORAGE_ACCOUNT_NAME `
            -aiSearchName $env:AZURE_COGNITIVE_SEARCH_NAME

  build_e2e_tests:
    name: Build End to End Tests
    if: ${{ !inputs.bypassAndTeardown }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4

      - name: Setup .NET
        uses: actions/setup-dotnet@v3
        with:
          dotnet-version: 8.x

      - name: Sets Credentials for Source (SBX)
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_SBX_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        env:
          FLLM_SBX_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }}

      - name: Pull Build Artifacts
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Push-Location ./deploy/common/scripts
          $env:AZCOPY_SPA_CLIENT_SECRET="$($info.clientSecret)"
          azcopy login `
            --login-type=SPN `
            --application-id "$($info.clientId)" `
            --tenant-id "$($info.tenantId)"
          ./Get-TestSettings.ps1 -storageAccountName foundationallmdata
          Pop-Location
        shell: pwsh
      
      - name: Build Tests
        shell: pwsh
        run: |
          dotnet restore ./src/FoundationaLLM.sln --output ./dist
          dotnet build ./tests/dotnet/Common.Tests/Common.Tests.csproj --output ./dist
          dotnet build ./tests/dotnet/Core.Examples/Core.Examples.csproj --output ./dist
          dotnet build ./tests/dotnet/Core.Test/Core.Tests.csproj --output ./dist
          dotnet build ./tests/dotnet/Gatekeeper.Tests/Gatekeeper.Tests.csproj --output ./dist
          dotnet build ./tests/dotnet/Orchestration.Tests/Orchestration.Tests.csproj --output ./dist
          dotnet build ./tests/dotnet/SemanticKernel.Tests/SemanticKernel.Tests.csproj --output ./dist
          dotnet build ./tests/dotnet/Vectorization.Tests/Vectorization.Tests.csproj --output ./dist

      - name: Publish Build
        uses: actions/upload-artifact@v4
        with:
          name: build
          path: ./dist

  run_e2e_tests:
    name: Run End to End Tests
    if: ${{ !inputs.bypassAndTeardown }}
    continue-on-error: true
    runs-on: ubuntu-latest
    env:
      AZURE_ENV_NAME: ${{ inputs.environment }}
      AZURE_LOCATION: ${{ inputs.location }}
    needs: [generate_matrix, deploy_quickstart, add_test_data, build_e2e_tests]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        object: ${{ fromJson(needs.generate_matrix.outputs.test_matrix) }}
    steps:
      - name: Sets Credentials for E2E
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_E2E_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'e2e'
        env:
          FLLM_E2E_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }}

      - name: Sets Credentials for Sandbox
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_SBX_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'sandbox'
        env:
          FLLM_SBX_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }}

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download Build
        uses: actions/download-artifact@v4
        with:
          name: build
          path: ./dist

      - name: Download AZD Environment
        uses: actions/download-artifact@v4
        with:
          name: azd-env
          path: ./deploy/quick-start/.azure

      - name: Install azd
        uses: Azure/setup-azd@v1.0.0
        with:
          version: "1.9.3"

      - name: Log in with Azure (Client Credentials)
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          Push-Location ./deploy/quick-start
          azd auth login `
            --client-id "$($info.clientId)" `
            --client-secret "$($info.clientSecret)" `
            --tenant-id "$($info.tenantId)"
          Pop-Location

          az login `
            --service-principal `
            --username "$($info.clientId)" `
            --password "$($info.clientSecret)" `
            --tenant "$($info.tenantId)" `
            --scope api://FoundationaLLM-Core/.default
        shell: pwsh

      - name: Running End to End Tests
        id: e2e-test-exec
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          Push-Location ./deploy/quick-start
          (azd env get-values) | foreach {
              $name, $value = $_.split('=')
              set-content env:\$name $value
          }
          Pop-Location

          $connectionString = $(az appconfig credential list `
            --name $($env:AZURE_APP_CONFIG_NAME | ConvertFrom-Json) `
            --resource-group rg-$($env:AZURE_ENV_NAME) `
            --query "[].{name:name,connectionString:connectionString}[?name=='Primary Read Only'].connectionString" `
            --output tsv)
          $env:FoundationaLLM_AppConfig_ConnectionString=$connectionString

          echo "Running End to End Test: ${{ matrix.object.name }}"
          dotnet test `
            --output ./dist `
            --no-build `
            --filter "FullyQualifiedName=${{ matrix.object.namespace }}.${{ matrix.object.task_name }}" `
            --logger:"html;verbosity=detailed;LogFileName=${{ github.workspace }}/${{ matrix.object.name }}.html" `
            ${{ matrix.object.target }}
        shell: pwsh

      - name: Publish Test Results
        if: (success() || failure())
        id: publish-results
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.object.name }}-results
          path: ${{ github.workspace }}/${{ matrix.object.name }}.html

      - name: Send Test Failure Notification
        if: (inputs.notificationsEnabled && failure())
        uses: slackapi/slack-github-action@v1.26.0
        id: notify-failed-test
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.FLLM_SLACK_WEBHOOK_URL }}
          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
        with:
          payload: |
            {
              "blocks": [
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "*End to End Test Failure: ${{ matrix.object.name }}*"
                  }
                },
                {
                  "type": "section",
                  "text": {
                    "type": "mrkdwn",
                    "text": "End to End Test ${{ matrix.object.name }} failed in branch `${{ github.head_ref || github.ref_name }}`"
                  }
                },
                {
                  "type": "actions",
                  "elements": [
                    {
                      "type": "button",
                      "text": {
                        "type": "plain_text",
                        "text": "Download Test Results"
                      },
                      "url": "${{ steps.publish-results.outputs.artifact-url }}"
                    }
                  ]
                }
              ]
            }

      - name: Output Job Status
        if: (success() || failure())
        uses: cloudposse/github-action-matrix-outputs-write@v1
        id: job-status
        with:
          matrix-step-name: ${{ github.job }}
          matrix-key: ${{ matrix.object.name }}
          outputs: |
            passed: ${{ steps.e2e-test-exec.outcome }}
            
  teardown_quickstart:
    name: Tear Down Quick Start
    if: ${{ (cancelled() && inputs.bypassAndTeardown) || inputs.enableTeardown }}
    runs-on: ubuntu-latest
    env:
      AZURE_ENV_NAME: ${{ inputs.environment }}
      AZURE_LOCATION: ${{ inputs.location }}
    needs: [generate_matrix, run_e2e_tests]
    steps:
      - name: Sets Credentials for E2E
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_E2E_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'e2e'
        env:
          FLLM_E2E_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_E2E_AZURE_CREDENTIALS }}

      - name: Sets Credentials for Sandbox
        run: |
          {
            echo 'AZURE_CREDENTIALS<<EOF'
            echo "$FLLM_SBX_AZURE_CREDENTIALS"
            echo EOF
          } >> "$GITHUB_ENV"
        if: inputs.target == 'sandbox'
        env:
          FLLM_SBX_AZURE_CREDENTIALS: |
            ${{ secrets.FLLM_SBX_AZURE_CREDENTIALS }}

      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download AZD Environment
        uses: actions/download-artifact@v4
        with:
          name: azd-env
          path: ./deploy/quick-start/.azure

      - name: Install azd
        uses: Azure/setup-azd@v1.0.0
        with:
          version: "1.9.3"

      - name: Log in with Azure (Client Credentials)
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          cd ./deploy/quick-start
          azd auth login `
            --client-id "$($info.clientId)" `
            --client-secret "$($info.clientSecret)" `
            --tenant-id "$($info.tenantId)"

          az login `
            --service-principal `
            --username "$($info.clientId)" `
            --password "$($info.clientSecret)" `
            --tenant "$($info.tenantId)"
        shell: pwsh

      - name: Tearing Down Azure Machine Learning
        if: ${{ inputs.deployAMLResources }}
        env:
          AZURE_ENV_NAME: ext-${{ inputs.environment }}
          AZURE_LOCATION: ${{ inputs.location }}
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Push-Location ./tests/extended-test-env
          azd env new ext-${{ inputs.environment }}
          $env:AZURE_SUBSCRIPTION_ID = $($info.subscriptionId)
          azd env refresh --no-prompt
          azd down --force --purge --no-prompt
          Pop-Location
        shell: pwsh
      
      - name: Tearing Down AZD Deployment
        id: azd-down
        run: |
          $info = $Env:AZURE_CREDENTIALS | ConvertFrom-Json -AsHashtable;
          Write-Host "::add-mask::$($info.clientSecret)"

          Push-Location ./deploy/quick-start
          Write-Host "Removing App Registration OAuth Callbacks"
          pwsh -File ../common/scripts/Remove-OAuthCallbackUris.ps1 `
            -fllmChatUiName FoundationaLLM-Core-Portal `
            -fllmMgmtUiName FoundationaLLM-Management-Portal

          azd down --force --purge --no-prompt
          Pop-Location

          Write-Host "Remove Admin Group"
          az ad group delete --group FLLM-E2E-Admins-${{ needs.generate_matrix.outputs.unique_id }} --output json
        shell: pwsh

      - name: Get E2E Test Results
        uses: cloudposse/github-action-matrix-outputs-read@v1
        id: get-e2e-results
        with:
          matrix-step-name: run_e2e_tests
  
      - name: Check E2E Test Status
        uses: actions/github-script@v7
        env:
          TEST_RESULTS: ${{ steps.get-e2e-results.outputs.result }}
        with:
          script: |
            const jsonString = process.env.TEST_RESULTS
            var data = JSON.parse(jsonString)
            var tests = data.passed
            for (const testName of Object.keys(tests)) {
              if (tests[testName] !== 'success') {
                process.exit(1);
              }
            }