Merge branch 'main' of https://github.com/solliancenet/foundationallm …

…into skm-parameterize-load-test
solliancenet · Jun 26, 2024 · 9a979c4 · 9a979c4
2 parents fe92110 + e15bba7
commit 9a979c4
Show file tree

Hide file tree

Showing 9 changed files with 109 additions and 44 deletions.
diff --git a/.github/workflows/e2e-testing.yml b/.github/workflows/e2e-testing.yml
@@ -587,6 +587,7 @@ jobs:
     needs: [generate_matrix, deploy_quickstart, add_test_data, build_e2e_tests]
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         object: ${{ fromJson(needs.generate_matrix.outputs.test_matrix) }}
     steps:
@@ -829,11 +830,9 @@ jobs:
             -instanceId 6fa496ce-d5c0-4e02-9223-06a98c9c0176
 
           Write-Host "Removing App Registration OAuth Callbacks"
-          Push-Location ./deploy/quick-start
           pwsh -File ../../tests/scripts/Remove-OAuthCallbackUris.ps1 `
             -fllmChatUiName FoundationaLLM-Core-Portal `
             -fllmMgmtUiName FoundationaLLM-Management-Portal
-          Pop-Location
 
           azd env refresh --no-prompt
           azd down --force --purge --no-prompt

diff --git a/deploy/quick-start/infra/main.bicep b/deploy/quick-start/infra/main.bicep
@@ -442,22 +442,28 @@ module configTopic 'shared/config-system-topic.bicep' = {
   name: 'configTopic-${timestamp}'
   params: {
     name: '${abbrs.eventGridDomainsTopics}config${resourceToken}'
+    eventGridName: eventgrid.outputs.name
+    destinationTopicName: 'configuration'
     location: location
     tags: tags
     appConfigAccountName: appConfig.outputs.name
   }
   scope: rg
+  dependsOn: [eventgrid]
 }
 
 module storageTopic 'shared/storage-system-topic.bicep' = {
   name: 'storageTopic-${timestamp}'
   params: {
     name: '${abbrs.eventGridDomainsTopics}storage${resourceToken}'
+    eventGridName: eventgrid.outputs.name
+    destinationTopicName: 'storage'
     location: location
     tags: tags
     storageAccountName: storage.outputs.name
   }
   scope: rg
+  dependsOn: [eventgrid]
 }
 
 module storageSub 'shared/system-topic-subscription.bicep' = {
@@ -633,6 +639,23 @@ module openAiRoles './shared/roleAssignments.bicep' = [
   }
 ]
 
+var contentSafetyTargets = [
+  'gatekeeper-api'
+]
+
+module contentSafetyRoles './shared/roleAssignments.bicep' = [
+  for target in contentSafetyTargets: {
+    scope: rg
+    name: '${target}-cs-roles-${timestamp}'
+    params: {
+      principalId: acaServices[indexOf(serviceNames, target)].outputs.miPrincipalId
+      roleDefinitionNames: [
+        'Cognitive Services User'
+      ]
+    }
+  }
+]
+
 output AZURE_APP_CONFIG_NAME string = appConfig.outputs.name
 output AZURE_AUTHORIZATION_STORAGE_ACCOUNT_NAME string = authStore.outputs.name
 output AZURE_COGNITIVE_SEARCH_ENDPOINT string = cogSearch.outputs.endpoint

diff --git a/deploy/quick-start/infra/shared/config-system-topic.bicep b/deploy/quick-start/infra/shared/config-system-topic.bicep
@@ -1,4 +1,6 @@
 param name string
+param destinationTopicName string
+param eventGridName string
 param location string = resourceGroup().location
 param tags object = {}
 param appConfigAccountName string
@@ -7,6 +9,25 @@ resource appConfig 'Microsoft.AppConfiguration/configurationStores@2023-08-01-pr
   name: appConfigAccountName
 }
 
+resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2023-12-15-preview' existing = {
+  name: eventGridName
+}
+
+resource destinationTopic 'Microsoft.EventGrid/namespaces/topics@2023-12-15-preview' existing = {
+  name: destinationTopicName
+  parent: eventGridNamespace
+}
+
+resource eventSendRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  scope: destinationTopic
+  name: guid(subscription().id, resourceGroup().id, topic.id, 'sendEventRole')
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'd5a91429-5739-47e2-a06b-3470a27159e7')
+    principalType: 'ServicePrincipal'
+    principalId: topic.identity.principalId
+  }
+}
+
 resource topic 'Microsoft.EventGrid/systemTopics@2023-12-15-preview' = {
   name: name
   location: location

diff --git a/deploy/quick-start/infra/shared/roleAssignments.bicep b/deploy/quick-start/infra/shared/roleAssignments.bicep
@@ -10,6 +10,7 @@ var selectedRoleDefinitions = filter(items(roleDefinitions), (item) => contains(
 var roleDefinitions = {
   'App Configuration Data Reader':  '516239f1-63e1-4d78-a4de-a74fb236a071'
   'Cognitive Services OpenAI User': '5e0bd9bd-7b93-4f28-af87-19fc36ad61bd'
+  'Cognitive Services User':        'a97b65f3-24c7-4388-baec-2e87135dc908'
   'Contributor':                    'b24988ac-6180-42a0-ab88-20f7382dd24c'
   'EventGrid Contributor':          '1e241071-0855-49ea-94dc-649edcd759de'
   'Key Vault Certificate User':     'db79e9a7-68ee-4b58-9aeb-b90e7c24fcba'

diff --git a/deploy/quick-start/infra/shared/storage-system-topic.bicep b/deploy/quick-start/infra/shared/storage-system-topic.bicep
@@ -1,4 +1,6 @@
 param name string
+param destinationTopicName string
+param eventGridName string
 param location string = resourceGroup().location
 param tags object = {}
 param storageAccountName string
@@ -7,6 +9,25 @@ resource storage 'Microsoft.Storage/storageAccounts@2023-01-01' existing = {
   name: storageAccountName
 }
 
+resource eventGridNamespace 'Microsoft.EventGrid/namespaces@2023-12-15-preview' existing = {
+  name: eventGridName
+}
+
+resource destinationTopic 'Microsoft.EventGrid/namespaces/topics@2023-12-15-preview' existing = {
+  name: destinationTopicName
+  parent: eventGridNamespace
+}
+
+resource eventSendRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
+  scope: destinationTopic
+  name: guid(subscription().id, resourceGroup().id, topic.id, 'sendEventRole')
+  properties: {
+    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'd5a91429-5739-47e2-a06b-3470a27159e7')
+    principalType: 'ServicePrincipal'
+    principalId: topic.identity.principalId
+  }
+}
+
 resource topic 'Microsoft.EventGrid/systemTopics@2023-12-15-preview' = {
   name: name
   location: location

diff --git a/deploy/quick-start/infra/shared/system-topic-subscription.bicep b/deploy/quick-start/infra/shared/system-topic-subscription.bicep
@@ -19,16 +19,6 @@ resource topic 'Microsoft.EventGrid/systemTopics@2023-12-15-preview' existing =
   name: topicName
 }
 
-resource eventSendRole 'Microsoft.Authorization/roleAssignments@2022-04-01' = {
-  scope: destinationTopic
-  name: guid(subscription().id, resourceGroup().id, topic.id, 'sendEventRole')
-  properties: {
-    roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', 'd5a91429-5739-47e2-a06b-3470a27159e7')
-    principalType: 'ServicePrincipal'
-    principalId: topic.identity.principalId
-  }
-}
-
 resource resourceProviderSub 'Microsoft.EventGrid/systemTopics/eventSubscriptions@2023-12-15-preview' = {
   name: name
   parent: topic
@@ -56,5 +46,4 @@ resource resourceProviderSub 'Microsoft.EventGrid/systemTopics/eventSubscription
       eventTimeToLiveInMinutes: 1440
     }
   }
-  dependsOn: [ eventSendRole ]
 }
diff --git a/src/dotnet/DataSource/ResourceProviders/DataSourceResourceProviderService.cs b/src/dotnet/DataSource/ResourceProviders/DataSourceResourceProviderService.cs
@@ -444,14 +444,14 @@ protected override T GetResourceInternal<T>(ResourcePath resourcePath) where T :
 
             if (typeof(T) != typeof(DataSourceBase))
                 throw new ResourceProviderException($"The type of requested resource ({typeof(T)}) does not match the resource type specified in the path ({resourcePath.ResourceTypeInstances[0].ResourceType}).");
-
+                     
             _dataSourceReferences.TryGetValue(resourcePath.ResourceTypeInstances[0].ResourceId!, out var dataSourceReference);
-            if (dataSourceReference == null || dataSourceReference.Deleted)
-                throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId!} of type {resourcePath.ResourceTypeInstances[0].ResourceType} was not found.");
+            if (dataSourceReference is not null && dataSourceReference.Deleted)
+                throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId} of type {resourcePath.ResourceTypeInstances[0].ResourceType} has been soft deleted.");
 
-            var dataSource = LoadDataSource(dataSourceReference).Result;
+            var dataSource = LoadDataSource(dataSourceReference, resourcePath.ResourceTypeInstances[0].ResourceId).Result;
             return dataSource as T
-                ?? throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId!} of type {resourcePath.ResourceTypeInstances[0].ResourceType} was not found.");
+                ?? throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId} of type {resourcePath.ResourceTypeInstances[0].ResourceType} was not found.");
         }
 
 

diff --git a/src/dotnet/Vectorization/ResourceProviders/VectorizationResourceProviderService.cs b/src/dotnet/Vectorization/ResourceProviders/VectorizationResourceProviderService.cs
@@ -596,7 +596,8 @@ private T GetTextPartitioningProfile<T>(ResourcePath resourcePath) where T : cla
             if (typeof(T) != typeof(TextPartitioningProfile))
                 throw new ResourceProviderException($"The type of requested resource ({typeof(T)}) does not match the resource type specified in the path ({resourcePath.ResourceTypeInstances[0].ResourceType}).");
 
-            _textPartitioningProfiles.TryGetValue(resourcePath.ResourceTypeInstances[0].ResourceId!, out var textPartitioningProfile);
+            var textPartitioningProfileGetResult = LoadResources<TextPartitioningProfile, VectorizationProfileBase>(resourcePath.ResourceTypeInstances[0], _textPartitioningProfiles).Result;
+            var textPartitioningProfile = textPartitioningProfileGetResult.FirstOrDefault()?.Resource;           
             return textPartitioningProfile as T
                 ?? throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId!} of type {resourcePath.ResourceTypeInstances[0].ResourceType} was not found.");
         }
@@ -609,7 +610,8 @@ private T GetTextEmbeddingProfile<T>(ResourcePath resourcePath) where T : class
             if (typeof(T) != typeof(TextEmbeddingProfile))
                 throw new ResourceProviderException($"The type of requested resource ({typeof(T)}) does not match the resource type specified in the path ({resourcePath.ResourceTypeInstances[0].ResourceType}).");
 
-            _textEmbeddingProfiles.TryGetValue(resourcePath.ResourceTypeInstances[0].ResourceId!, out var textEmbeddingProfile);
+            var textEmbeddingProfileGetResult = LoadResources<TextEmbeddingProfile, VectorizationProfileBase>(resourcePath.ResourceTypeInstances[0], _textEmbeddingProfiles).Result;
+            var textEmbeddingProfile = textEmbeddingProfileGetResult.FirstOrDefault()?.Resource;
             return textEmbeddingProfile as T
                 ?? throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId!} of type {resourcePath.ResourceTypeInstances[0].ResourceType} was not found.");
         }
@@ -622,7 +624,9 @@ private T GetIndexingProfile<T>(ResourcePath resourcePath) where T : class
             if (typeof(T) != typeof(IndexingProfile))
                 throw new ResourceProviderException($"The type of requested resource ({typeof(T)}) does not match the resource type specified in the path ({resourcePath.ResourceTypeInstances[0].ResourceType}).");
 
-            _indexingProfiles.TryGetValue(resourcePath.ResourceTypeInstances[0].ResourceId!, out var indexingProfile);
+            var indexingProfileGetResult = LoadResources<IndexingProfile, VectorizationProfileBase>(resourcePath.ResourceTypeInstances[0], _indexingProfiles).Result;
+            var indexingProfile = indexingProfileGetResult.FirstOrDefault()?.Resource;
+
             return indexingProfile as T
                 ?? throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId!} of type {resourcePath.ResourceTypeInstances[0].ResourceType} was not found.");
         }
@@ -635,7 +639,9 @@ private T GetVectorizationProfile<T>(ResourcePath resourcePath) where T : class
             if (typeof(T) != typeof(VectorizationPipeline))
                 throw new ResourceProviderException($"The type of requested resource ({typeof(T)}) does not match the resource type specified in the path ({resourcePath.ResourceTypeInstances[0].ResourceType}).");
 
-            _pipelines.TryGetValue(resourcePath.ResourceTypeInstances[0].ResourceId!, out var pipeline);
+            var pipelineGetResult = LoadResources<VectorizationPipeline, VectorizationPipeline>(resourcePath.ResourceTypeInstances[0], _pipelines).Result;
+            var pipeline = pipelineGetResult.FirstOrDefault()?.Resource;
+
             return pipeline as T
                 ?? throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId!} of type {resourcePath.ResourceTypeInstances[0].ResourceType} was not found.");
         }
@@ -857,6 +863,9 @@ private async Task HandleVectorizationResourceProviderEvent(CloudEvent e)
                 case INDEXING_PROFILES_FILE_NAME:
                     _defaultIndexingProfileName = await LoadResourceStore<IndexingProfile, VectorizationProfileBase>(INDEXING_PROFILES_FILE_PATH, _indexingProfiles);
                     break;
+                case PIPELINES_FILE_NAME:
+                    _ = await LoadResourceStore<VectorizationPipeline, VectorizationPipeline>(PIPELINES_FILE_PATH, _pipelines);
+                    break;
                 default:
                     _logger.LogWarning("The file {FileName} is not managed by the FoundationaLLM.Vectorization resource provider.", fileName);
                     break;

diff --git a/tests/scripts/Remove-OAuthCallbackUris.ps1 b/tests/scripts/Remove-OAuthCallbackUris.ps1
@@ -68,31 +68,33 @@ $uris = @{
 }
 
 foreach ($uri in $uris.GetEnumerator()) {
-    $applicationUri = "https://graph.microsoft.com/v1.0/applications/" + $uri.Value.objectId
-    $redirects = @(az rest `
-        --method "get" `
-        --uri $applicationUri `
-        --headers "{'Content-Type': 'application/json'}" `
-        --query $uri.Value.query `
-        -o json | ConvertFrom-Json)
-
-    $redirect = ($uri.Value.endpoint | ConvertFrom-Json) + "/signin-oidc"
+    if ($uri.Value.endpoint -ne $null)
+    {
+        $applicationUri = "https://graph.microsoft.com/v1.0/applications/" + $uri.Value.objectId
+        $redirects = @(az rest `
+            --method "get" `
+            --uri $applicationUri `
+            --headers "{'Content-Type': 'application/json'}" `
+            --query $uri.Value.query `
+            -o json | ConvertFrom-Json)
 
+        $redirect = ($uri.Value.endpoint | ConvertFrom-Json) + "/signin-oidc"
 
-    if ($redirects.Contains($redirect)) {
-        $redirects -= $redirect
+        if ($redirects.Contains($redirect)) {
+            $redirects -= $redirect
 
-        $body = @{
-            spa = @{
-                redirectUris = $redirects
-            }
-        } | ConvertTo-Json -Compress
+            $body = @{
+                spa = @{
+                    redirectUris = $redirects
+                }
+            } | ConvertTo-Json -Compress
 
-        Set-Content -Path "$($uri.Key)`.json" $body
-        az rest `
-            --method "patch" `
-            --uri $applicationUri `
-            --headers "{'Content-Type': 'application/json'}" `
-            --body "@$($uri.Key)`.json"
+            Set-Content -Path "$($uri.Key)`.json" $body
+            az rest `
+                --method "patch" `
+                --uri $applicationUri `
+                --headers "{'Content-Type': 'application/json'}" `
+                --body "@$($uri.Key)`.json"
+        }
     }
 }