Skip to content

Commit

Permalink
Merge pull request #44 from Fr4nc3/PSL-US-10919_1
Browse files Browse the repository at this point in the history
feat: Disable Integrated Vectorization & Disable Computer Vision Client When PostgreSQL Is Enabled
  • Loading branch information
Fr4nc3 authored Nov 29, 2024
2 parents 185efe1 + 68a968f commit 76ad443
Show file tree
Hide file tree
Showing 5 changed files with 336 additions and 303 deletions.
77 changes: 40 additions & 37 deletions code/backend/batch/utilities/helpers/env_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,46 @@ def __load_config(self, **kwargs) -> None:
"AZURE_SEARCH_DATASOURCE_NAME", ""
)
self.AZURE_SEARCH_INDEXER_NAME = os.getenv("AZURE_SEARCH_INDEXER_NAME", "")
self.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION = self.get_env_var_bool(
"AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION", "False"
)

# Chat History DB Integration Settings
# Set default values based on DATABASE_TYPE
self.DATABASE_TYPE = os.getenv("DATABASE_TYPE", "").strip() or "CosmosDB"
# Cosmos DB configuration
if self.DATABASE_TYPE == DatabaseType.COSMOSDB.value:
azure_cosmosdb_info = self.get_info_from_env("AZURE_COSMOSDB_INFO", "")
self.AZURE_COSMOSDB_DATABASE = azure_cosmosdb_info.get("databaseName", "")
self.AZURE_COSMOSDB_ACCOUNT = azure_cosmosdb_info.get("accountName", "")
self.AZURE_COSMOSDB_CONVERSATIONS_CONTAINER = azure_cosmosdb_info.get(
"containerName", ""
)
self.AZURE_COSMOSDB_ACCOUNT_KEY = self.secretHelper.get_secret(
"AZURE_COSMOSDB_ACCOUNT_KEY"
)
self.AZURE_COSMOSDB_ENABLE_FEEDBACK = (
os.getenv("AZURE_COSMOSDB_ENABLE_FEEDBACK", "false").lower() == "true"
)
self.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION = self.get_env_var_bool(
"AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION", "False"
)
self.USE_ADVANCED_IMAGE_PROCESSING = self.get_env_var_bool(
"USE_ADVANCED_IMAGE_PROCESSING", "False"
)
# PostgreSQL configuration
elif self.DATABASE_TYPE == DatabaseType.POSTGRESQL.value:
self.AZURE_POSTGRES_SEARCH_TOP_K = self.get_env_var_int(
"AZURE_POSTGRES_SEARCH_TOP_K", 5
)
azure_postgresql_info = self.get_info_from_env("AZURE_POSTGRESQL_INFO", "")
self.POSTGRESQL_USER = azure_postgresql_info.get("user", "")
self.POSTGRESQL_DATABASE = azure_postgresql_info.get("dbname", "")
self.POSTGRESQL_HOST = azure_postgresql_info.get("host", "")
# Ensure integrated vectorization is disabled for PostgreSQL
self.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION = "False"
self.USE_ADVANCED_IMAGE_PROCESSING = "False"
else:
raise ValueError(
"Unsupported DATABASE_TYPE. Please set DATABASE_TYPE to 'CosmosDB' or 'PostgreSQL'."
)

self.AZURE_AUTH_TYPE = os.getenv("AZURE_AUTH_TYPE", "keys")
# Azure OpenAI
Expand Down Expand Up @@ -147,9 +184,6 @@ def __load_config(self, **kwargs) -> None:
self.AZURE_TOKEN_PROVIDER = get_bearer_token_provider(
DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default"
)
self.USE_ADVANCED_IMAGE_PROCESSING = self.get_env_var_bool(
"USE_ADVANCED_IMAGE_PROCESSING", "False"
)
self.ADVANCED_IMAGE_PROCESSING_MAX_IMAGES = self.get_env_var_int(
"ADVANCED_IMAGE_PROCESSING_MAX_IMAGES", 1
)
Expand Down Expand Up @@ -294,37 +328,6 @@ def __load_config(self, **kwargs) -> None:

self.PROMPT_FLOW_DEPLOYMENT_NAME = os.getenv("PROMPT_FLOW_DEPLOYMENT_NAME", "")

# Chat History DB Integration Settings
# Set default values based on DATABASE_TYPE
self.DATABASE_TYPE = os.getenv("DATABASE_TYPE", "").strip() or "CosmosDB"
# Cosmos DB configuration
if self.DATABASE_TYPE == DatabaseType.COSMOSDB.value:
azure_cosmosdb_info = self.get_info_from_env("AZURE_COSMOSDB_INFO", "")
self.AZURE_COSMOSDB_DATABASE = azure_cosmosdb_info.get("databaseName", "")
self.AZURE_COSMOSDB_ACCOUNT = azure_cosmosdb_info.get("accountName", "")
self.AZURE_COSMOSDB_CONVERSATIONS_CONTAINER = azure_cosmosdb_info.get(
"containerName", ""
)
self.AZURE_COSMOSDB_ACCOUNT_KEY = self.secretHelper.get_secret(
"AZURE_COSMOSDB_ACCOUNT_KEY"
)
self.AZURE_COSMOSDB_ENABLE_FEEDBACK = (
os.getenv("AZURE_COSMOSDB_ENABLE_FEEDBACK", "false").lower() == "true"
)
# PostgreSQL configuration
elif self.DATABASE_TYPE == DatabaseType.POSTGRESQL.value:
self.AZURE_POSTGRES_SEARCH_TOP_K = self.get_env_var_int(
"AZURE_POSTGRES_SEARCH_TOP_K", 5
)
azure_postgresql_info = self.get_info_from_env("AZURE_POSTGRESQL_INFO", "")
self.POSTGRESQL_USER = azure_postgresql_info.get("user", "")
self.POSTGRESQL_DATABASE = azure_postgresql_info.get("dbname", "")
self.POSTGRESQL_HOST = azure_postgresql_info.get("host", "")
else:
raise ValueError(
"Unsupported DATABASE_TYPE. Please set DATABASE_TYPE to 'CosmosDB' or 'PostgreSQL'."
)

def is_chat_model(self):
if "gpt-4" in self.AZURE_OPENAI_MODEL_NAME.lower():
return True
Expand Down
6 changes: 3 additions & 3 deletions infra/app/storekeys.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ param formRecognizerName string = ''
param contentSafetyName string = ''
param speechServiceName string = ''
param computerVisionName string = ''
param postgresServerName string = '' // PostgreSQL server name
param postgresDatabaseName string = 'postgres' // Default database name
param postgresInfoName string = 'AZURE-POSTGRESQL-INFO' // Secret name for PostgreSQL info
param postgresServerName string = '' // PostgreSQL server name
param postgresDatabaseName string = 'PostgreSQL' // Default database name
param postgresInfoName string = 'AZURE-POSTGRESQL-INFO' // Secret name for PostgreSQL info
param postgresDatabaseAdminUserName string = ''
param postgresDatabaseAdminPassword string = ''
param storageAccountKeyName string = 'AZURE-STORAGE-ACCOUNT-KEY'
Expand Down
203 changes: 108 additions & 95 deletions infra/app/web.bicep
Original file line number Diff line number Diff line change
Expand Up @@ -31,52 +31,62 @@ param useDocker bool = dockerFullImageName != ''
param healthCheckPath string = ''

// Database parameters
param databaseType string = 'cosmos' // 'cosmos' or 'postgres'
param databaseType string = 'CosmosDB' // 'CosmosDB' or 'PostgreSQL'
param cosmosDBKeyName string = ''
param postgresInfoName string = ''

var azureFormRecognizerInfoUpdated = useKeyVault
? azureFormRecognizerInfo
: replace(azureFormRecognizerInfo, '$FORM_RECOGNIZER_KEY', listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
formRecognizerName
),
'2023-05-01'
).key1)
: replace(
azureFormRecognizerInfo,
'$FORM_RECOGNIZER_KEY',
listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
formRecognizerName
),
'2023-05-01'
).key1
)

var azureBlobStorageInfoUpdated = useKeyVault
? azureBlobStorageInfo
: replace(azureBlobStorageInfo, '$STORAGE_ACCOUNT_KEY', listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.Storage/storageAccounts',
storageAccountName
),
'2021-09-01'
).keys[0].value)

// Database-specific settings
var databaseSettings = databaseType == 'cosmos' ? {
DATABASE_TYPE: 'cosmos'
AZURE_COSMOSDB_ACCOUNT_KEY: (useKeyVault || cosmosDBKeyName == '')
? cosmosDBKeyName
: listKeys(
: replace(
azureBlobStorageInfo,
'$STORAGE_ACCOUNT_KEY',
listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.DocumentDB/databaseAccounts',
cosmosDBKeyName
'Microsoft.Storage/storageAccounts',
storageAccountName
),
'2022-08-15'
).primaryMasterKey
} : {
DATABASE_TYPE: 'postgres'
AZURE_POSTGRESQL_INFO: useKeyVault ? postgresInfoName : ''
}
'2021-09-01'
).keys[0].value
)

// Database-specific settings
var databaseSettings = databaseType == 'CosmosDB'
? {
DATABASE_TYPE: 'CosmosDB'
AZURE_COSMOSDB_ACCOUNT_KEY: (useKeyVault || cosmosDBKeyName == '')
? cosmosDBKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.DocumentDB/databaseAccounts',
cosmosDBKeyName
),
'2022-08-15'
).primaryMasterKey
}
: {
DATABASE_TYPE: 'PostgreSQL'
AZURE_POSTGRESQL_INFO: useKeyVault ? postgresInfoName : ''
}

module web '../core/host/appservice.bicep' = {
name: '${name}-app-module'
Expand All @@ -88,67 +98,70 @@ module web '../core/host/appservice.bicep' = {
appCommandLine: useDocker ? '' : appCommandLine
applicationInsightsName: applicationInsightsName
appServicePlanId: appServicePlanId
appSettings: union(appSettings, union(databaseSettings, {
AZURE_AUTH_TYPE: authType
USE_KEY_VAULT: useKeyVault ? useKeyVault : ''
AZURE_OPENAI_API_KEY: useKeyVault
? openAIKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
azureOpenAIName
),
'2023-05-01'
).key1
AZURE_SEARCH_KEY: useKeyVault
? searchKeyName
: listAdminKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.Search/searchServices',
azureAISearchName
),
'2021-04-01-preview'
).primaryKey
AZURE_BLOB_STORAGE_INFO: azureBlobStorageInfoUpdated
AZURE_FORM_RECOGNIZER_INFO: azureFormRecognizerInfoUpdated
AZURE_CONTENT_SAFETY_KEY: useKeyVault
? contentSafetyKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
contentSafetyName
),
'2023-05-01'
).key1
AZURE_SPEECH_SERVICE_KEY: useKeyVault
? speechKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
speechServiceName
),
'2023-05-01'
).key1
AZURE_COMPUTER_VISION_KEY: (useKeyVault || computerVisionName == '')
? computerVisionKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
computerVisionName
),
'2023-05-01'
).key1
}))
appSettings: union(
appSettings,
union(databaseSettings, {
AZURE_AUTH_TYPE: authType
USE_KEY_VAULT: useKeyVault ? useKeyVault : ''
AZURE_OPENAI_API_KEY: useKeyVault
? openAIKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
azureOpenAIName
),
'2023-05-01'
).key1
AZURE_SEARCH_KEY: useKeyVault
? searchKeyName
: listAdminKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.Search/searchServices',
azureAISearchName
),
'2021-04-01-preview'
).primaryKey
AZURE_BLOB_STORAGE_INFO: azureBlobStorageInfoUpdated
AZURE_FORM_RECOGNIZER_INFO: azureFormRecognizerInfoUpdated
AZURE_CONTENT_SAFETY_KEY: useKeyVault
? contentSafetyKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
contentSafetyName
),
'2023-05-01'
).key1
AZURE_SPEECH_SERVICE_KEY: useKeyVault
? speechKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
speechServiceName
),
'2023-05-01'
).key1
AZURE_COMPUTER_VISION_KEY: (useKeyVault || computerVisionName == '')
? computerVisionKeyName
: listKeys(
resourceId(
subscription().subscriptionId,
resourceGroup().name,
'Microsoft.CognitiveServices/accounts',
computerVisionName
),
'2023-05-01'
).key1
})
)
keyVaultName: keyVaultName
runtimeName: runtimeName
runtimeVersion: runtimeVersion
Expand Down
Loading

0 comments on commit 76ad443

Please sign in to comment.