From d527c8036f8178e4bdffd151096e24f196663349 Mon Sep 17 00:00:00 2001 From: Sensational Code Date: Tue, 30 Jan 2024 20:38:46 -0800 Subject: [PATCH 1/5] Account for base url in public path asset requests --- src/ui/ManagementPortal/components/Sidebar.vue | 2 +- src/ui/ManagementPortal/nuxt.config.ts | 2 +- src/ui/ManagementPortal/plugins/filters.ts | 9 +++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/ui/ManagementPortal/components/Sidebar.vue b/src/ui/ManagementPortal/components/Sidebar.vue index c22f07ee1f..2f7ccfd790 100644 --- a/src/ui/ManagementPortal/components/Sidebar.vue +++ b/src/ui/ManagementPortal/components/Sidebar.vue @@ -2,7 +2,7 @@
- + {{ $appConfigStore.logoText }}
diff --git a/src/ui/ManagementPortal/nuxt.config.ts b/src/ui/ManagementPortal/nuxt.config.ts index 24413f9e89..606ce46e29 100644 --- a/src/ui/ManagementPortal/nuxt.config.ts +++ b/src/ui/ManagementPortal/nuxt.config.ts @@ -25,7 +25,7 @@ export default defineNuxtConfig({ { rel: 'icon', type: 'image/x-icon', - href: process.env.BRANDING_FAV_ICON_URL ?? '/favicon.ico', + href: process.env.NUXT_APP_BASE_URL + (process.env.BRANDING_FAV_ICON_URL ?? 'favicon.ico'), }, ], }, diff --git a/src/ui/ManagementPortal/plugins/filters.ts b/src/ui/ManagementPortal/plugins/filters.ts index ea5a46eb01..54bdc09776 100644 --- a/src/ui/ManagementPortal/plugins/filters.ts +++ b/src/ui/ManagementPortal/plugins/filters.ts @@ -1,10 +1,7 @@ const filters = { - enforceLeadingSlash(path: string) { - if (!path.startsWith('/')) { - return '/' + path; - } else { - return path; - } + publicDirectory(path: string) { + const config = useRuntimeConfig(); + return config.app.baseURL + path; } }; From f5e154512deb8258449e33e4586ab09f2f596e5c Mon Sep 17 00:00:00 2001 From: Ciprian Jichici Date: Wed, 31 Jan 2024 12:08:35 +0200 Subject: [PATCH 2/5] Vectorization configuration and profiles --- docs/setup-guides/vectorization/index.md | 13 +- .../vectorization/vectorization-api.md | 26 -- .../vectorization/vectorization-concepts.md | 35 +++ ...rker.md => vectorization-configuration.md} | 150 +++++----- .../vectorization-content-sources.md | 90 ------ ...ectorization-monitoring-troubleshooting.md | 1 + .../vectorization/vectorization-profiles.md | 269 ++++++++++++++++++ .../vectorization/vectorization-triggering.md | 55 ++++ docs/toc.yml | 16 +- 9 files changed, 463 insertions(+), 192 deletions(-) delete mode 100644 docs/setup-guides/vectorization/vectorization-api.md create mode 100644 docs/setup-guides/vectorization/vectorization-concepts.md rename docs/setup-guides/vectorization/{vectorization-worker.md => vectorization-configuration.md} (56%) delete mode 100644 docs/setup-guides/vectorization/vectorization-content-sources.md create mode 100644 docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md create mode 100644 docs/setup-guides/vectorization/vectorization-profiles.md create mode 100644 docs/setup-guides/vectorization/vectorization-triggering.md diff --git a/docs/setup-guides/vectorization/index.md b/docs/setup-guides/vectorization/index.md index 5d1cfdab38..280925a8fa 100644 --- a/docs/setup-guides/vectorization/index.md +++ b/docs/setup-guides/vectorization/index.md @@ -1,8 +1,13 @@ # Vectorization -Foundationa**LLM** (FLLM) provides utilities and services to support vectorizing your data in batch and on-demand modalities. Vectorization is a multi-step process, starting with loading your data, splitting (or chunking) the data as required, performing vector embeddings, and storing the vectors into a vector index so [an agent](../agents/index.md) can later retrieve relevant information through a vector search. Use the links below to learn more about configuring vectorization in FLLM: +This section describes the content vectorization concepts and how to configure vectorization in FoundationaLLM. -- [Vectorization API](vectorization-api.md) -- [Vectorization Worker](vectorization-worker.md) +Use the links below to learn more about configuring vectorization in FoundationaLLM: -To get started with the Vectorization API, import the [provided Postman collection.](./FoundationaLLM.Vectorization.API.postman_collection.json) Ensure to set the Vectorization API key (`X-API-KEY` header) on the **Authorization** tab of the collection and the Base URL (`baseUrl` collection variable) on the **Variables** tab. +- [Vectorization concepts](vectorization-concepts.md) +- [Configuring vectorization](vectorization-configuration.md) +- [Managing vectorization profiles](vectorization-profiles.md) +- [Triggering vectorization](vectorization-triggering.md) +- [Monitoring and troubleshooting vectorization](vectorization-monitoring.md) + +If you are already familiar with these details, you can start working with the Vectorization API by importing the [FoundationaLLM Vectorization Postman collection.](./FoundationaLLM.Vectorization.API.postman_collection.json) Ensure to set the Vectorization API key (`X-API-KEY` header) on the **Authorization** tab of the collection and the Base URL (`baseUrl` collection variable) on the **Variables** tab. \ No newline at end of file diff --git a/docs/setup-guides/vectorization/vectorization-api.md b/docs/setup-guides/vectorization/vectorization-api.md deleted file mode 100644 index 8b47c74cd1..0000000000 --- a/docs/setup-guides/vectorization/vectorization-api.md +++ /dev/null @@ -1,26 +0,0 @@ -# Vectorization API - -## Configuration - -The following table describes the Azure artifacts required for the vectorization pipelines. - -| Artifact name | Description | -| --- | --- | -| `vectorization-input` | Azure storage container used by default to store documents to be picked up by the vectorization pipeline. Must be created on a Data Lake storage account (with the hierarchical namespace enabled). | - -The following table describes the environment variables required for the vectorization pipelines. - -Environment variable | Description ---- | --- -`FoundationaLLM:AppConfig:ConnectionString` | Connection string to the Azure App Configuration instance. - -The following table describes the required configuration parameters for the vectorization pipelines. - -| App Configuration Key | Default Value | Description | -| --- | --- | --- | -| `FoundationaLLM:APIs:VectorizationAPI:APIUrl` | | The URL of the vectorization API. | -| `FoundationaLLM:APIs:VectorizationAPI:APIKey` | Key Vault secret name: `foundationallm-apis-vectorizationapi-apikey` | The API key of the vectorization API. | -| `FoundationaLLM:APIs:VectorizationAPI:AppInsightsConnectionString` | Key Vault secret name: `foundationallm-app-insights-connection-string` | The connection string to the Application Insights instance used by the vectorization API. | - -> [!NOTE] -> Refer to the [App Configuration values](../../app-configuration-values.md) page for more information on how to set these and other configuration values. diff --git a/docs/setup-guides/vectorization/vectorization-concepts.md b/docs/setup-guides/vectorization/vectorization-concepts.md new file mode 100644 index 0000000000..0c961edba3 --- /dev/null +++ b/docs/setup-guides/vectorization/vectorization-concepts.md @@ -0,0 +1,35 @@ +# Vectorization Concepts + +Foundationa**LLM** (FLLM) provides utilities and services to support vectorizing your data in batch and on-demand modalities. Vectorization is a multi-step process, starting with loading your data, splitting (or chunking) the data as required, performing vector embeddings, and storing the vectors into a vector index so [an agent](../agents/index.md) can later retrieve relevant information through a vector search. In FLLM, vectorization is an idempotent operation, meaning that vectorizing the same document multiple times will result in the same vector being stored in the vector index. This is useful for re-vectorizing documents that have been updated, or for cases where only parts of the vectorization process need to be run (e.g., only the vector embeddings need to be updated). + +For each individual document, vectorization is performed by executing a vectorization pipeline instance. The execution is started by the Vectorization API upon receiving a valid vectorization request. Based on the details of the vectorizaiton request, a vectorization pipeline can be executed in one of the following modes: + +- Synchonously - the pipeline steps start executing immediately and execute sequentially until the pipeline is completed or an error occurs. This type of execution is used for on-demand vectorization and is well suited for small to medium sized documents and relatively small numbers of documents at a time. +- Asynchronously - the pipeline steps are submitted to queues and executed by workers. This type of execution is used for batch vectorization and is well suited for large numbers of documents at a time. It is also well suited for vectorizing documents of any size. + +The FLLM platform components involved in vectorization are: + +- Vectorization API (processes vectorization requests and executes synchonous vectorization pipelines). +- Vectorization Worker(s) (execute asynchronous vectorization pipelines). + +A FLLM instance deploys one instance of the Vectorization API and one or more instances of the Vectorization Worker. See [Configuring vectorization](vectorization-configuration.md) for more details on configuring these components. + +> [!NOTE] +> The initialization of both the Vectorization API and the Vectorization Worker is a time consuming process, as it involves dowloading and initializing various elements (e.g., Byte-Pair encoding dictionaries). As a result, after restarting the API, it might take up to a minute until it becomes ready to accept vectorization requests. It is recommended to either use the status endpoint of the Vectorization API to determine when it is ready to accept requests, or to wait for a minute after restarting the API before sending vectorization requests. + +Vectorization pipelines are started when the Vectorization API receives a vectorization request. The following types of triggers are supported: + +- None (no triggering of vectorization pipelines). +- Manual (vectorization pipelines are triggered manually by calling the Vectorization API). The typical use cases for on-demand vectorization (either synchronous or asynchronous) are testing, manual vectorization (or re-vectorization), and application integration (where another platform component triggers vectorization). +- Content-based (vectorization pipelines are triggered automatically when either new content is added to a content source or existing content is updated). +- Schedule-based (vectorization pipelines are triggered automatically based on a schedule). + +> [!NOTE] +> Content-based and schedule-based triggering are currently in pre-release and are not yet available in public releases of FLLM. + +When working with vectorization in FLLM, the typical steps you have to perform are: + +- Ensure that the Vectorization API and Vectorization Worker are configured and running. This is a one-time operation. For more details, see [Configuring vectorization](vectorization-configuration.md). +- Create vectorization profiles. You can either reuse existing profiles or create new ones. For more details, see [Managing vectorization profiles](vectorization-profiles.md). +- Submit vectorization requests to the Vectorization API. For more details, see [Triggering vectorization](vectorization-triggering.md). + diff --git a/docs/setup-guides/vectorization/vectorization-worker.md b/docs/setup-guides/vectorization/vectorization-configuration.md similarity index 56% rename from docs/setup-guides/vectorization/vectorization-worker.md rename to docs/setup-guides/vectorization/vectorization-configuration.md index 5b77a41f5e..1dd7e24e9c 100644 --- a/docs/setup-guides/vectorization/vectorization-worker.md +++ b/docs/setup-guides/vectorization/vectorization-configuration.md @@ -1,6 +1,37 @@ -# Vectorization Worker +# Configuring vectorization -## Configuration +This section provides details on how to configure the vectorization API and workers in FoundationaLLM. + +> [!NOTE] +> These configurations should already be in place if you deployed FoundationaLLM (FLLM) using the recommended deployment scripts. +> The detailes presented here are provided for cases in which you need to troubleshoot or customize the configuration. + +## Configuration for Vectorization API + +The following table describes the Azure artifacts required for the vectorization pipelines. + +| Artifact name | Description | +| --- | --- | +| `vectorization-input` | Azure storage container used by default to store documents to be picked up by the vectorization pipeline. Must be created on a Data Lake storage account (with the hierarchical namespace enabled). | + +The following table describes the environment variables required for the vectorization pipelines. + +Environment variable | Description +--- | --- +`FoundationaLLM:AppConfig:ConnectionString` | Connection string to the Azure App Configuration instance. + +The following table describes the required configuration parameters for the vectorization pipelines. + +| App Configuration Key | Default Value | Description | +| --- | --- | --- | +| `FoundationaLLM:APIs:VectorizationAPI:APIUrl` | | The URL of the vectorization API. | +| `FoundationaLLM:APIs:VectorizationAPI:APIKey` | Key Vault secret name: `foundationallm-apis-vectorizationapi-apikey` | The API key of the vectorization API. | +| `FoundationaLLM:APIs:VectorizationAPI:AppInsightsConnectionString` | Key Vault secret name: `foundationallm-app-insights-connection-string` | The connection string to the Application Insights instance used by the vectorization API. | + +> [!NOTE] +> Refer to the [App Configuration values](../../deployment/app-configuration-values.md) page for more information on how to set these and other configuration values. + +## Configuration for Vectorization workers The following table describes the Azure artifacts required for the vectorization pipelines. @@ -13,7 +44,7 @@ The following table describes the Azure artifacts required for the vectorization | `vectorization-state` | Azure storage container used for the vectorization state service. Can be created on the storage account used for the other queues. | | `resource-provider`| Azure storage container used for the internal states of the FoundationaLLM resource providers. | | `resource-provider/FoundationaLLM.Vectorization/vectorization-content-source-profiles.json` | Azure storage blob used for the content sources managed by the `FoundationaLLM.Vectorization` resource provider. For more details, see [default vectorization content source profiles](#default-vectorization-content-source-profiles). -| `resource-provider/FoundationaLLM.Vectorization/vectorization-text-partition-profiles.json` | Azure storage blob used for the text partitioning profiles managed by the `FoundationaLLM.Vectorization` resource provider. For more details, see [default vectorization text partitioning profiles](#default-vectorization-text-partitioning-profiles). +| `resource-provider/FoundationaLLM.Vectorization/vectorization-text-partitioning-profiles.json` | Azure storage blob used for the text partitioning profiles managed by the `FoundationaLLM.Vectorization` resource provider. For more details, see [default vectorization text partitioning profiles](#default-vectorization-text-partitioning-profiles). | `resource-provider/FoundationaLLM.Vectorization/vectorization-text-embedding-profiles.json` | Azure storage blob used for the text embedding profiles managed by the `FoundationaLLM.Vectorization` resource provider. For more details, see [default vectorization text embedding profiles](#default-vectorization-text-embedding-profiles). | `resource-provider/FoundationaLLM.Vectorization/vectorization-indexing-profiles.json` | Azure storage blob used for the indexing profiles managed by the `FoundationaLLM.Vectorization` resource provider. For more details, see [default vectorization indexing profiles](#default-vectorization-indexing-profiles). @@ -54,14 +85,14 @@ The following table describes the external content used by the vectorization wor | Uri | Description | | --- | --- | -| `https://openaipublic.blob.core.windows.net` | The public Azure Blob Storage account used to download the OpenAI BPE ranking files. | +| `https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken` | The public Azure Blob Storage account used to download the OpenAI BPE ranking files. | > [!NOTE] > The vectorization worker must be able to open HTTPS connections to the external content listed above. ### Default vectorization worker settings -Default settings for the vectorization worker: +The default settings for the vectorization worker are stored in the `FoundationaLLM:Vectorization:VectorizationWorker` App Configuration key. The default structure for this key is: ```json { @@ -87,28 +118,37 @@ Default settings for the vectorization worker: { "Name": "extract", "ConnectionConfigurationName": "Extract:ConnectionString", - "VisibilityTimeoutSeconds": 120 + "VisibilityTimeoutSeconds": 600 }, { "Name": "partition", "ConnectionConfigurationName": "Partition:ConnectionString", - "VisibilityTimeoutSeconds": 120 + "VisibilityTimeoutSeconds": 600 }, { "Name": "embed", "ConnectionConfigurationName": "Embed:ConnectionString", - "VisibilityTimeoutSeconds": 120 + "VisibilityTimeoutSeconds": 600 }, { "Name": "index", "ConnectionConfigurationName": "Index:ConnectionString", - "VisibilityTimeoutSeconds": 120 + "VisibilityTimeoutSeconds": 600 } ], "QueuingEngine": "AzureStorageQueue" } ``` +The following table provides details about the configuration parameters: + +| Parameter | Description | +| --- | --- | +| `RequestManagers` | The list of request managers used by the vectorization worker. Each request manager is responsible for managing the execution of vectorization pipelines for a specific vectorization step. The configuration must include all request managers. | +| `RequestManagers.MaxHandlerInstances` | The maximum number of request handlers that process requests for the specified request source. By default, the value is 1. You can change the value to increase the processing capacity of each vectorization worker instance. The value applies to all istances of the vectorization worker. NOTE: It is important to align the value of this setting with the level of compute and memory resources allocated to the individual vectorization worker instances. | +| `RequestSources` | The list of request sources used by the vectorization worker. Each request source is responsible for managing the requests for a specific vectorization step. The configuration must include all request sources. | +| `RequestSources.VisibilityTimeoutSeconds` | In the case of queue-based request sources (the default for the vectorization worker), specifies the time in seconds until a dequeued vectorization step request must be executed. During this timeout, the message will not be visible to other handler instances within the same worker or from other worker instances. If the handler fails to process the vectorization step request successfully and remove it from the queue within the specified timeout, the message will become visibile again. The default value is 600 seconds and should not be changed.| + ### Default vectorization content source profiles Default structure for the `vectorization-content-source-profiles.json` file: @@ -117,29 +157,39 @@ Default structure for the `vectorization-content-source-profiles.json` file: { "ContentSourceProfiles": [ { - "Name": "SDZWAJournals", + "Name": "DefaultAzureDataLake", "Type": "AzureDataLake", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/contentsourceprofiles/DefaultAzureDataLake", "Settings": {}, "ConfigurationReferences": { - "AuthenticationType": "FoundationaLLM:Vectorization:ContentSources:SDZWAJournals:AuthenticationType", - "ConnectionString": "FoundationaLLM:Vectorization:ContentSources:SDZWAJournals:ConnectionString" + "AuthenticationType": "FoundationaLLM:Vectorization:ContentSources:DefaultAzureDataLake:AuthenticationType", + "ConnectionString": "FoundationaLLM:Vectorization:ContentSources:DefaultAzureDataLake:ConnectionString" } } ] } ``` +By default, FLLM includes one content source profile named `DefaultAzureDataLake`. You can add content source profiles to this file to configure the content sources used by the vectorization pipelines. For more details, see [Managing vectorization profiles](vectorization-profiles.md). + +Currently, the following content source types are supported: + +- `AzureDataLake` - uses an Azure Data Lake storage account as the content source (see [`AzureDataLakeContentSource`](./vectorization-profiles.md#azuredatalake)). +- `SharePointOnline` - uses a SharePoint Online site as the content source (see [`SharePointOnlineContentSource`](./vectorization-profiles.md#sharepointonline)). +- `AzureSQLDatabase` - uses an Azure SQL database as the content source (see [`AzureSQLDatabaseContentSource`](./vectorization-profiles.md#azuresqldatabase)). + ### Default vectorization text partitioning profiles -Default structure for the `vectorization-text-partition-profiles.json` file: +Default structure for the `vectorization-text-partitioning-profiles.json` file: ```json { "TextPartitioningProfiles": [ { "Name": "DefaultTokenTextPartition", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/textpartitioningprofiles/DefaultTokenTextPartition", "TextSplitter": "TokenTextSplitter", - "TextSplitterSettings": { + "Settings": { "Tokenizer": "MicrosoftBPETokenizer", "TokenizerEncoder": "cl100k_base", "ChunkSizeTokens": "2000", @@ -150,6 +200,12 @@ Default structure for the `vectorization-text-partition-profiles.json` file: } ``` +By default, FLLM includes one text partitioning profile named `DefaultTokenTextPartition` which uses the `TokenTextSplitter` text splitter. You can add text partitioning profiles to this file to configure the text partitioning used by the vectorization pipelines. For more details, see [Managing vectorization profiles](vectorization-profiles.md). + +Currently, the following text splitters are supported: + +- `TokenTextSplitter` - splits the text into chunks based on the number of tokens (see [`TextTokenSplitter`](./vectorization-profiles.md#texttokensplitter)). + ### Default vectorization text embedding profiles Default structure for the `vectorization-text-embedding-profiles.json` file: @@ -159,6 +215,7 @@ Default structure for the `vectorization-text-embedding-profiles.json` file: "TextEmbeddingProfiles": [ { "Name": "AzureOpenAI_Embedding", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/textembeddingprofiles/AzureOpenAI_Embedding", "TextEmbedding": "SemanticKernelTextEmbedding", "Settings": {}, "ConfigurationReferences": { @@ -172,6 +229,13 @@ Default structure for the `vectorization-text-embedding-profiles.json` file: } ``` +By default, FLLM includes one text embedding profile named `AzureOpenAI_Embedding` which uses the `SemanticKernelTextEmbedding` text embedder. You can add text embedding profiles to this file to configure the text embedding used by the vectorization pipelines. For more details, see [Managing vectorization profiles](vectorization-profiles.md). + +Currently, the following text embedders are supported: + +- `SemanticKernelTextEmbedding` - embeds the text using Semantic Kernel to call into the default FLLM Azure OpenAI embedding model (see [`SemanticKernelTextEmbedding`](./vectorization-profiles.md#semantickerneltextembedding)). + + ### Default vectorization indexing profiles Default structure for the `vectorization-indexing-profiles.json` file: @@ -180,10 +244,11 @@ Default structure for the `vectorization-indexing-profiles.json` file: { "IndexingProfiles": [ { - "Name": "AzureAISearch_Test_001", + "Name": "AzureAISearch_Default_001", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/indexingprofiles/AzureAISearch_Default_001", "Indexer": "AzureAISearchIndexer", "Settings": { - "IndexName": "fllm-test-001" + "IndexName": "fllm-default-001" }, "ConfigurationReferences": { "APIKey": "FoundationaLLM:Vectorization:AzureAISearchIndexingService:APIKey", @@ -195,55 +260,8 @@ Default structure for the `vectorization-indexing-profiles.json` file: } ``` -## Vectorization request +By default, FLLM includes one indexing profile named `AzureAISearch_Default_001` which uses the `AzureAISearchIndexer` indexer. You can add indexing profiles to this file to configure the indexing used by the vectorization pipelines. For more details, see [Managing vectorization profiles](vectorization-profiles.md). -Sample structure of a vectorization request: - -```json -{ - "id": "d4669c9c-e330-450a-a41c-a4d6649abdef", - "content_identifier": { - "content_source_profile_name": "SDZWAJournals", - "multipart_id": [ - "https://fllmaks14sa.blob.core.windows.net", - "vectorization-input", - "SDZWA-Journal-January-2024.pdf" - ], - "canonical_id": "sdzwa/journals/SDZWA-Journal-January-2024" - }, - "processing_type": "Asynchronous", - "steps": [ - { - "id": "extract", - "parameters": {} - }, - { - "id": "partition", - "parameters": { - "text_partition_profile_name": "DefaultTokenTextPartition" - } - }, - { - "id": "embed", - "parameters": { - "text_embedding_profile_name": "AzureOpenAI_Embedding" - } - }, - { - "id": "index", - "parameters": { - "indexing_profile_name": "AzureAISearch_Test_001" - } - } - ], - "completed_steps": [], - "remaining_steps": [ - "extract", - "partition", - "embed", - "index" - ] -} -``` +Currently, the following indexers are supported: -The `processing_type` property can be one of `Asynchronous` or `Synchronous`. The `Asynchronous` value indicates that the vectorization request is processed asynchronously via the Vectorization workers. The `Synchronous` value indicates that the vectorization request is processed synchronously via the Vectorization API. +- `AzureAISearchIndexer` - indexes the vectors into an Azure AI Search index (see [`AzureAISearchIndexer`](./vectorization-profiles.md#azureaisearchindexer)). diff --git a/docs/setup-guides/vectorization/vectorization-content-sources.md b/docs/setup-guides/vectorization/vectorization-content-sources.md deleted file mode 100644 index e8ad209daf..0000000000 --- a/docs/setup-guides/vectorization/vectorization-content-sources.md +++ /dev/null @@ -1,90 +0,0 @@ -# Vectorization Content Sources - -This document describes the types of vectorization content sources that are supported by the vectorization pipeline. - -## Azure Data Lake - -The following table describes the required configuration parameters for the Azure Data Lake vectorization content source. - -App configuration key | Referenced Key Vault key | Description ---- | --- | --- -`FoundationaLLM:Vectorization:ContentSources::AuthenticationType` | N/A | The authentication type used to connect to the underlying storage. Can be one of `AzureIdentity`, `AccountKey`, or `ConnectionString`. -`FoundationaLLM:Vectorization:ContentSources::ConnectionString` | `foundationallm-vectorization-contentsources--connectionstring` | The connection string to the Azure Storage account used for the the Azure Data Lake vectorization content source. - ->**NOTE** -> -> `` is a placeholder that needs to be replaced by the actual name of the vectorization content source. It should follow the App Configuration key naming conventions. Do not use special characters like: `%`, `.`, `..`, `:`, or `/`. -> -> Example: `FoundationaLLM:Vectorization:ContentSources:SDZWAJournals:ConnectionString` - -## Azure SQL Database - -The following table describes the required configuration parameters for the Azure SQL Database vectorization content source. - -App configuration key | Referenced Key Vault key | Description ---- | --- | --- -`FoundationaLLM:Vectorization:ContentSources::ConnectionString` | `foundationallm-vectorization-contentsources--connectionstring` | The connection string to the Azure SQL database used for the Azure SQL Database vectorization content source. - -## SharePoint Online - -The following table describes the required configuration parameters for the SharePoint Online vectorization content source. - -App configuration key | Referenced Key Vault key | Description ---- | --- | --- -`FoundationaLLM:Vectorization:ContentSources::ClientId` | N/A | The Application (client) Id of the Microsoft Entra ID App Registration. For more details, see the [Entra ID App Registration](#entra-id-app-registration) section. -`FoundationaLLM:Vectorization:ContentSources::TenantId` | N/A | The unique identifier of the SharePoint Online tenant. -`FoundationaLLM:Vectorization:ContentSources::KeyVaultURL` | N/A | The URL of the KeyVault where the X.509 Certificate is stored. -`FoundationaLLM:Vectorization:ContentSources::CertificateName` | N/A | The name of the X.509 Certificate. - -### Entra ID App Registration - -Apps typically access SharePoint Online through certificates: Anyone having the certificate and its private key can use the app with the permissions granted to it. - -1. Create a new **App registration** in your **Microsoft Entra ID** tenant. Next, provide a **Name** for your application and click on **Register** at the bottom of the blade. - -2. Navigate to the **API Permissions** blade and click on **Add a permission** button Here you choose the permissions that you will grant to this application. Select **SharePoint** from the **Microsoft APIs** tab, then select **Application permissions** as the type of permissions required, choose the desired permissions (i.e. **Sites.Read.All**) and click on **Add permissions**. Here are the required scopes: - - - `Group.ReadWrite.All` - - `User.ReadWrite.All` - - `Sites.Read.All` OR `Sites.Selected` - - `Sites.Read.All` will allow the application to read documents and list items in all site collections. - - `Sites.Selected` will allow the application to access only a subset of site collections. The specific site collections and the permissions granted will be configured separately, in SharePoint Online. - -3. The application permission requires admin consent in a tenant before it can be used. In order to do this, click on **API permissions** in the left menu again. At the bottom you will see a section **Grant consent**. Click on the **Grant admin consent for {{organization}}** button and confirm the action by clicking on the **Yes** button that appears at the top. - -4. To invoke SharePoint Online with an app-only access token, you have to create and configure a **self-signed X.509 certificate**, which will be used to authenticate your application against Microsoft Entra ID. You can find additional details on how to do this in [this document](https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azuread#setting-up-an-azure-ad-app-for-app-only-access). - -5. Next step is to register the certificate you created to this application. Click on **Certificates & secrets** blade. Next, click on the **Upload certificate** button, select the .CER file you generated earlier and click on **Add** to upload it. - - To confirm that the certificate was successfully registered, click on **Manifest** blade and search for the `keyCredentials` property, which contains your certificate details. It should look like this: - ```json - "keyCredentials": [ - { - "customKeyIdentifier": "<$base64CertHash>", - "endDate": "yyyy-MM-ddThh:mm:ssZ", - "keyId": "<$guid>", - "startDate": "yyyy-MM-ddThh:mm:ssZ", - "type": "AsymmetricX509Cert", - "usage": "Verify", - "value": "<$base64Cert>", - "displayName": "CN=<$name of your cert>" - } - ] - ``` - -6. Upload and store the certificate in the **KeyVault** where the FoundationaLLM Vectorization API has permissions to read **Secrets**. You will need the **Certificate Name** for the App Configuration settings listed in the table above. - - > **NOTE** - > - > Can I use other means besides certificates for realizing app-only access for my Azure AD app? - > - > **NO**, all other options are blocked by SharePoint Online and will result in an `Access Denied` message. - -7. Navigate to the deployment storage account and edit the `resource-provider/FoundationaLLM.Vectorization/vectorization-content-source-profiles.json` file. Append the following snippet to the `ContentSourceProfiles` array. - - ```json - { - "Name": "[CONTENT SOURCE NAME]", - "Type": "SharePointOnline" - } - ``` \ No newline at end of file diff --git a/docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md b/docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md new file mode 100644 index 0000000000..f46cd9f843 --- /dev/null +++ b/docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md @@ -0,0 +1 @@ +# Monitoring and troubleshooting vectorization \ No newline at end of file diff --git a/docs/setup-guides/vectorization/vectorization-profiles.md b/docs/setup-guides/vectorization/vectorization-profiles.md new file mode 100644 index 0000000000..76d3ebe863 --- /dev/null +++ b/docs/setup-guides/vectorization/vectorization-profiles.md @@ -0,0 +1,269 @@ +# Managing vectorization profiles + +The FoundationaLLM (FLLM) vectorization pipelines required the following types of profiles: + +- [Content source profiles](#content-source-profiles) +- [Text partitioning profiles](#text-partitioning-profiles) +- [Text embedding profiles](#text-embedding-profiles) +- [Indexing profiles](#indexing-profiles) + +## Content source profiles + +The structure of a content source profile is the following: + +```json +{ + "Name": "", + "Type": "", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/contentsourceprofiles/", + "Settings": {}, + "ConfigurationReferences": {} +} +``` +where: + +- `` is the name of the content source profile. +- `` is the type of the content source profile. The supported types are `AzureDataLake`, `SharePointOnline`, and `AzureSQLDatabase`. +- `` is a JSON object containing the profile settings. +- `` is a JSON object containing the profile configuration references. + +The reminder of this section describes the configuration parameters for each of the supported content source types. + +### `AzureDataLake` + +```json +"Settings": {}, +"ConfigurationReferences": { + "AuthenticationType": "FoundationaLLM:Vectorization:ContentSources::AuthenticationType", + "ConnectionString": "FoundationaLLM:Vectorization:ContentSources::ConnectionString" +} +``` + +The configuration parameters for `AzureDataLake` are the following: + +| Parameter | Description | +| --- | --- | +| `ConfigurationReferences.AuthenticationType` | The authentication type used to connect to the underlying storage. Can be one of `AzureIdentity`, `AccountKey`, or `ConnectionString`. | +| `ConfigurationReferences.ConnectionString` | The connection string to the Azure Storage account used for the the Azure Data Lake vectorization content source. | + +### `SharePointOnline` + +```json +"Settings": {}, +"ConfigurationReferences": { + "CertificateName": "FoundationaLLM:Vectorization:ContentSources::CertificateName", + "ClientId": "FoundationaLLM:Vectorization:ContentSources::ClientId", + "KeyVaultURL": "FoundationaLLM:Vectorization:ContentSources::KeyVaultURL", + "TenantId": "FoundationaLLM:Vectorization:ContentSources::TenantId" +} +``` + +The configuration parameters for `SharePointOnline` are the following: + +| Parameter | Description | +| --- | --- | +| `ConfigurationReferences.CertificateName` | The name of the X.509 Certificate. The certificate must be valid and be uploaded into an Azure Key Vault certificate store. | +| `ConfigurationReferences.KeyVaultURL` | The URL of the KeyVault where the X.509 Certificate is stored. | +| `ConfigurationReferences.ClientId` | The Application (client) Id of the Microsoft Entra ID App Registration. See [Entra ID app registration for SharePoint Online content source](#entra-id-app-registration-for-sharepoint-online-content-source). | +| `ConfigurationReferences.TenantId` | The unique identifier of the SharePoint Online tenant. | + +### `AzureSQLDatabase` + +```json +"Settings": {}, +"ConfigurationReferences": { + "ConnectionString": "FoundationaLLM:Vectorization:ContentSources::ConnectionString" +} +``` + +The configuration parameters for `AzureSQLDatabase` are the following: + +| Parameter | Description | +| --- | --- | +| `ConfigurationReferences.ConnectionString` | The connection string to the Azure SQL database used for the Azure SQL Database vectorization content source. | + +## Text partitioning profiles + +The structure of a text partitioning profile is the following: + +```json +{ + "Name": "", + "TextSplitter": "", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/textpartitioningprofiles/", + "Settings": {}, + "ConfigurationReferences": {} +} +``` +where: + +- `` is the name of the text partitioning profile. +- `` is the type of the text splitter. The supported types are `TextTokenSplitter`. +- `` is a JSON object containing the profile settings. +- `` is a JSON object containing the profile configuration references. + +The reminder of this section describes the configuration parameters for each of the supported text splitters. + +### `TextTokenSplitter` + +```json +"Settings" : { + "Tokenizer": "MicrosoftBPETokenizer", + "TokenizerEncoder": "cl100k_base", + "ChunkSizeTokens": "2000", + "OverlapSizeTokens": "200" +}, +"ConfigurationReferences": {} +``` + +The configuration parameters for `TokenTextSplitter` are the following: + +| Parameter | Description | +| --- | --- | +| `Tokenizer` | The tokenizer used to split the text into tokens. Currently, the only supported tokenizer is `MicrosoftBPETokenizer`. Under the hood, it uses the .NET equivalent of OpenAI's [tiktoken](https://github.com/openai/tiktoken). | +| `TokenizerEncoder` | The encoder used by the tokenizer. Currently, the only supported encoder is `cl100k_base`. This encoder is the one currently used by Azure OpenAI (and OpenAI) in `gpt-3.5-turbo` and `gpt-4`. | +| `ChunkSizeTokens` | The maximum number of tokens in each text chunk. | +| `OverlapSizeTokens` | The maximum number of tokens that overlap between two consecutive chunks. | + +## Text embedding profiles + +The structure of a text embedding profile is the following: + +```json +{ + "Name": "", + "TextEmbedding": "", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/textembeddingprofiles/", + "Settings": {}, + "ConfigurationReferences": {} +} +``` +where: + +- `` is the name of the content source profile. +- `` is the type of the text embedder. The supported types are `SemanticKernelTextEmbedding`. +- `` is a JSON object containing the profile settings. +- `` is a JSON object containing the profile configuration references. + +The reminder of this section describes the configuration parameters for each of the supported text embedders. + +### `SemanticKernelTextEmbedding` + +```json +"Settings": {}, +"ConfigurationReferences": { + "APIKey": "FoundationaLLM:Vectorization:SemanticKernelTextEmbeddingService:APIKey", + "AuthenticationType": "FoundationaLLM:Vectorization:SemanticKernelTextEmbeddingService:AuthenticationType", + "DeploymentName": "FoundationaLLM:Vectorization:SemanticKernelTextEmbeddingService:DeploymentName", + "Endpoint": "FoundationaLLM:Vectorization:SemanticKernelTextEmbeddingService:Endpoint" +} +``` + +The configuration parameters for `SemanticKernelTextEmbedding` are the following: + +| Parameter | Description | +| --- | --- | +| `ConfigurationReferences.APIKey` | The API key used to connect to the Azure OpenAI service. By default, this maps to the Azure OpenAI service deployed by FLLM.| +| `ConfigurationReferences.AuthenticationType` | The authentication type used to connect to the Azure OpenAI service. Can be one of `AzureIdentity` or `APIKey`. By default, it is set to `APIKey`.| +| `ConfigurationReferences.DeploymentName` | The name of the Azure OpenAI model deployment. The default value is `embeddings`.| +| `ConfigurationReferences.Endpoint` | The endpoint of the Azure OpenAI service. By default, this maps to the Azure OpenAI service deployed by FLLM. | + +## Indexing profiles + +The structure of an indexing profile is the following: + +```json +{ + "Name": "", + "Indexer": "", + "ObjectId": "/instances//providers/FoundationaLLM.Vectorization/indexingprofiles/", + "Settings": {}, + "ConfigurationReferences": {} +} +``` +where: + +- `` is the name of the content source profile. +- `` is the type of the indexer. The supported types are `AzureAISearchIndexer`. +- `` is a JSON object containing the profile settings. +- `` is a JSON object containing the profile configuration references. + +The reminder of this section describes the configuration parameters for each of the supported indexers. + +### `AzureAISearchIndexer` + +```json +"Settings": { + "IndexName": "fllm-default-001" +}, +"ConfigurationReferences": { + "APIKey": "FoundationaLLM:Vectorization:AzureAISearchIndexingService:APIKey", + "AuthenticationType": "FoundationaLLM:Vectorization:AzureAISearchIndexingService:AuthenticationType", + "Endpoint": "FoundationaLLM:Vectorization:AzureAISearchIndexingService:Endpoint" +} +``` + +The configuration parameters for `AzureAISearchIndexer` are the following: + +| Parameter | Description | +| --- | --- | +| `Settings.IndexName` | The name of the Azure AI Search index. If the index does not exist, it will be created. | +| `ConfigurationReferences.APIKey` | The API key used to connect to the Azure AI Search service. By default, this maps to the Azure AI Search service deployed by FLLM.| +| `ConfigurationReference.AuthenticationType` | The authentication type used to connect to the Azure AI Search service. Can be one of `AzureIdentity` or `APIKey`. By default, it is set to `APIKey`.| +| `ConfigurationReferences.Endpoint` | The endpoint of the Azure AI Search service. By default, this maps to the Azure AI Search service deployed by FLLM. | + + +## Additional configuration steps + +### Entra ID app registration for SharePoint Online content source + +Apps typically access SharePoint Online through certificates: Anyone having the certificate and its private key can use the app with the permissions granted to it. + +1. Create a new **App registration** in your **Microsoft Entra ID** tenant. Next, provide a **Name** for your application and click on **Register** at the bottom of the blade. + +2. Navigate to the **API Permissions** blade and click on **Add a permission** button Here you choose the permissions that you will grant to this application. Select **SharePoint** from the **Microsoft APIs** tab, then select **Application permissions** as the type of permissions required, choose the desired permissions (i.e. **Sites.Read.All**) and click on **Add permissions**. Here are the required scopes: + + - `Group.ReadWrite.All` + - `User.ReadWrite.All` + - `Sites.Read.All` OR `Sites.Selected` + - `Sites.Read.All` will allow the application to read documents and list items in all site collections. + - `Sites.Selected` will allow the application to access only a subset of site collections. The specific site collections and the permissions granted will be configured separately, in SharePoint Online. + +3. The application permission requires admin consent in a tenant before it can be used. In order to do this, click on **API permissions** in the left menu again. At the bottom you will see a section **Grant consent**. Click on the **Grant admin consent for {{organization}}** button and confirm the action by clicking on the **Yes** button that appears at the top. + +4. To invoke SharePoint Online with an app-only access token, you have to create and configure a **self-signed X.509 certificate**, which will be used to authenticate your application against Microsoft Entra ID. You can find additional details on how to do this in [this document](https://learn.microsoft.com/en-us/sharepoint/dev/solution-guidance/security-apponly-azuread#setting-up-an-azure-ad-app-for-app-only-access). + +5. Next step is to register the certificate you created to this application. Click on **Certificates & secrets** blade. Next, click on the **Upload certificate** button, select the .CER file you generated earlier and click on **Add** to upload it. + + To confirm that the certificate was successfully registered, click on **Manifest** blade and search for the `keyCredentials` property, which contains your certificate details. It should look like this: + ```json + "keyCredentials": [ + { + "customKeyIdentifier": "<$base64CertHash>", + "endDate": "yyyy-MM-ddThh:mm:ssZ", + "keyId": "<$guid>", + "startDate": "yyyy-MM-ddThh:mm:ssZ", + "type": "AsymmetricX509Cert", + "usage": "Verify", + "value": "<$base64Cert>", + "displayName": "CN=<$name of your cert>" + } + ] + ``` + +6. Upload and store the certificate in the **KeyVault** where the FoundationaLLM Vectorization API has permissions to read **Secrets**. You will need the **Certificate Name** for the App Configuration settings listed in the table above. + + > **NOTE** + > + > Can I use other means besides certificates for realizing app-only access for my Azure AD app? + > + > **NO**, all other options are blocked by SharePoint Online and will result in an `Access Denied` message. + +7. Navigate to the deployment storage account and edit the `resource-provider/FoundationaLLM.Vectorization/vectorization-content-source-profiles.json` file. Append the following snippet to the `ContentSourceProfiles` array. + + ```json + { + "Name": "[CONTENT SOURCE NAME]", + "Type": "SharePointOnline" + } + ``` \ No newline at end of file diff --git a/docs/setup-guides/vectorization/vectorization-triggering.md b/docs/setup-guides/vectorization/vectorization-triggering.md new file mode 100644 index 0000000000..fb8280411b --- /dev/null +++ b/docs/setup-guides/vectorization/vectorization-triggering.md @@ -0,0 +1,55 @@ +# Triggering Vectorization + + +## Vectorization request + +Sample structure of a vectorization request: + +```json +{ + "id": "d4669c9c-e330-450a-a41c-a4d6649abdef", + "content_identifier": { + "content_source_profile_name": "SDZWAJournals", + "multipart_id": [ + "https://fllmaks14sa.blob.core.windows.net", + "vectorization-input", + "SDZWA-Journal-January-2024.pdf" + ], + "canonical_id": "sdzwa/journals/SDZWA-Journal-January-2024" + }, + "processing_type": "Asynchronous", + "steps": [ + { + "id": "extract", + "parameters": {} + }, + { + "id": "partition", + "parameters": { + "text_partition_profile_name": "DefaultTokenTextPartition" + } + }, + { + "id": "embed", + "parameters": { + "text_embedding_profile_name": "AzureOpenAI_Embedding" + } + }, + { + "id": "index", + "parameters": { + "indexing_profile_name": "AzureAISearch_Test_001" + } + } + ], + "completed_steps": [], + "remaining_steps": [ + "extract", + "partition", + "embed", + "index" + ] +} +``` + +The `processing_type` property can be one of `Asynchronous` or `Synchronous`. The `Asynchronous` value indicates that the vectorization request is processed asynchronously via the Vectorization workers. The `Synchronous` value indicates that the vectorization request is processed synchronously via the Vectorization API. diff --git a/docs/toc.yml b/docs/toc.yml index 978d5763bf..9b36dcd4d7 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -58,12 +58,16 @@ items: - name: Vectorization href: setup-guides/vectorization/index.md items: - - name: Vectorization - href: setup-guides/vectorization/index.md - - name: Vectorization API - href: setup-guides/vectorization/vectorization-api.md - - name: Vectorization Worker - href: setup-guides/vectorization/vectorization-worker.md + - name: Vectorization concepts + href: setup-guides/vectorization/vectorization-concepts.md + - name: Configuring vectorization + href: setup-guides/vectorization/vectorization-configuration.md + - name: Managing vectorization profiles + href: setup-guides/vectorization/vectorization-profiles.md + - name: Triggering vectorization + href: setup-guides/vectorization/vectorization-triggering.md + - name: Vectorization monitoring and troubleshooting + href: setup-guides/vectorization/vectorization-monitoring-troubleshooting.md - name: Operations href: operations/index.md items: From c294ba1a635a683bc4dff634db40139346aa43ff Mon Sep 17 00:00:00 2001 From: Ciprian Jichici Date: Wed, 31 Jan 2024 14:17:11 +0200 Subject: [PATCH 3/5] Vectorization profiles documentation --- .../vectorization/vectorization-profiles.md | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/docs/setup-guides/vectorization/vectorization-profiles.md b/docs/setup-guides/vectorization/vectorization-profiles.md index 76d3ebe863..c22dcf7e4b 100644 --- a/docs/setup-guides/vectorization/vectorization-profiles.md +++ b/docs/setup-guides/vectorization/vectorization-profiles.md @@ -82,6 +82,31 @@ The configuration parameters for `AzureSQLDatabase` are the following: | --- | --- | | `ConfigurationReferences.ConnectionString` | The connection string to the Azure SQL database used for the Azure SQL Database vectorization content source. | +### Managing content source profiles + +**Retrieve** + +``` +HTTP GET {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/contentsourceprofiles +``` + +**Create or update** + +``` +HTTP POST {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/contentsourceprofiles/ +Content-Type: application/json + +BODY + +``` +where `` is a JSON object with the structure described above. + +**Delete** + +``` +HTTP DELETE {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/contentsourceprofiles/ +``` + ## Text partitioning profiles The structure of a text partitioning profile is the following: @@ -125,6 +150,33 @@ The configuration parameters for `TokenTextSplitter` are the following: | `ChunkSizeTokens` | The maximum number of tokens in each text chunk. | | `OverlapSizeTokens` | The maximum number of tokens that overlap between two consecutive chunks. | +### Managing text partitioning profiles + +**Retrieve** + +``` +HTTP GET {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/textpartitioningprofiles +``` + +**Create or update** + +``` +HTTP POST {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/textpartitioningprofiles/ +Content-Type: application/json + +BODY + +``` + +where `` is a JSON object with the structure described above. + +**Delete** + +``` +HTTP DELETE {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/textpartitioningprofiles/ +``` + + ## Text embedding profiles The structure of a text embedding profile is the following: @@ -168,6 +220,32 @@ The configuration parameters for `SemanticKernelTextEmbedding` are the following | `ConfigurationReferences.DeploymentName` | The name of the Azure OpenAI model deployment. The default value is `embeddings`.| | `ConfigurationReferences.Endpoint` | The endpoint of the Azure OpenAI service. By default, this maps to the Azure OpenAI service deployed by FLLM. | +### Managing text embedding profiles + +**Retrieve** + +``` +HTTP GET {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/textembeddingprofiles +``` + +**Create or update** + +``` +HTTP POST {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/textembeddingprofiles/ +Content-Type: application/json + +BODY + +``` + +where `` is a JSON object with the structure described above. + +**Delete** + +``` +HTTP DELETE {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/textembeddingprofiles/ +``` + ## Indexing profiles The structure of an indexing profile is the following: @@ -212,6 +290,31 @@ The configuration parameters for `AzureAISearchIndexer` are the following: | `ConfigurationReference.AuthenticationType` | The authentication type used to connect to the Azure AI Search service. Can be one of `AzureIdentity` or `APIKey`. By default, it is set to `APIKey`.| | `ConfigurationReferences.Endpoint` | The endpoint of the Azure AI Search service. By default, this maps to the Azure AI Search service deployed by FLLM. | +### Managing indexing profiles + +**Retrieve** + +``` +HTTP GET {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/indexingprofiles +``` + +**Create or update** + +``` +HTTP POST {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/indexingprofiles/ +Content-Type: application/json + +BODY + +``` + +where `` is a JSON object with the structure described above. + +**Delete** + +``` +HTTP DELETE {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/indexingprofiles/ +``` ## Additional configuration steps From fb9526da9aec630f9984a7d7c777ed8976be263a Mon Sep 17 00:00:00 2001 From: Ciprian Jichici Date: Wed, 31 Jan 2024 14:47:09 +0200 Subject: [PATCH 4/5] Vectorization triggering documentation --- docs/setup-guides/vectorization/index.md | 2 +- ...ectorization-monitoring-troubleshooting.md | 10 +++- .../vectorization/vectorization-profiles.md | 13 ++++- .../vectorization/vectorization-triggering.md | 57 +++++++++++++++---- 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/docs/setup-guides/vectorization/index.md b/docs/setup-guides/vectorization/index.md index 280925a8fa..92bb820b2a 100644 --- a/docs/setup-guides/vectorization/index.md +++ b/docs/setup-guides/vectorization/index.md @@ -8,6 +8,6 @@ Use the links below to learn more about configuring vectorization in Foundationa - [Configuring vectorization](vectorization-configuration.md) - [Managing vectorization profiles](vectorization-profiles.md) - [Triggering vectorization](vectorization-triggering.md) -- [Monitoring and troubleshooting vectorization](vectorization-monitoring.md) +- [Monitoring and troubleshooting vectorization](vectorization-monitoring-troubleshooting.md) If you are already familiar with these details, you can start working with the Vectorization API by importing the [FoundationaLLM Vectorization Postman collection.](./FoundationaLLM.Vectorization.API.postman_collection.json) Ensure to set the Vectorization API key (`X-API-KEY` header) on the **Authorization** tab of the collection and the Base URL (`baseUrl` collection variable) on the **Variables** tab. \ No newline at end of file diff --git a/docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md b/docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md index f46cd9f843..ca3ea3244a 100644 --- a/docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md +++ b/docs/setup-guides/vectorization/vectorization-monitoring-troubleshooting.md @@ -1 +1,9 @@ -# Monitoring and troubleshooting vectorization \ No newline at end of file +# Monitoring and troubleshooting vectorization + +The typical steps you have to perform when monitoring and troubleshooting vectorization in FoundationaLLM (FLLM) are: + +1. Check the configuration of the Vectorization API and Vectorization Worker. For more details, see [Configuring vectorization](vectorization-configuration.md). +2. Check the status of the Vectorization API and Vectorization Worker(s). Ensure the services have started and initialized successfully. +3. Check the logs of the Vectorization API and Vectorization Worker(s) for errors. By default, the logs are written to the Azure App Insights Log Analytics Workspace deployed by FLLM. +4. Check the definitions of the vectorization profiles used in the vectorization requests. For more details, see [Managing vectorization profiles](vectorization-profiles.md). Ensure all the required app configuration elements are present and have the correct values. +5. Check the state of the vectorization requests. By default, the vectorization requests are stored in the `vectorization-state` container of the Azure Storage account deployed by FLLM. \ No newline at end of file diff --git a/docs/setup-guides/vectorization/vectorization-profiles.md b/docs/setup-guides/vectorization/vectorization-profiles.md index c22dcf7e4b..43fc150675 100644 --- a/docs/setup-guides/vectorization/vectorization-profiles.md +++ b/docs/setup-guides/vectorization/vectorization-profiles.md @@ -84,6 +84,9 @@ The configuration parameters for `AzureSQLDatabase` are the following: ### Managing content source profiles +This section describes how to manage content source profiles using the Management API. +`{{baseUrl}}` is the base URL of the Management API. `{{instanceId}}` is the unique identifier of the FLLM instance. + **Retrieve** ``` @@ -152,6 +155,9 @@ The configuration parameters for `TokenTextSplitter` are the following: ### Managing text partitioning profiles +This section describes how to manage text partitioning profiles using the Management API. +`{{baseUrl}}` is the base URL of the Management API. `{{instanceId}}` is the unique identifier of the FLLM instance. + **Retrieve** ``` @@ -176,7 +182,6 @@ where `` is a JSON object with the structure describe HTTP DELETE {{baseUrl}}/instances/{{instanceId}}/providers/FoundationaLLM.Vectorization/textpartitioningprofiles/ ``` - ## Text embedding profiles The structure of a text embedding profile is the following: @@ -222,6 +227,9 @@ The configuration parameters for `SemanticKernelTextEmbedding` are the following ### Managing text embedding profiles +This section describes how to manage text embedding profiles using the Management API. +`{{baseUrl}}` is the base URL of the Management API. `{{instanceId}}` is the unique identifier of the FLLM instance. + **Retrieve** ``` @@ -292,6 +300,9 @@ The configuration parameters for `AzureAISearchIndexer` are the following: ### Managing indexing profiles +This section describes how to manage indexing profiles using the Management API. +`{{baseUrl}}` is the base URL of the Management API. `{{instanceId}}` is the unique identifier of the FLLM instance. + **Retrieve** ``` diff --git a/docs/setup-guides/vectorization/vectorization-triggering.md b/docs/setup-guides/vectorization/vectorization-triggering.md index fb8280411b..6cff1f7851 100644 --- a/docs/setup-guides/vectorization/vectorization-triggering.md +++ b/docs/setup-guides/vectorization/vectorization-triggering.md @@ -1,21 +1,30 @@ -# Triggering Vectorization +# Triggering vectorization +Vectorization pipelines are started when the Vectorization API receives a vectorization request. The following types of triggers are supported: -## Vectorization request +- None (no triggering of vectorization pipelines). +- Manual (vectorization pipelines are triggered manually by calling the Vectorization API). The typical use cases for on-demand vectorization (either synchronous or asynchronous) are testing, manual vectorization (or re-vectorization), and application integration (where another platform component triggers vectorization). +- Content-based (vectorization pipelines are triggered automatically when either new content is added to a content source or existing content is updated). +- Schedule-based (vectorization pipelines are triggered automatically based on a schedule). -Sample structure of a vectorization request: +> [!NOTE] +> Content-based and schedule-based triggering are currently in pre-release and are not yet available in public releases of FLLM. + +## Vectorization requests + +The typical structure of a vectorization request is the following ```json { - "id": "d4669c9c-e330-450a-a41c-a4d6649abdef", + "id": "", "content_identifier": { - "content_source_profile_name": "SDZWAJournals", + "content_source_profile_name": "", "multipart_id": [ - "https://fllmaks14sa.blob.core.windows.net", + "https://xyz.blob.core.windows.net", "vectorization-input", - "SDZWA-Journal-January-2024.pdf" + "The-fabulous-life-of-Jack-the-Cat.pdf" ], - "canonical_id": "sdzwa/journals/SDZWA-Journal-January-2024" + "canonical_id": "friends/stories/The-fabulous-life-of-Jack-the-Cat" }, "processing_type": "Asynchronous", "steps": [ @@ -38,7 +47,7 @@ Sample structure of a vectorization request: { "id": "index", "parameters": { - "indexing_profile_name": "AzureAISearch_Test_001" + "indexing_profile_name": "AzureAISearch_Default_001" } } ], @@ -52,4 +61,32 @@ Sample structure of a vectorization request: } ``` -The `processing_type` property can be one of `Asynchronous` or `Synchronous`. The `Asynchronous` value indicates that the vectorization request is processed asynchronously via the Vectorization workers. The `Synchronous` value indicates that the vectorization request is processed synchronously via the Vectorization API. +The following table describes the properties of a vectorization request. + +| Property | Description | +| --- | --- | +| `id` | A unique identifier for the vectorization request. The caller is responsible for the generation of this identifier. The recommended format is GUID. | +| `content_identifier` | The content identifier of the content to be vectorized. | +| `content_identifier.content_source_profile_name` | The name of the content source profile to be used for loading the content. | +| `content_identifier.multipart_id` | The multipart ID of the content to be vectorized. The multipart ID is a list of strings that uniquely identifies the content. The multipart ID is specific to the content source profile. | +| `content_identifier.canonical_id` | The canonical ID of the content to be vectorized. The canonical ID is a string that uniquely identifies the content in a logical namespace. The caller is responsible for the generation of this identifier. The identifier should have a path form (using the `/` separator). The last part of the path should always be equal to the file name (without its extension). | +| `processing_type` | The type of processing to be performed. The following values are supported: `Synchronous` and `Asynchronous`. See [Vectorization concepts](./vectorization-concepts.md) for more details. | +| `steps` | The vectorization steps to be executed. Most vectorization requests will contain the full set of standard steps: `extract`, `partition`, `embed`, and `index`. Each step (except for the `extract` one) will contain one parameter specifying the name of the associated vectorization profile name. | +| `completed_steps` | Must always be an empty array. | +| `remaining_steps` | Must always be an array with the step names to be executed (order sensitive). | + +### Submitting vectorization requests + +This section describes how to submit vectorization requests using the Vectorization API. +`{{baseUrl}}` is the base URL of the Vectorization API. + +``` +POST {{baseUrl}}/vectorizationrequest +Content-Type: application/json +X-API-KEY: + +BODY + +``` + +where is the API key of the Vectorization API and `` is the vectorization request to be submitted. From 56488602563c3a4153aafc9ce0f59043ecbc8608 Mon Sep 17 00:00:00 2001 From: Ciprian Jichici Date: Wed, 31 Jan 2024 15:14:51 +0200 Subject: [PATCH 5/5] Fix issue with TOC in docs --- docs/toc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/toc.yml b/docs/toc.yml index 9b36dcd4d7..c07217de9a 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -66,7 +66,7 @@ items: href: setup-guides/vectorization/vectorization-profiles.md - name: Triggering vectorization href: setup-guides/vectorization/vectorization-triggering.md - - name: Vectorization monitoring and troubleshooting + - name: Monitoring and troubleshooting href: setup-guides/vectorization/vectorization-monitoring-troubleshooting.md - name: Operations href: operations/index.md