Merge pull request #512 from solliancenet/cj-synchronous-vectorization

Support for synchronous vectorization

src/dotnet/Common/Constants/DependencyInjectionKeys.cs

@@ -40,5 +40,15 @@ public static class DependencyInjectionKeys
         /// The dependency injection key for the Azure AI Search indexing service.
         /// </summary>
         public const string FoundationaLLM_Vectorization_AzureAISearchIndexingService = "FoundationaLLM:Vectorization:AzureAISearchIndexingService";
+
+        /// <summary>
+        /// The dependency injection key for the vectorization queues configuration section.
+        /// </summary>
+        public const string FoundationaLLM_Vectorization_Queues = "FoundationaLLM:Vectorization:Queues";
+
+        /// <summary>
+        /// The dependency injection key for the vectorization steps configuration section.
+        /// </summary>
+        public const string FoundationaLLM_Vectorization_Steps = "FoundationaLLM:Vectorization:Steps";
     }
 }

src/dotnet/Vectorization/Interfaces/IVectorizationService.cs

@@ -17,6 +17,6 @@ public interface IVectorizationService
         /// </summary>
         /// <param name="vectorizationRequest">The <see cref="VectorizationRequest"/> object containing the details of the vectorization request.</param>
         /// <returns></returns>
-        Task ProcessRequest(VectorizationRequest vectorizationRequest);
+        Task<VectorizationProcessingResult> ProcessRequest(VectorizationRequest vectorizationRequest);
     }
 }

src/dotnet/Vectorization/Models/VectorizationProcessingResult.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FoundationaLLM.Vectorization.Models
+{
+    /// <summary>
+    /// Represents the result of processing a vectorization request.
+    /// </summary>
+    /// <param name="IsSuccess">Indicates whether the processing was completed successfully.</param>
+    /// <param name="OperationId">The identifier of the vectorization operation. Can be used to request the status of the operation.</param>
+    /// <param name="ErrorMessage">When IsSuccess is false, contains an error message with details.</param>
+    public record VectorizationProcessingResult(
+        bool IsSuccess,
+        Guid? OperationId,
+        string? ErrorMessage)
+    {
+    }
+}

src/dotnet/Vectorization/Models/VectorizationProcessingType.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace FoundationaLLM.Vectorization.Models
+{
+    /// <summary>
+    /// The type of vectorization request processing.
+    /// </summary>
+    public enum VectorizationProcessingType
+    {
+        /// <summary>
+        /// Asynchronous processing using vectorization workers.
+        /// </summary>
+        Asynchronous,
+
+        /// <summary>
+        /// Synchronous processing using the vectorization API.
+        /// </summary>
+        Synchronous
+    }
+}

src/dotnet/Vectorization/Models/VectorizationRequest.cs

@@ -24,6 +24,14 @@ public class VectorizationRequest
         [JsonPropertyName("content_identifier")]
         public required ContentIdentifier ContentIdentifier { get; set; }
 
+        /// <summary>
+        /// The <see cref="VectorizationProcessingType"/> indicating how should the request be processed.
+        /// </summary>
+        [JsonPropertyOrder(2)]
+        [JsonPropertyName("processing_type")]
+        [JsonConverter(typeof(JsonStringEnumConverter))]
+        public required VectorizationProcessingType ProcessingType { get; set; }
+
         /// <summary>
         /// The list of vectorization steps requested by the vectorization request.
         /// Vectorization steps are identified by unique names like "extract", "partition", "embed", "index", etc.

src/dotnet/Vectorization/Services/RequestSources/RequestSourcesCache.cs

@@ -1,6 +1,8 @@
-using FoundationaLLM.Vectorization.Interfaces;
+using FoundationaLLM.Common.Constants;
+using FoundationaLLM.Vectorization.Interfaces;
 using FoundationaLLM.Vectorization.Models.Configuration;
 using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
@@ -17,7 +19,7 @@ namespace FoundationaLLM.Vectorization.Services.RequestSources
     /// <param name="loggerFactory">The <see cref="ILoggerFactory"/> used to create new loggers for child objects.</param>
     public class RequestSourcesCache(
         IOptions<VectorizationWorkerSettings> vectorizationWorkerOptions,
-        IConfigurationSection queuesConfiguration,
+        [FromKeyedServices(DependencyInjectionKeys.FoundationaLLM_Vectorization_Queues)] IConfigurationSection queuesConfiguration,
         ILoggerFactory loggerFactory) : IRequestSourcesCache
     {
         private readonly Dictionary<string, IRequestSourceService> _requestSources = (new RequestSourcesBuilder())

src/dotnet/Vectorization/Services/VectorizationService.cs

@@ -1,7 +1,14 @@
-using FoundationaLLM.Vectorization.Exceptions;
+using FoundationaLLM.Common.Constants;
+using FoundationaLLM.Common.Models.Chat;
+using FoundationaLLM.Vectorization.Exceptions;
+using FoundationaLLM.Vectorization.Handlers;
 using FoundationaLLM.Vectorization.Interfaces;
 using FoundationaLLM.Vectorization.Models;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using System.Runtime;
+using System.Threading;
 
 namespace FoundationaLLM.Vectorization.Services
 {
@@ -12,21 +19,47 @@ namespace FoundationaLLM.Vectorization.Services
     /// Creates a new instance of the <see cref="VectorizationService"/> service.
     /// </remarks>
     /// <param name="requestSourcesCache">The <see cref="IRequestSourcesCache"/> cache of request sources.</param>
-    /// <param name="logger">The logger instance used for logging.</param>
+    /// <param name="vectorizationStateService">The service providing vectorization state management.</param>
+    /// <param name="stepsConfiguration">The <see cref="IConfigurationSection"/> object providing access to the settings.</param>
+    /// <param name="serviceProvider">The <see cref="IServiceProvider"/> implemented by the dependency injection container.</param>
+    /// <param name="loggerFactory">The logger factory used to create loggers.</param>
     public class VectorizationService(
         IRequestSourcesCache requestSourcesCache,
-        ILogger<VectorizationService> logger) : IVectorizationService
+        IVectorizationStateService vectorizationStateService,
+        [FromKeyedServices(DependencyInjectionKeys.FoundationaLLM_Vectorization_Steps)] IConfigurationSection stepsConfiguration,
+        IServiceProvider serviceProvider,
+        ILoggerFactory loggerFactory) : IVectorizationService
     {
         private readonly Dictionary<string, IRequestSourceService> _requestSources = requestSourcesCache.RequestSources;
-        private readonly ILogger<VectorizationService> _logger = logger;
+        private readonly IVectorizationStateService _vectorizationStateService = vectorizationStateService;
+        private readonly IConfigurationSection? _stepsConfiguration = stepsConfiguration;
+        private readonly IServiceProvider _serviceProvider = serviceProvider;
+        private readonly ILoggerFactory _loggerFactory = loggerFactory;
+        private readonly ILogger<VectorizationService> _logger = loggerFactory.CreateLogger<VectorizationService>();
 
         /// <inheritdoc/>
-        public async Task ProcessRequest(VectorizationRequest vectorizationRequest)
+        public async Task<VectorizationProcessingResult> ProcessRequest(VectorizationRequest vectorizationRequest)
         {
-            ValidateRequest(vectorizationRequest);
+            try
+            {
+                ValidateRequest(vectorizationRequest);
 
-            var firstRequestSource = _requestSources[vectorizationRequest.Steps.First().Id];
-            await firstRequestSource.SubmitRequest(vectorizationRequest);
+                switch (vectorizationRequest.ProcessingType)
+                {
+                    case VectorizationProcessingType.Asynchronous:
+                        var firstRequestSource = _requestSources[vectorizationRequest.Steps.First().Id];
+                        await firstRequestSource.SubmitRequest(vectorizationRequest);
+                        return new VectorizationProcessingResult(true, null, null);
+                    case VectorizationProcessingType.Synchronous:
+                        return await ProcessRequestInternal(vectorizationRequest);
+                    default:
+                        throw new VectorizationException($"The vectorization processing type {vectorizationRequest.ProcessingType} is not supported.");
+                }
+            }
+            catch (Exception ex)
+            {
+                return new VectorizationProcessingResult(false, null, ex.Message);
+            }
         }
 
         private void ValidateRequest(VectorizationRequest vectorizationRequest)
@@ -60,5 +93,51 @@ private void HandleValidationError(string validationError)
             _logger.LogError(validationError);
             throw new VectorizationException(validationError);
         }
+
+        private async Task<VectorizationProcessingResult> ProcessRequestInternal(VectorizationRequest request)
+        {
+            _logger.LogInformation("Starting synchronous processing for request {RequestId}.", request.Id);
+
+            var state = VectorizationState.FromRequest(request);
+
+            foreach (var step in request.Steps)
+            {
+                _logger.LogInformation("Starting step [{Step}] for request {RequestId}.", step.Id, request.Id);
+
+                var stepHandler = VectorizationStepHandlerFactory.Create(
+                    step.Id,
+                    "N/A",
+                    step.Parameters,
+                    _stepsConfiguration,
+                    _vectorizationStateService,
+                    _serviceProvider,
+                    _loggerFactory);
+                var handlerSuccess = await stepHandler.Invoke(request, state, default).ConfigureAwait(false);
+                if (!handlerSuccess)
+                    break;
+
+                var steps = request.MoveToNextStep();
+
+                if (!string.IsNullOrEmpty(steps.CurrentStep))
+                    _logger.LogInformation("The pipeline for request id {RequestId} was advanced from step [{PreviousStepName}] to step [{CurrentStepName}].",
+                        request.Id, steps.PreviousStep, steps.CurrentStep);
+                else
+                    _logger.LogInformation("The pipeline for request id {RequestId} was advanced from step [{PreviousStepName}] to finalized state.",
+                        request.Id, steps.PreviousStep);
+            }
+
+            if (request.Complete)
+            {
+                _logger.LogInformation("Finished synchronous processing for request {RequestId}. All steps were processed successfully.", request.Id);
+                return new VectorizationProcessingResult(true, null, null);
+            }
+            else
+            {
+                var errorMessage =
+                    $"Execution stopped at step [{request.CurrentStep}] due to an error.";
+                _logger.LogInformation("Finished synchronous processing for request {RequestId}. {ErrorMessage}", request.Id, errorMessage);
+                return new VectorizationProcessingResult(false, null, errorMessage);
+            }
+        }
     }
 }

src/dotnet/Vectorization/Services/VectorizationStates/MemoryVectorizationStateService.cs

@@ -33,11 +33,8 @@ public async Task<VectorizationState> ReadState(VectorizationRequest request)
         }
 
         /// <inheritdoc/>
-        public async Task LoadArtifacts(VectorizationState state, VectorizationArtifactType artifactType)
-        {
+        public async Task LoadArtifacts(VectorizationState state, VectorizationArtifactType artifactType) =>
             await Task.CompletedTask;
-            throw new NotImplementedException();
-        }
 
         /// <inheritdoc/>
         public async Task SaveState(VectorizationState state)

src/dotnet/VectorizationAPI/Controllers/VectorizationRequestController.cs

@@ -28,7 +28,7 @@ public class VectorizationRequestController(
         /// <param name="vectorizationRequest"></param>
         /// <returns></returns>
         [HttpPost]
-        public async Task ProcessRequest([FromBody] VectorizationRequest vectorizationRequest) =>
-            await _vectorizationService.ProcessRequest(vectorizationRequest);
+        public async Task<IActionResult> ProcessRequest([FromBody] VectorizationRequest vectorizationRequest) =>
+            new OkObjectResult(await _vectorizationService.ProcessRequest(vectorizationRequest));
     }
 }

src/dotnet/VectorizationAPI/Program.cs

@@ -4,11 +4,21 @@
 using FoundationaLLM.Common.Constants;
 using FoundationaLLM.Common.Interfaces;
 using FoundationaLLM.Common.OpenAPI;
+using FoundationaLLM.Common.Services.Tokenizers;
+using FoundationaLLM.Common.Services;
+using FoundationaLLM.Common.Settings;
+using FoundationaLLM.SemanticKernel.Core.Models.Configuration;
+using FoundationaLLM.SemanticKernel.Core.Services;
 using FoundationaLLM.Vectorization.Interfaces;
 using FoundationaLLM.Vectorization.Models.Configuration;
+using FoundationaLLM.Vectorization.ResourceProviders;
 using FoundationaLLM.Vectorization.Services;
+using FoundationaLLM.Vectorization.Services.ContentSources;
 using FoundationaLLM.Vectorization.Services.RequestSources;
+using FoundationaLLM.Vectorization.Services.Text;
+using FoundationaLLM.Vectorization.Services.VectorizationStates;
 using Microsoft.ApplicationInsights.AspNetCore.Extensions;
+using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -52,20 +62,75 @@
 builder.Services.AddOptions<VectorizationWorkerSettings>()
     .Bind(builder.Configuration.GetSection(AppConfigurationKeys.FoundationaLLM_Vectorization_VectorizationWorker));
 
-builder.Services.AddSingleton(
+builder.Services.AddOptions<BlobStorageServiceSettings>(
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_ResourceProviderService)
+    .Bind(builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_ResourceProviderService_Storage));
+
+builder.Services.AddOptions<SemanticKernelTextEmbeddingServiceSettings>()
+    .Bind(builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_SemanticKernelTextEmbeddingService));
+
+builder.Services.AddOptions<AzureAISearchIndexingServiceSettings>()
+    .Bind(builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_AzureAISearchIndexingService));
+
+builder.Services.AddKeyedSingleton(
     typeof(IConfigurationSection),
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_Queues,
     builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_Queues));
 
+builder.Services.AddKeyedSingleton(
+    typeof(IConfigurationSection),
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_Steps,
+    builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_Steps));
+
 // Add services to the container.
 
-builder.Services.AddTransient<IAPIKeyValidationService, APIKeyValidationService>();
-builder.Services.AddScoped<IVectorizationService, VectorizationService>();
-builder.Services.AddSingleton<IRequestSourcesCache, RequestSourcesCache>();
+builder.Services.AddKeyedSingleton<IStorageService, BlobStorageService>(
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_ResourceProviderService, (sp, obj) =>
+    {
+        var settings = sp.GetRequiredService<IOptionsMonitor<BlobStorageServiceSettings>>()
+            .Get(DependencyInjectionKeys.FoundationaLLM_Vectorization_ResourceProviderService);
+        var logger = sp.GetRequiredService<ILogger<BlobStorageService>>();
+
+        return new BlobStorageService(
+            Options.Create<BlobStorageServiceSettings>(settings),
+            logger);
+    });
+
+// Vectorization state
+builder.Services.AddSingleton<IVectorizationStateService, MemoryVectorizationStateService>();
+
+// Vectorization resource provider
+builder.Services.AddKeyedSingleton<IResourceProviderService, VectorizationResourceProviderService>(
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_ResourceProviderService);
+builder.Services.ActivateKeyedSingleton<IResourceProviderService>(
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_ResourceProviderService);
+
+// Service factories
+builder.Services.AddSingleton<IVectorizationServiceFactory<IContentSourceService>, ContentSourceServiceFactory>();
+builder.Services.AddSingleton<IVectorizationServiceFactory<ITextSplitterService>, TextSplitterServiceFactory>();
+builder.Services.AddSingleton<IVectorizationServiceFactory<ITextEmbeddingService>, TextEmbeddingServiceFactory>();
+builder.Services.AddSingleton<IVectorizationServiceFactory<IIndexingService>, IndexingServiceFactory>();
+
+// Tokenizer
+builder.Services.AddKeyedSingleton<ITokenizerService, MicrosoftBPETokenizerService>(TokenizerServiceNames.MICROSOFT_BPE_TOKENIZER);
+builder.Services.ActivateKeyedSingleton<ITokenizerService>(TokenizerServiceNames.MICROSOFT_BPE_TOKENIZER);
 
-// Activate singleton services
+// Text embedding
+builder.Services.AddKeyedSingleton<ITextEmbeddingService, SemanticKernelTextEmbeddingService>(
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_SemanticKernelTextEmbeddingService);
 
+// Indexing
+builder.Services.AddKeyedSingleton<IIndexingService, AzureAISearchIndexingService>(
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_AzureAISearchIndexingService);
+
+// Request sources cache
+builder.Services.AddSingleton<IRequestSourcesCache, RequestSourcesCache>();
 builder.Services.ActivateSingleton<IRequestSourcesCache>();
 
+// Vectorization
+builder.Services.AddScoped<IVectorizationService, VectorizationService>();
+
+builder.Services.AddTransient<IAPIKeyValidationService, APIKeyValidationService>();
 builder.Services.AddControllers();
 
 // Add API Key Authorization

src/dotnet/VectorizationWorker/Program.cs

@@ -72,12 +72,15 @@
 builder.Services.AddOptions<AzureAISearchIndexingServiceSettings>()
     .Bind(builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_AzureAISearchIndexingService));
 
-builder.Services.AddSingleton(
-    typeof(IEnumerable<IConfigurationSection>),
-    new IConfigurationSection[] {
-        builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_Queues),
-        builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_Steps)
-    });
+builder.Services.AddKeyedSingleton(
+    typeof(IConfigurationSection),
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_Queues,
+    builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_Queues));
+
+builder.Services.AddKeyedSingleton(
+    typeof(IConfigurationSection),
+    DependencyInjectionKeys.FoundationaLLM_Vectorization_Steps,
+    builder.Configuration.GetSection(AppConfigurationKeySections.FoundationaLLM_Vectorization_Steps));
 
 // Add services to the container.