diff --git a/docs/release-notes/breaking-changes.md b/docs/release-notes/breaking-changes.md index 2fbcd9ba3d..82a2146ec5 100644 --- a/docs/release-notes/breaking-changes.md +++ b/docs/release-notes/breaking-changes.md @@ -3,6 +3,18 @@ > [!NOTE] > This section is for changes that are not yet released but will affect future releases. +## Starting with 0.9.1 + +Prompt prefixes and suffixes support FoundationaLLM variables for dynamic replacement at runtime. The variable format is `{{foundationallm:variable_name[:format]}}` where +- `variable_name` is the name of the well-known variable. +- `format` is the optional formatting applied to the value of the variable. + +The following variables are supported: + +| Name | Value | Example +| --- | --- | --- | +| `current_datetime_utc` | The current UTC date and time. | `The current date is {{foundationallm:current_datetime_utc:dddd, MMMM dd, yyyy}}. This looks great.` -> `The current date is Sunday, December 15, 2024. This looks great.` + ## Starting with 0.9.0 ### Configuration changes diff --git a/src/dotnet/Common/Constants/Templates/TemplateVariables.cs b/src/dotnet/Common/Constants/Templates/TemplateVariables.cs new file mode 100644 index 0000000000..6e016c89b3 --- /dev/null +++ b/src/dotnet/Common/Constants/Templates/TemplateVariables.cs @@ -0,0 +1,13 @@ +namespace FoundationaLLM.Common.Constants.Templates +{ + /// + /// Provides template variables that can be used in templates. + /// + public static class TemplateVariables + { + /// + /// Token for current date in UTC format. + /// + public const string CurrentDateTimeUTC = "current_datetime_utc"; + } +} diff --git a/src/dotnet/Common/Interfaces/ITemplatingService.cs b/src/dotnet/Common/Interfaces/ITemplatingService.cs new file mode 100644 index 0000000000..ae51731269 --- /dev/null +++ b/src/dotnet/Common/Interfaces/ITemplatingService.cs @@ -0,0 +1,15 @@ +namespace FoundationaLLM.Common.Interfaces +{ + /// + /// Defines the interface for a templating engine. + /// + public interface ITemplatingService + { + /// + /// Transforms the input string by replacing tokens with the corresponding values. + /// + /// The input string to be transformed. + /// The transformed string where all the valid tokens have been replaced. + string Transform(string s); + } +} diff --git a/src/dotnet/Common/Services/DependencyInjection.cs b/src/dotnet/Common/Services/DependencyInjection.cs index 0eba9c221f..a85a404270 100644 --- a/src/dotnet/Common/Services/DependencyInjection.cs +++ b/src/dotnet/Common/Services/DependencyInjection.cs @@ -10,6 +10,7 @@ using FoundationaLLM.Common.Services.API; using FoundationaLLM.Common.Services.Azure; using FoundationaLLM.Common.Services.Security; +using FoundationaLLM.Common.Services.Templates; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.Azure.Cosmos; using Microsoft.Azure.Cosmos.Fluent; @@ -297,5 +298,19 @@ public static void AddAzureCosmosDBService(this IServiceCollection services, ICo services.AddSingleton(); } + + /// + /// Registers the implementation with the dependency injection container. + /// + /// The application builder managing the dependency injection container. + public static void AddRegexTemplatingEngine(this IHostApplicationBuilder builder) => + builder.Services.AddSingleton(); + + /// + /// Registers the implementation with the dependency injection container. + /// + /// The dependency injection container service collection. + public static void AddRegexTemplatingEngine(this IServiceCollection services) => + services.AddSingleton(); } } diff --git a/src/dotnet/Common/Services/Templates/RegexTemplatingService.cs b/src/dotnet/Common/Services/Templates/RegexTemplatingService.cs new file mode 100644 index 0000000000..0989b6b3a0 --- /dev/null +++ b/src/dotnet/Common/Services/Templates/RegexTemplatingService.cs @@ -0,0 +1,78 @@ +using FoundationaLLM.Common.Constants.Templates; +using FoundationaLLM.Common.Interfaces; +using Microsoft.Extensions.Logging; +using System.Text.RegularExpressions; + +namespace FoundationaLLM.Common.Services.Templates +{ + /// + /// Templating engine that uses regular expressions to replace tokens in strings. + /// + /// The logger used for logging. + public partial class RegexTemplatingService( + ILogger logger) : ITemplatingService + { + /// + /// Regular expression pattern for template variables. + /// + private const string REGEX_VARIABLE_PATTERN = "\\{\\{foundationallm:(.*?)\\}\\}"; + + private readonly ILogger _logger = logger; + + [GeneratedRegex(REGEX_VARIABLE_PATTERN, RegexOptions.Compiled)] + private static partial Regex VariableRegex(); + + /// + public string Transform(string s) + { + if (string.IsNullOrWhiteSpace(s)) + { + return string.Empty; + } + + try + { + // Expects the format {{foundationallm:variable_name[:format]}} + + var matches = VariableRegex().Matches(s); + Dictionary replacements = []; + + foreach (Match match in matches) + { + var matchedVariable = match.Value; + + var variableTokens = match.Groups[1].Value.Split(":", 2); + var variableName = variableTokens[0]; + var variableFormat = variableTokens.Length > 1 ? variableTokens[1] : null; + + switch (variableName) + { + case TemplateVariables.CurrentDateTimeUTC: + replacements.Add( + matchedVariable, + string.IsNullOrWhiteSpace(variableFormat) + ? DateTime.UtcNow.ToString() + : DateTime.UtcNow.ToString(variableFormat)); + break; + default: + break; + } + } + + var transformedString = s; + foreach (var replacement in replacements) + { + transformedString = transformedString.Replace(replacement.Key, replacement.Value); + } + + return transformedString; + } + catch (Exception ex) + { + _logger.LogError(ex, "Error occurred while transforming the string."); + } + + return s; + } + } +} diff --git a/src/dotnet/Orchestration/Orchestration/OrchestrationBuilder.cs b/src/dotnet/Orchestration/Orchestration/OrchestrationBuilder.cs index 26e2333a0a..a73ef25d26 100644 --- a/src/dotnet/Orchestration/Orchestration/OrchestrationBuilder.cs +++ b/src/dotnet/Orchestration/Orchestration/OrchestrationBuilder.cs @@ -40,6 +40,7 @@ public class OrchestrationBuilder /// A dictionary of resource providers hashed by resource provider name. /// The that manages internal and external orchestration services. /// The used to interact with the Cosmos DB database. + /// The used to render templates. /// The provding dependency injection services for the current scope. /// The logger factory used to create new loggers. /// @@ -53,6 +54,7 @@ public class OrchestrationBuilder Dictionary resourceProviderServices, ILLMOrchestrationServiceManager llmOrchestrationServiceManager, IAzureCosmosDBService cosmosDBService, + ITemplatingService templatingService, IServiceProvider serviceProvider, ILoggerFactory loggerFactory) { @@ -64,6 +66,7 @@ public class OrchestrationBuilder originalRequest.SessionId, originalRequest.Settings?.ModelParameters, resourceProviderServices, + templatingService, callContext.CurrentUserIdentity!, logger); @@ -176,6 +179,7 @@ await cosmosDBService.PatchOperationsItemPropertiesAsync? modelParameterOverrides, Dictionary resourceProviderServices, + ITemplatingService templatingService, UnifiedUserIdentity currentUserIdentity, ILogger logger) { @@ -256,6 +260,17 @@ await cosmosDBService.PatchOperationsItemPropertiesAsync( resourceObjectId.ObjectId, currentUserIdentity); + + if (retrievedPrompt is MultipartPrompt multipartPrompt) + { + //check for token replacements, multipartPrompt variable has the same reference as retrievedPrompt therefore this edits the prefix/suffix in place + if (multipartPrompt is not null) + { + + multipartPrompt.Prefix = templatingService.Transform(multipartPrompt.Prefix!); + multipartPrompt.Suffix = templatingService.Transform(multipartPrompt.Suffix!); + } + } explodedObjectsManager.TryAdd( retrievedPrompt.ObjectId!, retrievedPrompt); diff --git a/src/dotnet/Orchestration/Services/OrchestrationService.cs b/src/dotnet/Orchestration/Services/OrchestrationService.cs index 88f6bbc382..a24d84adf7 100644 --- a/src/dotnet/Orchestration/Services/OrchestrationService.cs +++ b/src/dotnet/Orchestration/Services/OrchestrationService.cs @@ -25,6 +25,7 @@ public class OrchestrationService : IOrchestrationService { private readonly ILLMOrchestrationServiceManager _llmOrchestrationServiceManager; private readonly IAzureCosmosDBService _cosmosDBService; + private readonly ITemplatingService _templatingService; private readonly ICallContext _callContext; private readonly IConfiguration _configuration; private readonly ILogger _logger; @@ -39,6 +40,7 @@ public class OrchestrationService : IOrchestrationService /// A list of resource providers hashed by resource provider name. /// The managing the internal and external LLM orchestration services. /// The used to interact with the Cosmos DB database. + /// The used to render templates. /// The call context of the request being handled. /// The used to retrieve app settings from configuration. /// The provding dependency injection services for the current scope. @@ -47,6 +49,7 @@ public OrchestrationService( IEnumerable resourceProviderServices, ILLMOrchestrationServiceManager llmOrchestrationServiceManager, IAzureCosmosDBService cosmosDBService, + ITemplatingService templatingService, ICallContext callContext, IConfiguration configuration, IServiceProvider serviceProvider, @@ -56,6 +59,7 @@ public OrchestrationService( rps => rps.Name); _llmOrchestrationServiceManager = llmOrchestrationServiceManager; _cosmosDBService = cosmosDBService; + _templatingService = templatingService; _callContext = callContext; _configuration = configuration; @@ -100,6 +104,7 @@ public async Task GetCompletion(string instanceId, Completio _resourceProviderServices, _llmOrchestrationServiceManager, _cosmosDBService, + _templatingService, _serviceProvider, _loggerFactory) ?? throw new OrchestrationException($"The orchestration builder was not able to create an orchestration for agent [{completionRequest.AgentName ?? string.Empty}]."); @@ -139,6 +144,7 @@ public async Task StartCompletionOperation(string instance _resourceProviderServices, _llmOrchestrationServiceManager, _cosmosDBService, + _templatingService, _serviceProvider, _loggerFactory) ?? throw new OrchestrationException($"The orchestration builder was not able to create an orchestration for agent [{completionRequest.AgentName ?? string.Empty}]."); @@ -225,6 +231,7 @@ private async Task GetCompletionForAgentConversation( _resourceProviderServices, _llmOrchestrationServiceManager, _cosmosDBService, + _templatingService, _serviceProvider, _loggerFactory); diff --git a/src/dotnet/OrchestrationAPI/Program.cs b/src/dotnet/OrchestrationAPI/Program.cs index 98ea442628..47d8d015db 100644 --- a/src/dotnet/OrchestrationAPI/Program.cs +++ b/src/dotnet/OrchestrationAPI/Program.cs @@ -117,6 +117,9 @@ public static void Main(string[] args) builder.AddGroupMembership(); builder.AddAuthorizationServiceClient(); + // Add the templating engine. + builder.AddRegexTemplatingEngine(); + //---------------------------- // Resource providers //---------------------------- diff --git a/tests/dotnet/Core.Tests/Services/CoreServiceTests.cs b/tests/dotnet/Core.Tests/Services/CoreServiceTests.cs index ca37ed593b..a78917aba0 100644 --- a/tests/dotnet/Core.Tests/Services/CoreServiceTests.cs +++ b/tests/dotnet/Core.Tests/Services/CoreServiceTests.cs @@ -1,9 +1,10 @@ using FoundationaLLM.Common.Constants; using FoundationaLLM.Common.Interfaces; using FoundationaLLM.Common.Models.Authentication; -using FoundationaLLM.Common.Models.Chat; using FoundationaLLM.Common.Models.Configuration.Branding; +using FoundationaLLM.Common.Models.Conversation; using FoundationaLLM.Common.Models.Orchestration.Request; +using FoundationaLLM.Common.Models.Orchestration.Response; using FoundationaLLM.Core.Interfaces; using FoundationaLLM.Core.Models.Configuration; using FoundationaLLM.Core.Services; @@ -45,7 +46,8 @@ public CoreServiceTests() _options = Options.Create(new CoreServiceSettings { BypassGatekeeper = true, - SessionSummarization = ChatSessionNameSummarizationType.LLM + SessionSummarization = ChatSessionNameSummarizationType.LLM, + AzureOpenAIAssistantsFileSearchFileExtensions = "" }); _brandingConfig.Value.Returns(new ClientBrandingConfiguration()); @@ -55,7 +57,7 @@ public CoreServiceTests() UPN = "test@foundationallm.ai", Username = "test@foundationallm.ai" }); - _testedService = new CoreService(_cosmosDbService, _downstreamAPIServices, _logger, _brandingConfig, _options, _callContext, _resourceProviderServices); + _testedService = new CoreService(_cosmosDbService, _downstreamAPIServices, _logger, _brandingConfig, _options, _callContext, _resourceProviderServices, null, null); } #region GetAllChatSessionsAsync @@ -64,7 +66,7 @@ public CoreServiceTests() public async Task GetAllChatSessionsAsync_ShouldReturnAllChatSessions() { // Arrange - var expectedSessions = new List() { new Session() }; + var expectedSessions = new List() { }; _cosmosDbService.GetConversationsAsync(Arg.Any(), Arg.Any()).Returns(expectedSessions); // Act @@ -83,7 +85,7 @@ public async Task GetChatSessionMessagesAsync_ShouldReturnAllChatSessionMessages { // Arrange string sessionId = Guid.NewGuid().ToString(); - var message = new Message(sessionId, "sender", 0, "text", null, null, "test_upn"); + var message = new Message(); var expectedMessages = new List() { message }; _cosmosDbService.GetSessionMessagesAsync(sessionId, Arg.Any()) @@ -121,16 +123,16 @@ public async Task CreateNewChatSessionAsync_ShouldReturnANewChatSession() var currentUserUPN = "testuser@example.com"; var sessionType = "Test_type"; var chatSessionProperties = new ChatSessionProperties() { Name = "Test_name" }; - var newSession = new Session { Name = chatSessionProperties.Name, Type = sessionType, UPN = currentUserUPN }; + var newSession = new Conversation() { Name = chatSessionProperties.Name, Type = sessionType, UPN = currentUserUPN, SessionId = "" }; // Set up mock returns _callContext.CurrentUserIdentity.Returns(new UnifiedUserIdentity { UPN = currentUserUPN }); - _cosmosDbService.InsertSessionAsync(Arg.Any()) - .Returns(Task.FromResult(newSession)); + //_cosmosDbService.In(Arg.Any()) + // .Returns(Task.FromResult(newSession)); // Act - var resultSession = await _testedService.CreateNewChatSessionAsync(_instanceId, chatSessionProperties); + var resultSession = await _testedService.CreateConversationAsync(_instanceId, chatSessionProperties); // Assert Assert.NotNull(resultSession); @@ -147,10 +149,10 @@ public async Task CreateNewChatSessionAsync_ShouldReturnANewChatSession() public async Task RenameChatSessionAsync_ShouldReturnTheRenamedChatSession() { // Arrange - var session = new Session() { Name = "OldName" }; + var session = new Conversation() { Name = "OldName", SessionId = "" }; var chatSessionProperties = new ChatSessionProperties() { Name = "NewName" }; - var expectedSession = new Session() + var expectedSession = new Conversation() { Id = session.Id, Messages = session.Messages, @@ -159,10 +161,10 @@ public async Task RenameChatSessionAsync_ShouldReturnTheRenamedChatSession() TokensUsed = session.TokensUsed, Type = session.Type, }; - _cosmosDbService.UpdateSessionNameAsync(session.Id, chatSessionProperties.Name).Returns(expectedSession); + //_cosmosDbService.CreateOrUpdateConversationAsync(session.Id, chatSessionProperties.Name).Returns(expectedSession); // Act - var actualSession = await _testedService.RenameChatSessionAsync(_instanceId, session.Id, chatSessionProperties); + var actualSession = await _testedService.RenameConversationAsync(_instanceId, session.Id, chatSessionProperties); // Assert Assert.Equivalent(expectedSession, actualSession); @@ -178,7 +180,7 @@ public async Task RenameChatSessionAsync_ShouldThrowExceptionWhenSessionIdIsNull // Assert await Assert.ThrowsAsync((Func)(async () => { - await _testedService.RenameChatSessionAsync(_instanceId, null!, chatSessionProperties); + await _testedService.RenameConversationAsync(_instanceId, null!, chatSessionProperties); })); } @@ -196,7 +198,7 @@ await Assert.ThrowsAsync(async () => await Assert.ThrowsAsync(async () => { - await _testedService.RenameChatSessionAsync(_instanceId, sessionId, new ChatSessionProperties() { Name = string.Empty }); + await _testedService.RenameConversationAsync(_instanceId, sessionId, new ChatSessionProperties() { Name = string.Empty }); }); } @@ -243,7 +245,7 @@ public async Task GetChatCompletionAsync_ShouldReturnACompletion() var userPrompt = "Prompt"; var orchestrationRequest = new CompletionRequest { SessionId = sessionId, UserPrompt = userPrompt }; var upn = "test@foundationallm.ai"; - var expectedCompletion = new Completion() { Text = "Completion" }; + var expectedCompletion = new Message() { Text = "Completion" }; var expectedMessages = new List(); _cosmosDbService.GetSessionMessagesAsync(sessionId, upn).Returns(expectedMessages); @@ -251,7 +253,7 @@ public async Task GetChatCompletionAsync_ShouldReturnACompletion() var completionResponse = new CompletionResponse() { Completion = "Completion" }; _downstreamAPIServices.Last().GetCompletion(_instanceId, Arg.Any()).Returns(completionResponse); - _cosmosDbService.GetConversationAsync(sessionId).Returns(new Session()); + _cosmosDbService.GetConversationAsync(sessionId).Returns(new Conversation() { Name = "", SessionId = "" }); _cosmosDbService.UpsertSessionBatchAsync().Returns(Task.CompletedTask); // Act @@ -267,7 +269,7 @@ public async Task GetChatCompletionAsync_ShouldReturnAnErrorMessageWhenSessionId // Arrange var userPrompt = "Prompt"; var orchestrationRequest = new CompletionRequest { UserPrompt = userPrompt }; - var expectedCompletion = new Completion { Text = "Could not generate a completion due to an internal error." }; + var expectedCompletion = new Message { Text = "Could not generate a completion due to an internal error." }; // Act var actualCompletion = await _testedService.GetChatCompletionAsync(_instanceId, orchestrationRequest); @@ -318,14 +320,14 @@ public async Task RateMessageAsync_ShouldReturnARatedMessage() var id = Guid.NewGuid().ToString(); var sessionId = Guid.NewGuid().ToString(); var upn = ""; - var expectedMessage = new Message(sessionId, string.Empty, default, "Text", null, rating, upn); - _cosmosDbService.UpdateMessageRatingAsync(id, sessionId, rating).Returns(expectedMessage); + var expectedMessage = new Message(); + //_cosmosDbService.UpdateMessageAsync(id, sessionId, rating).Returns(expectedMessage); // Act - var actualMessage = await _testedService.RateMessageAsync(_instanceId, id, sessionId, rating); + await _testedService.RateMessageAsync(_instanceId, id, sessionId, null); // Assert - Assert.Equivalent(expectedMessage, actualMessage); + Assert.Equivalent(expectedMessage, null); } [Fact] @@ -338,7 +340,7 @@ public async Task RateMessageAsync_ShouldThrowExceptionWhenIdIsNull() // Assert await Assert.ThrowsAsync(async () => { - await _testedService.RateMessageAsync(_instanceId, null!, sessionId, rating); + await _testedService.RateMessageAsync(_instanceId, null!, sessionId, null); }); } @@ -352,7 +354,7 @@ public async Task RateMessageAsync_ShouldThrowExceptionWhenSessionIdIsNull() // Assert await Assert.ThrowsAsync(async () => { - await _testedService.RateMessageAsync(_instanceId, id, null!, rating); + await _testedService.RateMessageAsync(_instanceId, id, null!, null); }); } @@ -367,7 +369,7 @@ public async Task GetCompletionPrompt_ShouldReturnACompletionPrompt() var sessionId = Guid.NewGuid().ToString(); var messageId = Guid.NewGuid().ToString(); var completionPromptId = Guid.NewGuid().ToString(); - var expectedPrompt = new CompletionPrompt(sessionId, messageId, "Text"); + var expectedPrompt = new CompletionPrompt(); _cosmosDbService.GetCompletionPromptAsync(sessionId, completionPromptId).Returns(expectedPrompt); // Act diff --git a/tests/dotnet/Core.Tests/Services/GatekeeperAPIServiceTests.cs b/tests/dotnet/Core.Tests/Services/GatekeeperAPIServiceTests.cs index 38a46eac72..0884154283 100644 --- a/tests/dotnet/Core.Tests/Services/GatekeeperAPIServiceTests.cs +++ b/tests/dotnet/Core.Tests/Services/GatekeeperAPIServiceTests.cs @@ -1,5 +1,7 @@ using FoundationaLLM.Common.Interfaces; -using FoundationaLLM.Common.Models.Chat; +using FoundationaLLM.Common.Models.Conversation; +using FoundationaLLM.Common.Models.Orchestration.Request; +using FoundationaLLM.Common.Models.Orchestration.Response; using FoundationaLLM.Core.Services; using FoundationaLLM.TestUtils.Helpers; using NSubstitute; diff --git a/tests/dotnet/Core.Tests/Services/RegexTemplatingServiceTests.cs b/tests/dotnet/Core.Tests/Services/RegexTemplatingServiceTests.cs new file mode 100644 index 0000000000..f5cb04b65a --- /dev/null +++ b/tests/dotnet/Core.Tests/Services/RegexTemplatingServiceTests.cs @@ -0,0 +1,29 @@ +using FoundationaLLM.Common.Interfaces; +using FoundationaLLM.Common.Services.Templates; +using Microsoft.Extensions.Logging; +using NSubstitute; + +namespace FoundationaLLM.Core.Tests.Services +{ + public class RegexTemplatingServiceTests + { + private readonly ILogger _logger = Substitute.For>(); + + [Fact] + public async Task ReplaceCurrentDateTimeVariable() + { + // Arrange + + ITemplatingService templatingService = new RegexTemplatingService(_logger); + + // Act + var inputString = "The current date is {{foundationallm:current_datetime_utc:dddd, MMMM dd, yyyy}}. This looks great."; + var outputString = templatingService.Transform(inputString); + var expectedOutputString = $"The current date is {DateTime.UtcNow:dddd, MMMM dd, yyyy}. This looks great."; + + + // Assert + Assert.Equal(expectedOutputString, outputString); + } + } +} diff --git a/tests/dotnet/Orchestration.Tests/Services/OrchestrationServiceTests.cs b/tests/dotnet/Orchestration.Tests/Services/OrchestrationServiceTests.cs index 5deef88e0d..c0e9a56157 100644 --- a/tests/dotnet/Orchestration.Tests/Services/OrchestrationServiceTests.cs +++ b/tests/dotnet/Orchestration.Tests/Services/OrchestrationServiceTests.cs @@ -35,6 +35,7 @@ public OrchestrationServiceTests() _resourceProviderServices, null, _cosmosDBService, + null, _callContext, _configuration, null,