feat(Vision Model Integration): Compettion of support for GPT-V.

src/Semantic.Core.Examples/Functions/ExampleCustomFunctions.cs

@@ -28,9 +28,26 @@ public ExampleCustomFunctions(IServiceProvider serviceProvider, ISemanticKernelL
     /// <returns>The response body as a string.</returns>
     [KernelFunction, Description("Get the current system time.")]
     public async Task<string> GetTimeAsync(
-        // [Description("Example Argument")] string exampleArgument,
+        //[Description("Example Argument")] string exampleArgument,
         CancellationToken token = default)
     {
         return DateTime.Now.ToLongTimeString();
     }
+
+    /// <summary>
+    /// Get the current system time.
+    /// </summary>
+    /// <param name="token">The token to use to request cancellation.</param>
+    /// <returns>The response body as a string.</returns>
+    [KernelFunction, Description("Get the system time X hour and Y days from now. Both arguments are required.")]
+    public async Task<string> GetFutureTimeAsync(
+        [Description("int:mandatory:The count of hours to get the time for in the future. This argument MUST be provided and NON_ZERO (Ask the user if required).")] string hours,
+        [Description("int:mandatory:The count of days to get the time for in the future. This argument MUST be provided and NON_ZERO (Ask the user if required).")] string days, 
+        CancellationToken token = default)
+    {
+        var hoursInFuture = int.Parse(hours);
+        var daysInFuture = int.Parse(days);
+
+        return DateTime.Now.AddDays(daysInFuture).AddHours(hoursInFuture).ToLongTimeString();
+    }
 }

src/Semantic.Core.Examples/Functions/FreeGameFunctions.cs

@@ -0,0 +1,95 @@
+using System.ComponentModel;
+using FrostAura.Libraries.Semantic.Core.Abstractions.Thoughts;
+using FrostAura.Libraries.Semantic.Core.Interfaces.Data;
+using Microsoft.Extensions.Logging;
+using Microsoft.SemanticKernel;
+
+namespace Semantic.Core.Examples.Functions;
+
+/// <summary>
+/// Free games functions.
+/// </summary>
+public class FreeGameFunctions : BaseThought
+{
+    /// <summary>
+    /// Overloaded constructor to provide dependencies.
+    /// </summary>
+    /// <param name="serviceProvider">The dependency service provider.</param>
+    /// <param name="semanticKernelLanguageModels">A component for communicating with language models.</param>
+    /// <param name="logger">Instance logger.</param>
+    public FreeGameFunctions(IServiceProvider serviceProvider, ISemanticKernelLanguageModelsDataAccess semanticKernelLanguageModels, ILogger<FreeGameFunctions> logger)
+        : base(serviceProvider, semanticKernelLanguageModels, logger)
+    { }
+
+    /// <summary>
+    /// Creates a new free game offer that is simplfied by providing default values for certain fields. The cost per bet for each game provided is used to calculate a combination of chip size and number of coins that match the cost per bet value. The free game offer uses currency levelling for bets. The cost per bet takes on the user's currency when the free game offer is awarded to a user. Currency levelling causes the cost per bet to take on a base currency which then converted to the user's currency.
+    ///
+    /// The base currency is configured for the gaming system, and the exchange rates are generally static, rounded values.
+    /// For example, the betting model may dictate that a bet costs 20.00 decimal currency units.If currency levelling isn't used, a user playing in ZAR would then wager R20 per bet, and a user playing in USD will wager $20 per bet. If currency levelling is used, the wager per bet would have roughly the same monetary value so that the user with the weaker currency isn't disadvantaged.
+    /// </summary>
+    /// <param name="token">The token to use to request cancellation.</param>
+    /// <returns>The response body as a string.</returns>
+    [KernelFunction, Description("Creates a new free game offer that is simplfied by providing default values for certain fields. The cost per bet for each game provided is used to calculate a combination of chip size and number of coins that match the cost per bet value. The free game offer uses currency levelling for bets. The cost per bet takes on the user's currency when the free game offer is awarded to a user. Currency levelling causes the cost per bet to take on a base currency which then converted to the user's currency.\n\nThe base currency is configured for the gaming system, and the exchange rates are generally static, rounded values.\n\nFor example, the betting model may dictate that a bet costs 20.00 decimal currency units. If currency levelling isn't used, a user playing in ZAR would then wager R20 per bet, and a user playing in USD will wager $20 per bet. If currency levelling is used, the wager per bet would have roughly the same monetary value so that the user with the weaker currency isn't disadvantaged.")]
+    public async Task<string> AddFreeGameOfferByNearestCostAsync(
+        [Description("int: The ProductId. This identifies the system containing the free game offer. This value is domain-specific and MUST NOT be made up. The user MUST provide this value / Should be asked to provide this value")] string productId,
+        [Description("int: The default number of free game wagers that will be awarded to a user. This value may be changed when the free game offer is allocated. This value is domain-specific and MUST NOT be made up. The user MUST provide this value / Should be asked to provide this value.")] string defaultNumberOfGames,
+        [Description("int: Uniquely identifies the game in conjunction with the ModuleId. Typically used to identify the graphical skin or client software used to play the game. This value is domain-specific and MUST NOT be made up. This value can be obtained from the list of supported free games.")] string clientId,
+        [Description("int: Uniquely identifies the game in conjunction with the ClientId. Typically used to identify the game engine or rules. This value is domain-specific and MUST NOT be made up. This value can be obtained from the list of supported free games.")] string moduleId,
+        CancellationToken token = default)
+    {
+        // True API implementation goes here.
+        // For the sake of the POC I simply went with a single game but games[] can be done too.
+
+        return "Your free game offer was successfully created!";
+    }
+
+    /// <summary>
+    /// Get Simplified Free Game Supported Games Returns basic information of free games supported games like clientId, moduleId, game name, nearest cost etc.
+    /// </summary>
+    /// <param name="token">The token to use to request cancellation.</param>
+    /// <returns>The response body as a string.</returns>
+    [KernelFunction, Description("Get Simplified Free Game Supported Games Returns basic information of free games supported games like clientId, moduleId, game name, nearest cost etc.")]
+    public async Task<string> GetSimplifiedFreeGameSupportedGamesAsync(
+        [Description("int: The ProductId. This identifies the system containing the free game offer. This value is domain-specific and MUST NOT be made up. The user MUST provide this value / Should be asked to provide this value.")] string productId,
+        CancellationToken token = default)
+    {
+        // True API implementation goes here.
+        // For the sake of the POC I simply went with a single game but games[] can be done too.
+
+        return """
+            {
+              "paging": {
+                "pageNumber": 5,
+                "itemsPerPage": 20,
+                "total": 720
+              },
+              "data": [
+                {
+                  "moduleId": 19596,
+                  "clientId": 10001,
+                  "gameName": "The Twisted Circus (Flash)",
+                  "clientTypeDescription": "HTML5 Desktop Client",
+                  "clientTypeId": 1202,
+                  "nearestCost": [
+                    5.0,
+                    6.0,
+                    7.0
+                  ]
+                },
+                {
+                  "moduleId": 19626,
+                  "clientId": 40300,
+                  "gameName": "Thunderstruck II Scratch(HTML5-Mobile)",
+                  "clientTypeDescription": "HTML5 Desktop Client",
+                  "clientTypeId": 1202,
+                  "nearestCost": [
+                    5.0,
+                    6.0,
+                    7.0
+                  ]
+                }
+              ]
+            }
+            """;
+    }
+}

src/Semantic.Core.Examples/Semantic.Core.Examples.csproj

@@ -11,7 +11,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
-    <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="1.2.0" />
+    <PackageReference Include="Microsoft.SemanticKernel.Abstractions" Version="1.4.0" />
   </ItemGroup>
   <ItemGroup>
     <None Remove="Data\" />

src/Semantic.Core.Tests/Semantic.Core.Tests.csproj

@@ -12,17 +12,17 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
-    <PackageReference Include="xunit" Version="2.6.6" />
-    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.6">
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
+    <PackageReference Include="xunit" Version="2.7.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.5.7">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
     <PackageReference Include="coverlet.collector" Version="6.0.0">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
       <PrivateAssets>all</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.SemanticKernel" Version="1.2.0" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="1.4.0" />
     <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
     <PackageReference Include="NSubstitute" Version="5.1.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />

src/Semantic.Core.Tests/Thoughts/Cognitive/LanguageModelThoughtsTests.cs

@@ -211,4 +211,64 @@ public async Task GetStringEmbeddingsAsync_WithValidInput_ShouldRespond()
 
         Assert.NotEmpty(actual);
     }
+
+    [Fact]
+    public async Task PromptLLMAboutImageFromUrlAsync_WithInvalidPrompt_ShouldThrow()
+    {
+        var serviceCollection = new ServiceCollection()
+            .AddSemanticCore(out var configuration)
+            .AddSingleton(Substitute.For<IUserProxyDataAccess>());
+        var serviceProvider = serviceCollection.BuildServiceProvider();
+        var semanticKernelLanguageModels = serviceProvider.GetRequiredService<ISemanticKernelLanguageModelsDataAccess>();
+        var httpClientFactory = serviceProvider.GetRequiredService<IHttpClientFactory>();
+        var openAIConfigOptions = serviceProvider.GetRequiredService<IOptions<OpenAIConfig>>();
+        var logger = Substitute.For<ILogger<LanguageModelThoughts>>();
+        var instance = new LanguageModelThoughts(serviceCollection.BuildServiceProvider(), semanticKernelLanguageModels, httpClientFactory, openAIConfigOptions, logger);
+        string prompt = default;
+        string imageUrl = "http://via.placeholder.com/200x200";
+
+        var actual = await Assert.ThrowsAsync<ArgumentNullException>(async () => await instance.PromptLLMAboutImageFromUrlAsync(prompt, imageUrl, CancellationToken.None));
+
+        Assert.Equal(nameof(prompt), actual.ParamName);
+    }
+
+    [Fact]
+    public async Task PromptLLMAboutImageFromUrlAsync_WithInvalidImageUrl_ShouldThrow()
+    {
+        var serviceCollection = new ServiceCollection()
+            .AddSemanticCore(out var configuration)
+            .AddSingleton(Substitute.For<IUserProxyDataAccess>());
+        var serviceProvider = serviceCollection.BuildServiceProvider();
+        var semanticKernelLanguageModels = serviceProvider.GetRequiredService<ISemanticKernelLanguageModelsDataAccess>();
+        var httpClientFactory = serviceProvider.GetRequiredService<IHttpClientFactory>();
+        var openAIConfigOptions = serviceProvider.GetRequiredService<IOptions<OpenAIConfig>>();
+        var logger = Substitute.For<ILogger<LanguageModelThoughts>>();
+        var instance = new LanguageModelThoughts(serviceCollection.BuildServiceProvider(), semanticKernelLanguageModels, httpClientFactory, openAIConfigOptions, logger);
+        string prompt = "Describe the image.";
+        string imageUrl = default;
+
+        var actual = await Assert.ThrowsAsync<ArgumentNullException>(async () => await instance.PromptLLMAboutImageFromUrlAsync(prompt, imageUrl, CancellationToken.None));
+
+        Assert.Equal(nameof(imageUrl), actual.ParamName);
+    }
+
+    [Fact]
+    public async Task PromptLLMAboutImageFromUrlAsync_WithValidInput_ShouldRespond()
+    {
+        var serviceCollection = new ServiceCollection()
+            .AddSemanticCore(out var configuration)
+            .AddSingleton(Substitute.For<IUserProxyDataAccess>());
+        var serviceProvider = serviceCollection.BuildServiceProvider();
+        var semanticKernelLanguageModels = serviceProvider.GetRequiredService<ISemanticKernelLanguageModelsDataAccess>();
+        var httpClientFactory = serviceProvider.GetRequiredService<IHttpClientFactory>();
+        var openAIConfigOptions = serviceProvider.GetRequiredService<IOptions<OpenAIConfig>>();
+        var logger = Substitute.For<ILogger<LanguageModelThoughts>>();
+        var instance = new LanguageModelThoughts(serviceCollection.BuildServiceProvider(), semanticKernelLanguageModels, httpClientFactory, openAIConfigOptions, logger);
+        string prompt = "What’s in this image?";
+        string imageUrl = "https://dalleprodsec.blob.core.windows.net/private/images/bc8c4fc7-7309-4167-9f10-0e70c63a0d3f/generated_00.png?se=2024-02-19T11%3A50%3A19Z&sig=GCA2L%2BsAxZquVcuXGBTjpwKr98So9kJ0VwUPT0YwVFk%3D&ske=2024-02-23T15%3A58%3A59Z&skoid=e52d5ed7-0657-4f62-bc12-7e5dbb260a96&sks=b&skt=2024-02-16T15%3A58%3A59Z&sktid=33e01921-4d64-4f8c-a055-5bdaffd5e33d&skv=2020-10-02&sp=r&spr=https&sr=b&sv=2020-10-02";
+
+        var actual = await instance.PromptLLMAboutImageFromUrlAsync(prompt, imageUrl, CancellationToken.None);
+
+        Assert.NotEmpty(actual);
+    }
 }

src/Semantic.Core/Data/OpenAILanguageModelsDataAccess.cs

@@ -155,6 +155,7 @@ public Task<Kernel> GetKernelAsync(CancellationToken token)
                 .Services
                 .AddOpenAIChatCompletion(_openAIConfig.LargeModelName, _openAIConfig.ApiKey, serviceId: ModelType.LargeLLM.ToString(), orgId: _openAIConfig.OrgId)
                 .AddOpenAIChatCompletion(_openAIConfig.SmallModelName, _openAIConfig.ApiKey, serviceId: ModelType.SmallLLM.ToString(), orgId: _openAIConfig.OrgId)
+                .AddOpenAIChatCompletion(_openAIConfig.VisionModelName, _openAIConfig.ApiKey, serviceId: ModelType.Vision.ToString(), orgId: _openAIConfig.OrgId)
                 .AddOpenAITextEmbeddingGeneration(_openAIConfig.EmbeddingModelName, _openAIConfig.ApiKey, serviceId: ModelType.Embedding.ToString(), orgId: _openAIConfig.OrgId)
                 .AddOpenAITextToImage(_openAIConfig.ApiKey, serviceId: ModelType.ImageGeneration.ToString(), orgId: _openAIConfig.OrgId);
         }
@@ -165,6 +166,7 @@ public Task<Kernel> GetKernelAsync(CancellationToken token)
                 .Services
                 .AddAzureOpenAIChatCompletion(_openAIConfig.LargeModelName, _openAIConfig.Endpoint, _openAIConfig.ApiKey, serviceId: ModelType.LargeLLM.ToString())
                 .AddAzureOpenAIChatCompletion(_openAIConfig.SmallModelName, _openAIConfig.Endpoint, _openAIConfig.ApiKey, serviceId: ModelType.SmallLLM.ToString())
+                .AddAzureOpenAIChatCompletion(_openAIConfig.VisionModelName, _openAIConfig.Endpoint, _openAIConfig.ApiKey, serviceId: ModelType.Vision.ToString())
                 .AddAzureOpenAITextEmbeddingGeneration(_openAIConfig.EmbeddingModelName, _openAIConfig.Endpoint, _openAIConfig.ApiKey, serviceId: ModelType.Embedding.ToString())
                 .AddAzureOpenAITextToImage(_openAIConfig.TextToImageModelName, _openAIConfig.Endpoint, _openAIConfig.ApiKey, serviceId: ModelType.ImageGeneration.ToString());
         }

src/Semantic.Core/Enums/Models/ModelTypes.cs

@@ -8,5 +8,6 @@ public enum ModelType
 	LargeLLM,
 	SmallLLM,
 	Embedding,
-	ImageGeneration
+	ImageGeneration,
+	Vision
 }

src/Semantic.Core/FrostAura.Libraries.Intelligence.Semantic.Core.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
     <metadata>
         <id>FrostAura.Libraries.Intelligence.Semantic.Core</id>
-        <version>1.4.2</version>
+        <version>1.5.0</version>
         <title>FrostAura.Libraries.Intelligence.Semantic.Core</title>
         <authors>Dean Martin</authors>
         <owners>FrostAura</owners>

src/Semantic.Core/Models/Configuration/OpenAIConfig.cs

@@ -25,6 +25,10 @@ public class OpenAIConfig
     /// </summary>
     public string TextToImageModelName { get; set; }
     /// <summary>
+    /// The name of the vision model.
+    /// </summary>
+    public string VisionModelName { get; set; }
+    /// <summary>
     /// The endpoint to use for the OpenAI service. If null, the public OpenAI server is configured by default.
     /// </summary>
     public string Endpoint { get; set; }

src/Semantic.Core/Semantic.Core.csproj

@@ -74,22 +74,22 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" />
     <PackageReference Include="FrostAura.Libraries.Core" Version="7.1.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
-    <PackageReference Include="Microsoft.SemanticKernel" Version="1.2.0" />
+    <PackageReference Include="Microsoft.SemanticKernel" Version="1.4.0" />
     <PackageReference Include="Microsoft.SemanticKernel.Connectors.Memory.Pinecone" Version="1.0.0-rc3" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
     <PackageReference Include="PexelsDotNetSDK" Version="1.0.11" />
-    <PackageReference Include="ElevenLabs-DotNet" Version="2.1.0" />
+    <PackageReference Include="ElevenLabs-DotNet" Version="2.1.1" />
     <PackageReference Include="Selenium.WebDriver" Version="4.17.0" />
-    <PackageReference Include="Google.Apis.YouTube.v3" Version="1.66.0.3309" />
+    <PackageReference Include="Google.Apis.YouTube.v3" Version="1.66.0.3330" />
     <PackageReference Include="Google.Apis.Auth" Version="1.66.0" />
     <PackageReference Include="Google.Apis" Version="1.66.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.1.0" />
-    <PackageReference Include="Polly" Version="8.2.1" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="8.2.0" />
+    <PackageReference Include="Polly" Version="8.3.0" />
     <PackageReference Include="Polly.Extensions.Http" Version="3.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.1" />
-    <PackageReference Include="Polly.Core" Version="8.2.1" />
+    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.2" />
+    <PackageReference Include="Polly.Core" Version="8.3.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />