From f07b54d123750fa0d62085bdb9977fcb72780da6 Mon Sep 17 00:00:00 2001 From: kylebunting Date: Thu, 8 Aug 2024 14:57:50 -0600 Subject: [PATCH 1/6] Adding message content items for interaction with Assistants API --- src/dotnet/Common/Models/Chat/Message.cs | 10 +++++- .../Orchestration/CompletionResponseBase.cs | 6 ++++ .../Orchestration/MessageContentItemBase.cs | 20 +++++++++++ .../Orchestration/MessageContentItemTypes.cs | 18 ++++++++++ .../OpenAIFilePathContentItem.cs | 34 +++++++++++++++++++ .../OpenAIImageFileMessageContentItem.cs | 32 +++++++++++++++++ .../OpenAITextMessageContentItem.cs | 32 +++++++++++++++++ 7 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs create mode 100644 src/dotnet/Common/Models/Orchestration/MessageContentItemTypes.cs create mode 100644 src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs create mode 100644 src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs create mode 100644 src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs diff --git a/src/dotnet/Common/Models/Chat/Message.cs b/src/dotnet/Common/Models/Chat/Message.cs index a7e730d651..f30e5d8378 100644 --- a/src/dotnet/Common/Models/Chat/Message.cs +++ b/src/dotnet/Common/Models/Chat/Message.cs @@ -1,5 +1,6 @@ using Azure.Search.Documents.Indexes; using FoundationaLLM.Common.Models.Orchestration; +using System.Text.Json.Serialization; namespace FoundationaLLM.Common.Models.Chat; @@ -82,12 +83,18 @@ public record Message /// public Citation[]? Citations { get; set; } + /// + /// Content returned from the Assistants API. + /// + [JsonPropertyName("content")] + public List? Content { get; set; } + /// /// Constructor for Message. /// public Message(string sessionId, string sender, int? tokens, string text, float[]? vector, bool? rating, string upn, string? senderDisplayName = null, - Citation[]? citations = null, string? expectedCompletion = null) + Citation[]? citations = null, string? expectedCompletion = null, List? content = null) { Id = Guid.NewGuid().ToString(); Type = nameof(Message); @@ -102,5 +109,6 @@ public Message(string sessionId, string sender, int? tokens, string text, UPN = upn; ExpectedCompletion = expectedCompletion; Citations = citations; + Content = content; } } diff --git a/src/dotnet/Common/Models/Orchestration/CompletionResponseBase.cs b/src/dotnet/Common/Models/Orchestration/CompletionResponseBase.cs index 7c4008099b..27196d6d50 100644 --- a/src/dotnet/Common/Models/Orchestration/CompletionResponseBase.cs +++ b/src/dotnet/Common/Models/Orchestration/CompletionResponseBase.cs @@ -19,6 +19,12 @@ public class CompletionResponseBase [JsonPropertyName("completion")] public string Completion { get; set; } + /// + /// Content returned from the Assistants API. + /// + [JsonPropertyName("content")] + public List? Content { get; set; } + /// /// The citations used in building the completion response. /// diff --git a/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs new file mode 100644 index 0000000000..6476e054ca --- /dev/null +++ b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs @@ -0,0 +1,20 @@ +using System.Text.Json.Serialization; + +namespace FoundationaLLM.Common.Models.Orchestration +{ + /// + /// Base message content item model. + /// + [JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] + [JsonDerivedType(typeof(OpenAITextMessageContentItem), MessageContentItemTypes.Text)] + [JsonDerivedType(typeof(OpenAIImageFileMessageContentItem), MessageContentItemTypes.ImageFile)] + public class MessageContentItemBase + { + /// + /// The type of the message content item. + /// + [JsonPropertyName("type")] + [JsonPropertyOrder(-100)] + public virtual string? Type { get; set; } + } +} diff --git a/src/dotnet/Common/Models/Orchestration/MessageContentItemTypes.cs b/src/dotnet/Common/Models/Orchestration/MessageContentItemTypes.cs new file mode 100644 index 0000000000..e1785895c1 --- /dev/null +++ b/src/dotnet/Common/Models/Orchestration/MessageContentItemTypes.cs @@ -0,0 +1,18 @@ +namespace FoundationaLLM.Common.Models.Orchestration +{ + /// + /// Contains constants for the types of message content items. + /// + public static class MessageContentItemTypes + { + /// + /// Text content. + /// + public const string Text = "text"; + + /// + /// Image file content. + /// + public const string ImageFile = "image_file"; + } +} diff --git a/src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs new file mode 100644 index 0000000000..a40b5c29e8 --- /dev/null +++ b/src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs @@ -0,0 +1,34 @@ +using System.Text.Json.Serialization; + +namespace FoundationaLLM.Common.Models.Orchestration +{ + /// + /// File content item used to generate a message content item. + /// + public class OpenAIFilePathContentItem + { + /// + /// The text of the annotation. + /// + [JsonPropertyName("text")] + public string? Text { get; set; } + + /// + /// The starting index of the annotation. + /// + [JsonPropertyName("start_index")] + public int? StartIndex { get; set; } + + /// + /// The ending index of the annotation. + /// + [JsonPropertyName("end_index")] + public int? EndIndex { get; set; } + + /// + /// The ID of the file referenced by the annotation. + /// + [JsonPropertyName("file_id")] + public string? FileId { get; set; } + } +} diff --git a/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs new file mode 100644 index 0000000000..efea630648 --- /dev/null +++ b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs @@ -0,0 +1,32 @@ +using System.Text.Json.Serialization; + +namespace FoundationaLLM.Common.Models.Orchestration +{ + /// + /// An OpenAI image file message content item. + /// + public class OpenAIImageFileMessageContentItem : MessageContentItemBase + { + /// + [JsonIgnore] + public override string? Type { get; set; } + + /// + /// The ID of the image file. + /// + [JsonPropertyName("file_id")] + public string? FileId { get; set; } + + /// + /// The URL of the image file. + /// + [JsonPropertyName("file_url")] + public string? FileUrl { get; set; } + + /// + /// Set default property values. + /// + public OpenAIImageFileMessageContentItem() => + Type = MessageContentItemTypes.ImageFile; + } +} diff --git a/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs new file mode 100644 index 0000000000..4449125fe7 --- /dev/null +++ b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs @@ -0,0 +1,32 @@ +using System.Text.Json.Serialization; + +namespace FoundationaLLM.Common.Models.Orchestration +{ + /// + /// An OpenAI text message content item. + /// + public class OpenAITextMessageContentItem : MessageContentItemBase + { + /// + [JsonIgnore] + public override string? Type { get; set; } + + /// + /// A list of file annotations used to generate the message content item. + /// + [JsonPropertyName("annotations")] + public List Annotations { get; set; } = []; + + /// + /// The text value of the message content item. + /// + [JsonPropertyName("value")] + public string? Value { get; set; } + + /// + /// Set default property values. + /// + public OpenAITextMessageContentItem() => + Type = MessageContentItemTypes.Text; + } +} From 56e89d9962543dfd48a6256a007a1599e0443f00 Mon Sep 17 00:00:00 2001 From: kylebunting Date: Thu, 8 Aug 2024 15:13:58 -0600 Subject: [PATCH 2/6] Moving type enum to constants namespace --- .../Orchestration => Constants}/MessageContentItemTypes.cs | 2 +- .../Common/Models/Orchestration/MessageContentItemBase.cs | 1 + .../Models/Orchestration/OpenAIImageFileMessageContentItem.cs | 1 + .../Common/Models/Orchestration/OpenAITextMessageContentItem.cs | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) rename src/dotnet/Common/{Models/Orchestration => Constants}/MessageContentItemTypes.cs (88%) diff --git a/src/dotnet/Common/Models/Orchestration/MessageContentItemTypes.cs b/src/dotnet/Common/Constants/MessageContentItemTypes.cs similarity index 88% rename from src/dotnet/Common/Models/Orchestration/MessageContentItemTypes.cs rename to src/dotnet/Common/Constants/MessageContentItemTypes.cs index e1785895c1..862be31645 100644 --- a/src/dotnet/Common/Models/Orchestration/MessageContentItemTypes.cs +++ b/src/dotnet/Common/Constants/MessageContentItemTypes.cs @@ -1,4 +1,4 @@ -namespace FoundationaLLM.Common.Models.Orchestration +namespace FoundationaLLM.Common.Constants { /// /// Contains constants for the types of message content items. diff --git a/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs index 6476e054ca..484991bd78 100644 --- a/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs +++ b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using FoundationaLLM.Common.Constants; namespace FoundationaLLM.Common.Models.Orchestration { diff --git a/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs index efea630648..9d4b446c7c 100644 --- a/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs +++ b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using FoundationaLLM.Common.Constants; namespace FoundationaLLM.Common.Models.Orchestration { diff --git a/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs index 4449125fe7..7aafb88ecc 100644 --- a/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs +++ b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs @@ -1,4 +1,5 @@ using System.Text.Json.Serialization; +using FoundationaLLM.Common.Constants; namespace FoundationaLLM.Common.Models.Orchestration { From 21553ef3bc2b0b4bb40ba8823dfc0ed9c8818173 Mon Sep 17 00:00:00 2001 From: kylebunting Date: Thu, 8 Aug 2024 15:15:22 -0600 Subject: [PATCH 3/6] Moving to orchestration constants folder --- .../Constants/{ => Orchestration}/MessageContentItemTypes.cs | 2 +- .../Common/Models/Orchestration/MessageContentItemBase.cs | 2 +- .../Models/Orchestration/OpenAIImageFileMessageContentItem.cs | 2 +- .../Common/Models/Orchestration/OpenAITextMessageContentItem.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename src/dotnet/Common/Constants/{ => Orchestration}/MessageContentItemTypes.cs (87%) diff --git a/src/dotnet/Common/Constants/MessageContentItemTypes.cs b/src/dotnet/Common/Constants/Orchestration/MessageContentItemTypes.cs similarity index 87% rename from src/dotnet/Common/Constants/MessageContentItemTypes.cs rename to src/dotnet/Common/Constants/Orchestration/MessageContentItemTypes.cs index 862be31645..edfde5d420 100644 --- a/src/dotnet/Common/Constants/MessageContentItemTypes.cs +++ b/src/dotnet/Common/Constants/Orchestration/MessageContentItemTypes.cs @@ -1,4 +1,4 @@ -namespace FoundationaLLM.Common.Constants +namespace FoundationaLLM.Common.Constants.Orchestration { /// /// Contains constants for the types of message content items. diff --git a/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs index 484991bd78..45b292b068 100644 --- a/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs +++ b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using FoundationaLLM.Common.Constants; +using FoundationaLLM.Common.Constants.Orchestration; namespace FoundationaLLM.Common.Models.Orchestration { diff --git a/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs index 9d4b446c7c..38afe13c73 100644 --- a/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs +++ b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using FoundationaLLM.Common.Constants; +using FoundationaLLM.Common.Constants.Orchestration; namespace FoundationaLLM.Common.Models.Orchestration { diff --git a/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs index 7aafb88ecc..899ec0284e 100644 --- a/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs +++ b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using FoundationaLLM.Common.Constants; +using FoundationaLLM.Common.Constants.Orchestration; namespace FoundationaLLM.Common.Models.Orchestration { From c8b1582b9b5132a5425bcb707ca215a47c5050c5 Mon Sep 17 00:00:00 2001 From: kylebunting Date: Thu, 8 Aug 2024 15:27:39 -0600 Subject: [PATCH 4/6] Adding base class to File Path Content Item --- .../Orchestration/MessageContentItemTypes.cs | 5 +++++ .../Orchestration/MessageContentItemBase.cs | 5 +++-- .../Orchestration/OpenAIFilePathContentItem.cs | 15 +++++++++++++-- .../OpenAIImageFileMessageContentItem.cs | 4 ++-- .../Orchestration/OpenAITextMessageContentItem.cs | 4 ++-- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/dotnet/Common/Constants/Orchestration/MessageContentItemTypes.cs b/src/dotnet/Common/Constants/Orchestration/MessageContentItemTypes.cs index edfde5d420..4317366b6f 100644 --- a/src/dotnet/Common/Constants/Orchestration/MessageContentItemTypes.cs +++ b/src/dotnet/Common/Constants/Orchestration/MessageContentItemTypes.cs @@ -14,5 +14,10 @@ public static class MessageContentItemTypes /// Image file content. /// public const string ImageFile = "image_file"; + + /// + /// File path content. + /// + public const string FilePath = "file_path"; } } diff --git a/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs index 45b292b068..2800b645a7 100644 --- a/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs +++ b/src/dotnet/Common/Models/Orchestration/MessageContentItemBase.cs @@ -1,5 +1,5 @@ -using System.Text.Json.Serialization; -using FoundationaLLM.Common.Constants.Orchestration; +using FoundationaLLM.Common.Constants.Orchestration; +using System.Text.Json.Serialization; namespace FoundationaLLM.Common.Models.Orchestration { @@ -9,6 +9,7 @@ namespace FoundationaLLM.Common.Models.Orchestration [JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] [JsonDerivedType(typeof(OpenAITextMessageContentItem), MessageContentItemTypes.Text)] [JsonDerivedType(typeof(OpenAIImageFileMessageContentItem), MessageContentItemTypes.ImageFile)] + [JsonDerivedType(typeof(OpenAIFilePathContentItem), MessageContentItemTypes.FilePath)] public class MessageContentItemBase { /// diff --git a/src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs index a40b5c29e8..f23cb71090 100644 --- a/src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs +++ b/src/dotnet/Common/Models/Orchestration/OpenAIFilePathContentItem.cs @@ -1,12 +1,17 @@ -using System.Text.Json.Serialization; +using FoundationaLLM.Common.Constants.Orchestration; +using System.Text.Json.Serialization; namespace FoundationaLLM.Common.Models.Orchestration { /// /// File content item used to generate a message content item. /// - public class OpenAIFilePathContentItem + public class OpenAIFilePathContentItem : MessageContentItemBase { + /// + [JsonIgnore] + public override string? Type { get; set; } + /// /// The text of the annotation. /// @@ -30,5 +35,11 @@ public class OpenAIFilePathContentItem /// [JsonPropertyName("file_id")] public string? FileId { get; set; } + + /// + /// Set default property values. + /// + public OpenAIFilePathContentItem() => + Type = MessageContentItemTypes.FilePath; } } diff --git a/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs index 38afe13c73..d322eac2b8 100644 --- a/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs +++ b/src/dotnet/Common/Models/Orchestration/OpenAIImageFileMessageContentItem.cs @@ -1,5 +1,5 @@ -using System.Text.Json.Serialization; -using FoundationaLLM.Common.Constants.Orchestration; +using FoundationaLLM.Common.Constants.Orchestration; +using System.Text.Json.Serialization; namespace FoundationaLLM.Common.Models.Orchestration { diff --git a/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs index 899ec0284e..cd9cbdd023 100644 --- a/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs +++ b/src/dotnet/Common/Models/Orchestration/OpenAITextMessageContentItem.cs @@ -1,5 +1,5 @@ -using System.Text.Json.Serialization; -using FoundationaLLM.Common.Constants.Orchestration; +using FoundationaLLM.Common.Constants.Orchestration; +using System.Text.Json.Serialization; namespace FoundationaLLM.Common.Models.Orchestration { From 9c7b7c1b3ff8eb068193a9df5aff0cd6b327cfa0 Mon Sep 17 00:00:00 2001 From: joelhulen Date: Thu, 8 Aug 2024 19:28:36 -0400 Subject: [PATCH 5/6] Add message content object to store in Cosmos DB --- src/dotnet/Common/Models/Chat/Message.cs | 4 ++-- .../Common/Models/Chat/MessageContent.cs | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 src/dotnet/Common/Models/Chat/MessageContent.cs diff --git a/src/dotnet/Common/Models/Chat/Message.cs b/src/dotnet/Common/Models/Chat/Message.cs index f30e5d8378..222c68a023 100644 --- a/src/dotnet/Common/Models/Chat/Message.cs +++ b/src/dotnet/Common/Models/Chat/Message.cs @@ -84,10 +84,10 @@ public record Message public Citation[]? Citations { get; set; } /// - /// Content returned from the Assistants API. + /// The content of the message. /// [JsonPropertyName("content")] - public List? Content { get; set; } + public List? Content { get; set; } /// /// Constructor for Message. diff --git a/src/dotnet/Common/Models/Chat/MessageContent.cs b/src/dotnet/Common/Models/Chat/MessageContent.cs new file mode 100644 index 0000000000..81d646862a --- /dev/null +++ b/src/dotnet/Common/Models/Chat/MessageContent.cs @@ -0,0 +1,22 @@ +using System.Text.Json.Serialization; + +namespace FoundationaLLM.Common.Models.Chat +{ + /// + /// Contains parts that compose the message content. + /// + public class MessageContent + { + /// + /// The type of the message content. Could be text, image, etc. + /// + [JsonPropertyName("type")] + public string? Type { get; set; } + + /// + /// The value of the message content. + /// + [JsonPropertyName("value")] + public string? Value { get; set; } + } +} From 5fc864c88c14bdd47959f1710cca111cc6b5ec76 Mon Sep 17 00:00:00 2001 From: joelhulen Date: Thu, 8 Aug 2024 19:32:42 -0400 Subject: [PATCH 6/6] Update constructor --- src/dotnet/Common/Models/Chat/Message.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dotnet/Common/Models/Chat/Message.cs b/src/dotnet/Common/Models/Chat/Message.cs index 222c68a023..ae7cf53fb5 100644 --- a/src/dotnet/Common/Models/Chat/Message.cs +++ b/src/dotnet/Common/Models/Chat/Message.cs @@ -94,7 +94,7 @@ public record Message /// public Message(string sessionId, string sender, int? tokens, string text, float[]? vector, bool? rating, string upn, string? senderDisplayName = null, - Citation[]? citations = null, string? expectedCompletion = null, List? content = null) + Citation[]? citations = null, string? expectedCompletion = null, List? content = null) { Id = Guid.NewGuid().ToString(); Type = nameof(Message);