feat: Updated TargetFrameworks.

tryAGI · Jul 10, 2024 · 96fd223 · 96fd223
1 parent 0c73778
commit 96fd223
Show file tree

Hide file tree

Showing 10 changed files with 39 additions and 10 deletions.
diff --git a/global.json b/global.json
@@ -0,0 +1,6 @@
+{
+  "sdk": {
+    "rollForward": "latestMajor",
+    "allowPrerelease": false
+  }
+}
diff --git a/src/libs/Directory.Build.props b/src/libs/Directory.Build.props
@@ -9,7 +9,7 @@
     </ItemGroup>
 
     <PropertyGroup Label="Nuget">
-        <Version>2.0.2</Version>
+        <Version>2.0.3</Version>
         <Description>The fastest tokenizer for GPT-3.5 and GPT-4 inspired by Tiktoken.</Description>
         <PackageTags>chatgpt;openai;tiktoken;tokens;gpt-4;gpt-3.5-turbo;cl100k_base;p50k_base</PackageTags>
         <GeneratePackageOnBuild Condition=" '$(Configuration)' == 'Release' ">true</GeneratePackageOnBuild>

diff --git a/src/libs/Tiktoken.Core/Tiktoken.Core.csproj b/src/libs/Tiktoken.Core/Tiktoken.Core.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net4.6.1;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net6.0;net8.0</TargetFrameworks>
         <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
         <NoWarn>$(NoWarn);CA1724</NoWarn>
         <RootNamespace>Tiktoken</RootNamespace>
@@ -20,7 +20,7 @@
         <EmbeddedResource Include="Encodings/*.tiktoken" />
     </ItemGroup>
 
-    <ItemGroup Condition="'$(TargetFramework)' == 'net4.6.1'">
+    <ItemGroup Condition="'$(TargetFramework)' == 'net4.6.2'">
       <PackageReference Include="System.ValueTuple" />
     </ItemGroup>
 

diff --git a/src/libs/Tiktoken.Encodings.Abstractions/EncodingLoader.cs b/src/libs/Tiktoken.Encodings.Abstractions/EncodingLoader.cs
@@ -31,8 +31,31 @@ public static Dictionary<byte[], int> LoadEncodingFromManifestResource(
             throw new InvalidOperationException("Resource not found.");
         using var reader = new StreamReader(stream);
 
-        var dictionary = new Dictionary<byte[], int>(new ByteArrayComparer());
+        var lines = new List<string>();
         while (reader.ReadLine() is { } line)
+        {
+            lines.Add(line);
+        }
+
+        return LoadEncodingFromLines(lines, name);
+    }
+
+    /// <summary>
+    /// 
+    /// </summary>
+    /// <param name="lines"></param>
+    /// <param name="name"></param>
+    /// <returns></returns>
+    /// <exception cref="InvalidOperationException"></exception>
+    /// <exception cref="FormatException"></exception>
+    public static Dictionary<byte[], int> LoadEncodingFromLines(
+        this IReadOnlyList<string> lines,
+        string name)
+    {
+        lines = lines ?? throw new ArgumentNullException(nameof(lines));
+
+        var dictionary = new Dictionary<byte[], int>(new ByteArrayComparer());
+        foreach (var line in lines)
         {
             if (string.IsNullOrWhiteSpace(line))
             {

diff --git a/src/libs/Tiktoken.Encodings.Abstractions/Tiktoken.Encodings.Abstractions.csproj b/src/libs/Tiktoken.Encodings.Abstractions/Tiktoken.Encodings.Abstractions.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net4.6.1;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net6.0;net8.0</TargetFrameworks>
         <RootNamespace>Tiktoken.Encodings</RootNamespace>
     </PropertyGroup>
 

diff --git a/src/libs/Tiktoken.Encodings.cl100k/Tiktoken.Encodings.cl100k.csproj b/src/libs/Tiktoken.Encodings.cl100k/Tiktoken.Encodings.cl100k.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net4.6.1;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net6.0;net8.0</TargetFrameworks>
         <RootNamespace>Tiktoken.Encodings</RootNamespace>
     </PropertyGroup>
 

diff --git a/src/libs/Tiktoken.Encodings.o200k/Tiktoken.Encodings.o200k.csproj b/src/libs/Tiktoken.Encodings.o200k/Tiktoken.Encodings.o200k.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net4.6.1;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net6.0;net8.0</TargetFrameworks>
         <RootNamespace>Tiktoken.Encodings</RootNamespace>
     </PropertyGroup>
 

diff --git a/src/libs/Tiktoken.Encodings.p50k/Tiktoken.Encodings.p50k.csproj b/src/libs/Tiktoken.Encodings.p50k/Tiktoken.Encodings.p50k.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net4.6.1;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net6.0;net8.0</TargetFrameworks>
         <RootNamespace>Tiktoken.Encodings</RootNamespace>
     </PropertyGroup>
 

diff --git a/src/libs/Tiktoken.Encodings.r50k/Tiktoken.Encodings.r50k.csproj b/src/libs/Tiktoken.Encodings.r50k/Tiktoken.Encodings.r50k.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net4.6.1;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net6.0;net8.0</TargetFrameworks>
         <RootNamespace>Tiktoken.Encodings</RootNamespace>
     </PropertyGroup>
 

diff --git a/src/libs/Tiktoken/Tiktoken.csproj b/src/libs/Tiktoken/Tiktoken.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net4.6.1;netstandard2.0;netstandard2.1;net6.0;net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net4.6.2;netstandard2.0;netstandard2.1;net6.0;net8.0</TargetFrameworks>
     </PropertyGroup>
 
     <ItemGroup>