benchmark: Use AsSpan() for MicrosoftMLTokenizer.

tryAGI · Nov 15, 2024 · dc95d21 · dc95d21
1 parent 6413a0a
commit dc95d21
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/src/benchmarks/Tiktoken.Benchmarks/Benchmarks.cs b/src/benchmarks/Tiktoken.Benchmarks/Benchmarks.cs
@@ -41,7 +41,7 @@ public async Task GlobalSetup()
 
     [Benchmark]
     [BenchmarkCategory("Encode")]
-    public IReadOnlyCollection<int> MicrosoftMLTokenizerV1_0_0_Encode() => _microsoftMlTiktoken.EncodeToIds(Data);
+    public IReadOnlyCollection<int> MicrosoftMLTokenizerV1_0_0_Encode() => _microsoftMlTiktoken.EncodeToIds(Data.AsSpan());
 
     [Benchmark]
     [BenchmarkCategory("Encode")]
@@ -62,7 +62,7 @@ public async Task GlobalSetup()
 
     [Benchmark]
     [BenchmarkCategory("CountTokens")]
-    public int MicrosoftMLTokenizerV1_0_0_() => _microsoftMlTiktoken.CountTokens(Data);
+    public int MicrosoftMLTokenizerV1_0_0_() => _microsoftMlTiktoken.CountTokens(Data.AsSpan());
 
     [Benchmark]
     [BenchmarkCategory("CountTokens")]