-
Notifications
You must be signed in to change notification settings - Fork 0
/
bpe_benchmark_test.go
89 lines (78 loc) · 2.23 KB
/
bpe_benchmark_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
package bpe
import (
"testing"
"github.com/pkoukk/tiktoken-go"
)
var benchmarkTexts = []struct {
name string
text string
}{
{"Short", "Hello, world!"},
{"Medium", "This is a medium length text that contains multiple sentences. It should give us a good idea of tokenization performance."},
{"Long", `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.`},
{"Unicode", "Hello 👋 World 🌍! This test includes emojis 🎉 and other Unicode characters: 你好, こんにちは, Привет"},
}
func BenchmarkCL100kTokenizer(b *testing.B) {
tok, err := NewCL100kTokenizer()
if err != nil {
b.Fatalf("Failed to create CL100k tokenizer: %v", err)
}
for _, tc := range benchmarkTexts {
b.Run(tc.name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := tok.Encode(tc.text)
if err != nil {
b.Fatalf("Encode failed: %v", err)
}
}
})
}
}
func BenchmarkO200kTokenizer(b *testing.B) {
tok, err := NewO200kTokenizer()
if err != nil {
b.Fatalf("Failed to create O200k tokenizer: %v", err)
}
for _, tc := range benchmarkTexts {
b.Run(tc.name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := tok.Encode(tc.text)
if err != nil {
b.Fatalf("Encode failed: %v", err)
}
}
})
}
}
func BenchmarkTiktokenCL100k(b *testing.B) {
enc, err := tiktoken.GetEncoding("cl100k_base")
if err != nil {
b.Fatalf("Failed to create CL100k tokenizer: %v", err)
}
for _, tc := range benchmarkTexts {
b.Run(tc.name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(tc.text, nil, nil)
}
})
}
}
func BenchmarkTiktokenO200k(b *testing.B) {
enc, err := tiktoken.GetEncoding("o200k_base")
if err != nil {
b.Fatalf("Failed to create O200k tokenizer: %v", err)
}
for _, tc := range benchmarkTexts {
b.Run(tc.name, func(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = enc.Encode(tc.text, nil, nil)
}
})
}
}