Merge pull request #2 from knights-analytics/tokenizer-0.9.0

Tokenizer 0.9.0

Dockerfile

@@ -1,6 +1,6 @@
-ARG GO_VERSION=1.22rc1
+ARG GO_VERSION=1.22.0
 ARG RUST_VERSION=1.75
-ARG ONNXRUNTIME_VERSION=1.16.3
+ARG ONNXRUNTIME_VERSION=1.17.0
 
 #--- rust build of tokenizer
 
@@ -81,4 +81,4 @@ COPY ./scripts/entrypoint.sh /entrypoint.sh
 # convert windows line endings if present
 RUN sed -i 's/\r//g' /entrypoint.sh
 RUN chmod +x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]

go.mod

@@ -4,7 +4,7 @@ go 1.22
 
 require (
 	github.com/json-iterator/go v1.1.12
-	github.com/knights-analytics/tokenizers v0.8.0
+	github.com/knights-analytics/tokenizers v0.9.0
 	github.com/stretchr/testify v1.8.4
 	github.com/viant/afs v1.25.0
 	github.com/viant/afsc v1.9.1

go.sum

@@ -41,8 +41,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/knights-analytics/tokenizers v0.8.0 h1:8MNzDAhKxJH1M5nZcjlA6u3fX3u2a08MxRztBmjCTw8=
-github.com/knights-analytics/tokenizers v0.8.0/go.mod h1:8C6oR/TiOPKfsRek7SQHIMFEXeuOzgTANFNWSAPEx8g=
+github.com/knights-analytics/tokenizers v0.9.0 h1:r7BQXW1UHAFKMsyZmQAeWf2VcgcIc29Nlxbx/u8r+50=
+github.com/knights-analytics/tokenizers v0.9.0/go.mod h1:8C6oR/TiOPKfsRek7SQHIMFEXeuOzgTANFNWSAPEx8g=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=