From 72e71cf75a14fb08e72342fde7f01d97512b174a Mon Sep 17 00:00:00 2001 From: robkeevil Date: Sat, 10 Feb 2024 11:18:27 +0100 Subject: [PATCH] upgrade to tokenizer 0.9.0 --- Dockerfile | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 53301a7..860e110 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -ARG GO_VERSION=1.22rc1 +ARG GO_VERSION=1.22 ARG RUST_VERSION=1.75 -ARG ONNXRUNTIME_VERSION=1.16.3 +ARG ONNXRUNTIME_VERSION=1.17.0 #--- rust build of tokenizer @@ -81,4 +81,4 @@ COPY ./scripts/entrypoint.sh /entrypoint.sh # convert windows line endings if present RUN sed -i 's/\r//g' /entrypoint.sh RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/entrypoint.sh"] diff --git a/go.mod b/go.mod index ca733d3..be4b53b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( github.com/json-iterator/go v1.1.12 - github.com/knights-analytics/tokenizers v0.8.0 + github.com/knights-analytics/tokenizers v0.9.0 github.com/stretchr/testify v1.8.4 github.com/viant/afs v1.25.0 github.com/viant/afsc v1.9.1 diff --git a/go.sum b/go.sum index 416754b..89f4c5b 100644 --- a/go.sum +++ b/go.sum @@ -41,8 +41,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/knights-analytics/tokenizers v0.8.0 h1:8MNzDAhKxJH1M5nZcjlA6u3fX3u2a08MxRztBmjCTw8= -github.com/knights-analytics/tokenizers v0.8.0/go.mod h1:8C6oR/TiOPKfsRek7SQHIMFEXeuOzgTANFNWSAPEx8g= +github.com/knights-analytics/tokenizers v0.9.0 h1:r7BQXW1UHAFKMsyZmQAeWf2VcgcIc29Nlxbx/u8r+50= +github.com/knights-analytics/tokenizers v0.9.0/go.mod h1:8C6oR/TiOPKfsRek7SQHIMFEXeuOzgTANFNWSAPEx8g= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=