Tokenizer 0.9.0

.github/workflows/release.yaml

@@ -11,15 +11,15 @@ permissions:
 jobs:
   test:
       name: Run test suite
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-latest
       steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Build and run tests
         run: make run-tests
   release:
     name: Release pushed tag
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Create release
         env:
@@ -29,4 +29,4 @@ jobs:
           gh release create "$tag" \
               --repo="$GITHUB_REPOSITORY" \
               --title="${GITHUB_REPOSITORY#*/} ${tag#v}" \
-              --generate-notes
+              --generate-notes

.github/workflows/test.yaml

@@ -8,9 +8,9 @@ permissions:
 jobs:
   test:
       name: Run test suite
-      runs-on: ubuntu-22.04
+      runs-on: ubuntu-latest
       steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Build and run tests
-        run: make run-tests
+        run: make run-tests

Dockerfile

@@ -1,6 +1,6 @@
-ARG GO_VERSION=1.22rc1
+ARG GO_VERSION=1.22.0
 ARG RUST_VERSION=1.75
-ARG ONNXRUNTIME_VERSION=1.16.3
+ARG ONNXRUNTIME_VERSION=1.17.0
 
 #--- rust build of tokenizer
 
@@ -81,4 +81,4 @@ COPY ./scripts/entrypoint.sh /entrypoint.sh
 # convert windows line endings if present
 RUN sed -i 's/\r//g' /entrypoint.sh
 RUN chmod +x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/entrypoint.sh"]

go.mod

@@ -4,7 +4,7 @@ go 1.22
 
 require (
 	github.com/json-iterator/go v1.1.12
-	github.com/knights-analytics/tokenizers v0.8.0
+	github.com/knights-analytics/tokenizers v0.9.0
 	github.com/stretchr/testify v1.8.4
 	github.com/viant/afs v1.25.0
 	github.com/viant/afsc v1.9.1
@@ -13,7 +13,7 @@ require (
 
 require (
 	cloud.google.com/go/storage v1.37.0 // indirect
-	github.com/aws/aws-sdk-go v1.50.14 // indirect
+	github.com/aws/aws-sdk-go v1.50.15 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-errors/errors v1.5.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

go.sum

@@ -8,8 +8,8 @@ cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI=
 cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
 cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4=
 cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k=
-github.com/aws/aws-sdk-go v1.50.14 h1:m1bxKtd1lJpNnl+Owah0+UPRuS9f3GFvxBPgc8RiodE=
-github.com/aws/aws-sdk-go v1.50.14/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.50.15 h1:wEMnPfEQQFaoIJwuO18zq/vtG4Ft7NxQ3r9xlEi/8zg=
+github.com/aws/aws-sdk-go v1.50.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -41,8 +41,8 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/knights-analytics/tokenizers v0.8.0 h1:8MNzDAhKxJH1M5nZcjlA6u3fX3u2a08MxRztBmjCTw8=
-github.com/knights-analytics/tokenizers v0.8.0/go.mod h1:8C6oR/TiOPKfsRek7SQHIMFEXeuOzgTANFNWSAPEx8g=
+github.com/knights-analytics/tokenizers v0.9.0 h1:r7BQXW1UHAFKMsyZmQAeWf2VcgcIc29Nlxbx/u8r+50=
+github.com/knights-analytics/tokenizers v0.9.0/go.mod h1:8C6oR/TiOPKfsRek7SQHIMFEXeuOzgTANFNWSAPEx8g=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=

scripts/run-unit-tests.sh

@@ -15,9 +15,9 @@ echo "Downloading the models required for testing from the huggingface hub"
 
 if [[ ! -d "$src_dir/models" ]]; then
     mkdir -p $src_dir/models
-    (cd $src_dir/models && git clone https://huggingface.co/KnightsAnalytics/all-MiniLM-L6-v2 &&\
+    (cd $src_dir/models && git clone https://huggingface.co/KnightsAnalytics/all-MiniLM-L6-v2 && \
     git clone https://huggingface.co/KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english && \
-    git clone git clone https://huggingface.co/KnightsAnalytics/distilbert-NER)
+    git clone https://huggingface.co/KnightsAnalytics/distilbert-NER)
 fi
 
 # build with compose
@@ -28,4 +28,4 @@ echo "Running tests for commit hash: $commit_hash"
 
 docker compose -f $src_dir/.ci/docker-compose.yaml up && \
 docker compose -f $src_dir/.ci/docker-compose.yaml logs --no-color >& $test_folder/logs.txt && \
-docker compose -f $src_dir/.ci/docker-compose.yaml rm -fsv
+docker compose -f $src_dir/.ci/docker-compose.yaml rm -fsv