change: improvements to cicd

.ci/docker-compose.yaml

@@ -1,6 +1,7 @@
 services:
   hugot:
     image: hugot:$commit_hash
+    container_name: hugot
     build:
       context: ./..
       dockerfile: ./Dockerfile

.github/workflows/release.yaml

@@ -10,23 +10,42 @@ permissions:
 
 jobs:
   test:
-      name: Run test suite
-      runs-on: ubuntu-latest
-      steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-      - name: Build and run tests
-        run: make run-tests
+    name: Run test suite
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    - name: Build and run tests
+      run: make run-tests
+    - name: print
+      run: ls -la
+    - uses: actions/upload-artifact@v4
+      with:
+        name: libtokenizers.a
+        path: ./testTarget/libtokenizers.a
+    - uses: actions/upload-artifact@v4
+      with:
+        name: onnxruntime.so
+        path: ./testTarget/onnxruntime.so
   release:
-    name: Release pushed tag
+    name: Release
     runs-on: ubuntu-latest
+    needs: test
     steps:
-      - name: Create release
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          tag: ${{ github.ref_name }}
-        run: |
-          gh release create "$tag" \
-              --repo="$GITHUB_REPOSITORY" \
-              --title="${GITHUB_REPOSITORY#*/} ${tag#v}" \
-              --generate-notes
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: libtokenizers.a
+          path: .
+      - uses: actions/download-artifact@v4
+        with:
+          name: onnxruntime.so
+          path: .
+      - name: Display structure of downloaded files
+        run: ls -R
+      - uses: ncipollo/release-action@v1
+        with:
+          artifacts: "libtokenizers.a, onnxruntime.so"
+          generateReleaseNotes: true
+          skipIfReleaseExists: true

.github/workflows/test.yaml

@@ -14,3 +14,13 @@ jobs:
         uses: actions/checkout@v4
       - name: Build and run tests
         run: make run-tests
+      - name: print
+        run: ls -la
+      - uses: actions/upload-artifact@v4
+        with:
+          name: libtokenizers.a
+          path: ./testTarget/libtokenizers.a
+      - uses: actions/upload-artifact@v4
+        with:
+          name: onnxruntime.so
+          path: ./testTarget/onnxruntime.so

Dockerfile

@@ -1,5 +1,5 @@
 ARG GO_VERSION=1.22.0
-ARG RUST_VERSION=1.75
+ARG RUST_VERSION=1.76
 ARG ONNXRUNTIME_VERSION=1.17.0
 
 #--- rust build of tokenizer
@@ -46,7 +46,7 @@ RUN curl -LO https://github.com/gotestyourself/gotestsum/releases/download/v1.11
 COPY . /build
 WORKDIR /build
 RUN go mod download && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 && \
-    mkdir /unittest && go test -c ./pipelines -o /unittest && \
+    mkdir /unittest && go test -c . -o /unittest/pipelines.test && \
     go clean -r -cache -testcache -modcache
 
 # models

go.mod

@@ -13,7 +13,7 @@ require (
 
 require (
 	cloud.google.com/go/storage v1.37.0 // indirect
-	github.com/aws/aws-sdk-go v1.50.15 // indirect
+	github.com/aws/aws-sdk-go v1.50.21 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-errors/errors v1.5.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect

go.sum

@@ -8,8 +8,8 @@ cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI=
 cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
 cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4=
 cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k=
-github.com/aws/aws-sdk-go v1.50.15 h1:wEMnPfEQQFaoIJwuO18zq/vtG4Ft7NxQ3r9xlEi/8zg=
-github.com/aws/aws-sdk-go v1.50.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
+github.com/aws/aws-sdk-go v1.50.21 h1:W8awpwiInOt4qHQE6JghRYQJhHcf/cDJS3mlZYqioSQ=
+github.com/aws/aws-sdk-go v1.50.21/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

hugot_test.go

@@ -31,6 +31,9 @@ func TestTextClassificationPipeline(t *testing.T) {
 		Check(session.Destroy())
 	}(session)
 	modelFolder := os.Getenv("TEST_MODELS_FOLDER")
+	if modelFolder == "" {
+		modelFolder = "./models"
+	}
 	modelPath := path.Join(modelFolder, "distilbert-base-uncased-finetuned-sst-2-english")
 	sentimentPipeline, err := session.NewTextClassificationPipeline(modelPath, "testPipeline")
 	Check(err)
@@ -86,6 +89,9 @@ func TestTokenClassificationPipeline(t *testing.T) {
 	}(session)
 
 	modelFolder := os.Getenv("TEST_MODELS_FOLDER")
+	if modelFolder == "" {
+		modelFolder = "./models"
+	}
 	modelPath := path.Join(modelFolder, "distilbert-NER")
 	pipelineSimple, err2 := session.NewTokenClassificationPipeline(modelPath, "testPipeline", pipelines.WithSimpleAggregation())
 	Check(err2)
@@ -148,6 +154,9 @@ func TestFeatureExtractionPipeline(t *testing.T) {
 	}(session)
 
 	modelFolder := os.Getenv("TEST_MODELS_FOLDER")
+	if modelFolder == "" {
+		modelFolder = "./models"
+	}
 	modelPath := path.Join(modelFolder, "all-MiniLM-L6-v2")
 	pipeline, err := session.NewFeatureExtractionPipeline(modelPath, "testPipeline")
 	Check(err)
@@ -206,10 +215,11 @@ func TestFeatureExtractionPipeline(t *testing.T) {
 		}
 	}
 
-	assert.Greater(t, pipeline.PipelineTimings.NumCalls, 0, "PipelineTimings.NumCalls should be greater than 0")
-	assert.Greater(t, pipeline.PipelineTimings.TotalNS, 0, "PipelineTimings.TotalNS should be greater than 0")
-	assert.Greater(t, pipeline.TokenizerTimings.NumCalls, 0, "TokenizerTimings.NumCalls should be greater than 0")
-	assert.Greater(t, pipeline.TokenizerTimings.TotalNS, 0, "TokenizerTimings.TotalNS should be greater than 0")
+	zero := uint64(0)
+	assert.Greater(t, pipeline.PipelineTimings.NumCalls, zero, "PipelineTimings.NumCalls should be greater than 0")
+	assert.Greater(t, pipeline.PipelineTimings.TotalNS, zero, "PipelineTimings.TotalNS should be greater than 0")
+	assert.Greater(t, pipeline.TokenizerTimings.NumCalls, zero, "TokenizerTimings.NumCalls should be greater than 0")
+	assert.Greater(t, pipeline.TokenizerTimings.TotalNS, zero, "TokenizerTimings.TotalNS should be greater than 0")
 }
 
 // utilities

scripts/run-unit-tests.sh

@@ -22,10 +22,12 @@ fi
 
 # build with compose
 docker compose -f $src_dir/.ci/docker-compose.yaml build
-# (cd $src_dir/.ci && docker compose build)
-
 echo "Running tests for commit hash: $commit_hash"
-
 docker compose -f $src_dir/.ci/docker-compose.yaml up && \
 docker compose -f $src_dir/.ci/docker-compose.yaml logs --no-color >& $test_folder/logs.txt && \
+echo "Extracting lib artifacts"
+id=$(docker ps -aqf "name=hugot")
+docker cp $id:/usr/lib/libtokenizers.a ./testTarget/libtokenizers.a
+docker cp $id:/usr/lib64/onnxruntime.so ./testTarget/onnxruntime.so
+echo $id
 docker compose -f $src_dir/.ci/docker-compose.yaml rm -fsv