fix: fix permission and data race issues in testing suite

knights-analytics · Mar 21, 2024 · 20a0e18 · 20a0e18
1 parent 7e5ca25
commit 20a0e18
Show file tree

Hide file tree

Showing 7 changed files with 70 additions and 72 deletions.
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -7,6 +7,7 @@ on:
 
 permissions:
   contents: write
+  checks: write
 
 jobs:
   test:
@@ -42,6 +43,11 @@ jobs:
         with:
           name: hugot-cli-linux-amd64 
           path: ./artifacts/hugot-cli-linux-amd64
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure()
+        with:
+          report_paths: '/build/testTarget/unit/unit.xml'
   release:
     name: Release
     runs-on: ubuntu-latest

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -4,6 +4,7 @@ on: workflow_dispatch
 
 permissions:
   contents: write
+  checks: write
 
 jobs:
   test:
@@ -39,3 +40,10 @@ jobs:
         with:
           name: hugot-cli-linux-amd64 
           path: ./artifacts/hugot-cli-linux-amd64
+      - name: print files
+        run: ls -la /build/testTarget/unit
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: success() || failure()
+        with:
+          report_paths: '/build/testTarget/unit/unit.xml'
diff --git a/cmd/main_test.go b/cmd/main_test.go
@@ -9,7 +9,6 @@ import (
 
 	_ "embed"
 
-	"github.com/knights-analytics/hugot"
 	util "github.com/knights-analytics/hugot/utils"
 	"github.com/urfave/cli/v2"
 )
@@ -20,41 +19,6 @@ var textClassificationData []byte
 //go:embed testData/tokenClassification.jsonl
 var tokenClassificationData []byte
 
-func TestMain(m *testing.M) {
-	// model setup
-	if ok, err := util.FileSystem.Exists(context.Background(), "../models"); err == nil {
-		if !ok {
-			session, err := hugot.NewSession()
-			if err != nil {
-				panic(err)
-			}
-			err = os.MkdirAll("../models", os.ModePerm)
-			if err != nil {
-				panic(err)
-			}
-			downloadOptions := hugot.NewDownloadOptions()
-			for _, modelName := range []string{
-				"KnightsAnalytics/all-MiniLM-L6-v2",
-				"KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english",
-				"KnightsAnalytics/distilbert-NER"} {
-				_, err := session.DownloadModel(modelName, "../models", downloadOptions)
-				if err != nil {
-					panic(err)
-				}
-			}
-			err = session.Destroy()
-			if err != nil {
-				panic(err)
-			}
-		}
-	} else {
-		panic(err)
-	}
-	// run all tests
-	code := m.Run()
-	os.Exit(code)
-}
-
 func TestTextClassificationCli(t *testing.T) {
 	app := &cli.App{
 		Name:     "hugot",
@@ -184,6 +148,7 @@ func TestModelChain(t *testing.T) {
 }
 
 func check(t *testing.T, err error) {
+	t.Helper()
 	if err != nil {
 		t.Fatalf("%s", err.Error())
 	}

diff --git a/hugot_test.go b/hugot_test.go
@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
-	"os"
 	"path"
 	"strings"
 	"testing"
@@ -25,41 +24,6 @@ var resultsByte []byte
 // use the system library for the tests
 var onnxruntimeSharedLibrary = "/usr/lib64/onnxruntime.so"
 
-func TestMain(m *testing.M) {
-	// model setup
-	if ok, err := util.FileSystem.Exists(context.Background(), "./models"); err == nil {
-		if !ok {
-			session, err := NewSession(WithOnnxLibraryPath(onnxruntimeSharedLibrary))
-			if err != nil {
-				panic(err)
-			}
-			err = os.MkdirAll("./models", os.ModePerm)
-			if err != nil {
-				panic(err)
-			}
-			downloadOptions := NewDownloadOptions()
-			for _, modelName := range []string{
-				"KnightsAnalytics/all-MiniLM-L6-v2",
-				"KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english",
-				"KnightsAnalytics/distilbert-NER"} {
-				_, err := session.DownloadModel(modelName, "./models", downloadOptions)
-				if err != nil {
-					panic(err)
-				}
-			}
-			err = session.Destroy()
-			if err != nil {
-				panic(err)
-			}
-		}
-	} else {
-		panic(err)
-	}
-	// run all tests
-	code := m.Run()
-	os.Exit(code)
-}
-
 // test download validation
 
 func TestDownloadValidation(t *testing.T) {
@@ -407,6 +371,7 @@ func floatsEqual(a, b []float32) error {
 }
 
 func checkClassificationOutput(t *testing.T, inputResult []pipelines.ClassificationOutput, inputExpected []pipelines.ClassificationOutput) {
+	t.Helper()
 	assert.Equal(t, len(inputResult), len(inputExpected))
 	for i, output := range inputResult {
 		resultExpected := inputExpected[i]
@@ -420,6 +385,7 @@ func almostEqual(a, b float64) bool {
 }
 
 func check(t *testing.T, err error) {
+	t.Helper()
 	if err != nil {
 		t.Fatalf("Test failed with error %s", err.Error())
 	}

diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh
@@ -14,9 +14,15 @@ user_name=testuser # the container non-privileged user (see dockerfile)
 
 if [[ -n $HOST_UID ]] && [[ $HOST_UID != 1000 ]]; then
     # patching internal $user_name to have the required host user id
+    echo "Patching $user_name with host id: $HOST_UID"
     usermod -u "$HOST_UID" $user_name
 fi
 
+if [[ -d "/build" ]]; then
+    echo "Patching build directory"
+    chown -R $user_name:$user_name /build
+fi
+
 echo "$1 running with user: $user_name"
 chown "$user_name:$user_name" "$1" && chmod +x "$1"
 exec runuser -u $user_name -- "$1" "${arguments[@]}"
diff --git a/scripts/run-unit-tests-container.sh b/scripts/run-unit-tests-container.sh
@@ -4,6 +4,7 @@ set -e
 
 cd /build && \
 mkdir -p /test/unit && \
+go run ./testData/downloadModels.go && \
 gotestsum --junitfile=/test/unit/unit.xml --jsonfile=/test/unit/unit.json -- -coverprofile=/test/unit/cover.out -race -covermode=atomic ./...
 
 echo Done.
diff --git a/testData/downloadModels.go b/testData/downloadModels.go
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"context"
+	"os"
+
+	"github.com/knights-analytics/hugot"
+	util "github.com/knights-analytics/hugot/utils"
+)
+
+var onnxruntimeSharedLibrary = "/usr/lib64/onnxruntime.so"
+
+// download the test models.
+func main() {
+	if ok, err := util.FileSystem.Exists(context.Background(), "./models"); err == nil {
+		if !ok {
+			session, err := hugot.NewSession(hugot.WithOnnxLibraryPath(onnxruntimeSharedLibrary))
+			if err != nil {
+				panic(err)
+			}
+			defer func(s hugot.Session) {
+				err := s.Destroy()
+				if err != nil {
+					panic(err)
+				}
+			}(*session)
+
+			err = os.MkdirAll("./models", os.ModePerm)
+			if err != nil {
+				panic(err)
+			}
+			downloadOptions := hugot.NewDownloadOptions()
+			for _, modelName := range []string{
+				"KnightsAnalytics/all-MiniLM-L6-v2",
+				"KnightsAnalytics/distilbert-base-uncased-finetuned-sst-2-english",
+				"KnightsAnalytics/distilbert-NER"} {
+				_, err := session.DownloadModel(modelName, "./models", downloadOptions)
+				if err != nil {
+					panic(err)
+				}
+			}
+		}
+	} else {
+		panic(err)
+	}
+}