downgrade minimum required go version to v1.20 (#23)

cmd/main.go

@@ -195,7 +195,7 @@ var runCommand = &cli.Command{
 		nProcessWorkers := 1
 		var processedWg, writeWg sync.WaitGroup
 
-		for range nProcessWorkers {
+		for i := 0; i < nProcessWorkers; i++ {
 			go processWithPipeline(&processedWg, inputChannel, processedChannel, errorsChannel, pipe)
 			processedWg.Add(1)
 		}
@@ -205,7 +205,7 @@ var runCommand = &cli.Command{
 			Type   string
 		}
 
-		for i := range nWriteWorkers {
+		for i := 0; i < nWriteWorkers; i++ {
 			var writer io.WriteCloser
 
 			if outputPath != "" {
@@ -329,7 +329,7 @@ func writeOutputs(wg *sync.WaitGroup, processedChannel chan []byte, errorChannel
 func processWithPipeline(wg *sync.WaitGroup, inputChannel chan []input, processedChannel chan []byte, errorsChannel chan error, p pipelines.Pipeline) {
 	for inputBatch := range inputChannel {
 		inputStrings := make([]string, len(inputBatch))
-		for i := range len(inputBatch) {
+		for i := 0; i < len(inputBatch); i++ {
 			inputStrings[i] = inputBatch[i].Input
 		}
 		output, err := p.Run(inputStrings)

go.mod

@@ -1,6 +1,6 @@
 module github.com/knights-analytics/hugot
 
-go 1.22
+go 1.20
 
 replace github.com/viant/afsc => github.com/knights-analytics/afsc v0.0.0-20240425201009-7e46526445df
 
@@ -14,6 +14,7 @@ require (
 	github.com/viant/afs v1.25.1
 	github.com/viant/afsc v1.9.2
 	github.com/yalue/onnxruntime_go v1.9.0
+	golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
 )
 
 require (

go.sum

@@ -1,5 +1,3 @@
-github.com/aws/aws-sdk-go v1.51.29 h1:18I5kjEcbyJOH4l2EjQyJJG6v3uPjrzwG0uiSD3vJEM=
-github.com/aws/aws-sdk-go v1.51.29/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/aws/aws-sdk-go v1.51.31 h1:4TM+sNc+Dzs7wY1sJ0+J8i60c6rkgnKP1pvPx8ghsSY=
 github.com/aws/aws-sdk-go v1.51.31/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/bodaay/HuggingFaceModelDownloader v0.0.0-20240307153905-2f38356a6d6c h1:3TPq2BhzOquTGmbS53KeGcM1yalBUb/4zQM1wmaINrE=
@@ -23,12 +21,9 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/knights-analytics/afsc v0.0.0-20240425201009-7e46526445df h1:rVna1iJaI7gj5RonGys0dZ0iLy7upULdcbRQd9F2qg8=
 github.com/knights-analytics/afsc v0.0.0-20240425201009-7e46526445df/go.mod h1:yZo80n1EB2eMwmmec7BekX6clpd7uY+joUpDRIBbeYs=
-github.com/knights-analytics/tokenizers v0.12.0 h1:WRdh9qGgD+Q3gB9wMDjUl0C4c2WLnMkl9KuIvNGKxfk=
-github.com/knights-analytics/tokenizers v0.12.0/go.mod h1:2ngD6H76uCx8TzGUnsdkji1fI6ehQDzU5RgdO6KQenc=
 github.com/knights-analytics/tokenizers v0.12.1 h1:5bIxk3SQKXIHKxlzAOmqPXgFeKE+LCvbXS3hpTgOAX4=
 github.com/knights-analytics/tokenizers v0.12.1/go.mod h1:TD+zVXlFlS4QyP6/RN8SPSAKkT2hpMmF64WdrdbBfts=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
@@ -61,23 +56,20 @@ github.com/yalue/onnxruntime_go v1.9.0 h1:AhgkpBjphJZsHT5karKt93xPkPFNP0Iz6ENUbN
 github.com/yalue/onnxruntime_go v1.9.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
 golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
+golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg=
 golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
 golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
-golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

hugot.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"slices"
 
 	util "github.com/knights-analytics/hugot/utils"
 
@@ -34,7 +33,7 @@ func (m pipelineMap[T]) Destroy() error {
 func (m pipelineMap[T]) GetStats() []string {
 	var stats []string
 	for _, p := range m {
-		stats = slices.Concat(stats, p.GetStats())
+		stats = append(stats, p.GetStats()...)
 	}
 	return stats
 }
@@ -302,9 +301,10 @@ func (s *Session) Destroy() error {
 // the number of batch calls to the onnxruntime inference
 // the average time per onnxruntime inference batch call
 func (s *Session) GetStats() []string {
-	return slices.Concat(s.tokenClassificationPipelines.GetStats(),
-		s.textClassificationPipelines.GetStats(),
-		s.featureExtractionPipelines.GetStats(),
+	// slices.Concat() is not implemented in experimental x/exp/slices package
+	return append(append(s.tokenClassificationPipelines.GetStats(),
+		s.textClassificationPipelines.GetStats()...),
+		s.featureExtractionPipelines.GetStats()...,
 	)
 }
 

hugot_test.go

@@ -676,7 +676,7 @@ func BenchmarkCudaEmbedding(b *testing.B) {
 		b.SkipNow()
 	}
 	p := make([]string, 30000)
-	for i := range 30000 {
+	for i := 0; i < 30000; i++ {
 		p[i] = "The goal of this library is to provide an easy, scalable, and hassle-free way to run huggingface transformer pipelines in golang applications."
 	}
 	for i := 0; i < b.N; i++ {
@@ -689,7 +689,7 @@ func BenchmarkCPUEmbedding(b *testing.B) {
 		b.SkipNow()
 	}
 	p := make([]string, 30000)
-	for i := range 30000 {
+	for i := 0; i < 30000; i++ {
 		p[i] = "The goal of this library is to provide an easy, scalable, and hassle-free way to run huggingface transformer pipelines in golang applications."
 	}
 	for i := 0; i < b.N; i++ {

pipelines/pipeline.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"strings"
 	"sync/atomic"
@@ -331,7 +332,7 @@ func (p *BasePipeline) convertInputToTensors(inputs []TokenizedInput, maxSequenc
 func (p *BasePipeline) GetStats() []string {
 	return []string{
 		fmt.Sprintf("Statistics for pipeline: %s", p.PipelineName),
-		fmt.Sprintf("Tokenizer: Total time=%s, Execution count=%d, Average query time=%s", time.Duration(p.TokenizerTimings.TotalNS), p.TokenizerTimings.NumCalls, time.Duration(p.TokenizerTimings.TotalNS/max(1, p.TokenizerTimings.NumCalls))),
-		fmt.Sprintf("ONNX: Total time=%s, Execution count=%d, Average query time=%s", time.Duration(p.PipelineTimings.TotalNS), p.PipelineTimings.NumCalls, time.Duration(p.PipelineTimings.TotalNS/max(1, p.PipelineTimings.NumCalls))),
+		fmt.Sprintf("Tokenizer: Total time=%s, Execution count=%d, Average query time=%s", time.Duration(p.TokenizerTimings.TotalNS), p.TokenizerTimings.NumCalls, time.Duration(float64(p.TokenizerTimings.TotalNS)/math.Max(1, float64(p.TokenizerTimings.NumCalls)))),
+		fmt.Sprintf("ONNX: Total time=%s, Execution count=%d, Average query time=%s", time.Duration(p.PipelineTimings.TotalNS), p.PipelineTimings.NumCalls, time.Duration(float64(p.PipelineTimings.TotalNS)/math.Max(1, float64(p.PipelineTimings.NumCalls)))),
 	}
 }

pipelines/tokenClassification.go

@@ -3,9 +3,11 @@ package pipelines
 import (
 	"errors"
 	"fmt"
-	"slices"
 	"strings"
 
+	// according to https://freshman.tech/snippets/go/check-if-slice-contains-element
+	"golang.org/x/exp/slices"
+
 	ort "github.com/yalue/onnxruntime_go"
 
 	util "github.com/knights-analytics/hugot/utils"

utils/vectors.go

@@ -3,7 +3,7 @@ package util
 import (
 	"fmt"
 	"math"
-	"slices"
+	"golang.org/x/exp/slices" // like in tokenClassification.go
 )
 
 // Mean of a float32 vector.