Thread pool

README.md

@@ -14,7 +14,8 @@ Pre-reqs:
 # go get -u github.com/notnil/chess
 go get -u github.com/Tilps/chess
 go get -u github.com/nightlyone/lockfile
-
+go get -u github.com/jaypipes/ghw"
+go get -u github.com/shettyh/threadpool
 ```
 
 Pull or download the `master` branch
@@ -56,4 +57,4 @@ Building the client for each platform:
 GOOS=windows GOARCH=amd64 go build -o lczero-client.exe
 GOOS=darwin GOARCH=amd64 go build -o lczero-client_mac
 GOOS=linux GOARCH=amd64 go build -o lczero-client_linux
-```
+```

appveyor.yml

@@ -14,6 +14,8 @@ environment:
 install:
   - go get -u github.com/Tilps/chess
   - go get -u github.com/nightlyone/lockfile
+  - go get -u github.com/shettyh/threadpool
+  - go get -u github.com/jaypipes/ghw
 build_script:
   - go build -o lc0-training-client%NAME% lc0_main.go
 artifacts:
@@ -26,4 +28,3 @@ deploy:
       secure: USFAdwQKTXqOXQjCYQfzWvzRpUhvqJLBkN4hbOg+j876vDxGZHt9bMYayb5evePp
     on:
       appveyor_repo_tag: true
-

build.sh

@@ -0,0 +1,16 @@
+echo Downloading required dependencies...
+echo -ne [0/4] Tilps/chess\\r
+go get github.com/Tilps/chess
+echo -ne [1/4] nightlyone/lockfile\\r
+go get github.com/nightlyone/lockfile
+echo -ne [2/4] jaypipes/ghw\\r
+go get github.com/jaypipes/ghw
+echo -ne [3/4] shettyh/threadpool\\r
+go get github.com/shettyh/threadpool
+echo -ne [DONE] installed dependencies\\n
+echo building windows...
+GOOS=windows GOARCH=amd64 go build
+echo finished with errno: $?
+echo building linux...
+GOOS=linux GOARCH=arm go build
+echo finished with errno: $?

lc0_main.go

@@ -33,6 +33,8 @@ import (
 
 	"github.com/Tilps/chess"
 	"github.com/nightlyone/lockfile"
+	"github.com/jaypipes/ghw"
+	"github.com/shettyh/threadpool"
 )
 
 var (
@@ -57,6 +59,7 @@ var (
 	user      = flag.String("user", "", "Username")
 	password  = flag.String("password", "", "Password")
 	gpu       = flag.Int("gpu", -1, "GPU to use (ignored if --backend-opts used)")
+	quiet     = flag.Bool("quiet", false, "force quiet mode or force non quiet mode")
 	//	debug    = flag.Bool("debug", false, "Enable debug mode to see verbose output and save logs")
 	lc0Args  = flag.String("lc0args", "", "")
 	backopts = flag.String("backend-opts", "",
@@ -81,6 +84,35 @@ type Settings struct {
 	Localhost string
 }
 
+type GameTask struct { 
+	cli *http.Client 
+	ctr int
+}
+
+func isFlagPassed(name string) bool {
+    found := false
+    flag.Visit(func(f *flag.Flag) {
+        if f.Name == name {
+            found = true
+        }
+    })
+    return found
+}
+
+func (t *GameTask) Run() {
+	var err error
+	err = nextGame(t.cli, t.ctr)
+	if err != nil {
+		if err.Error() == "retry" {
+			time.Sleep(1 * time.Second)
+			err = nextGame(t.cli, t.ctr)
+		}
+		log.Print(err)
+		log.Print("Sleeping for 30 seconds...")
+		time.Sleep(30 * time.Second)
+	}
+}
+
 const inf = "inf"
 
 /*
@@ -401,7 +433,9 @@ func (c *cmdWrapper) launch(networkPath string, otherNetPath string, args []stri
 		c.Cmd.Args = append(c.Cmd.Args, "--no-share-trees")
 	}
 
-	fmt.Printf("Args: %v\n", c.Cmd.Args)
+	if !*quiet {
+		fmt.Printf("Args: %v\n", c.Cmd.Args)
+	}
 
 	stdout, err := c.Cmd.StdoutPipe()
 	if err != nil {
@@ -1002,7 +1036,10 @@ func nextGame(httpClient *http.Client, count int) error {
 	if err != nil {
 		return err
 	}
-	log.Printf("serverParams: %s", serverParams)
+	if !*quiet {
+		log.Println(*quiet)
+		log.Printf("serverParams: %s", serverParams)
+	}
 
 	if nextGame.BookUrl != "" {
 		book, err := getBook(&http.Client{}, nextGame.BookUrl, nextGame.BookSha)
@@ -1134,6 +1171,16 @@ func maybeSetTrainOnly() {
 	}
 }
 
+func getGpuNumber() (int) {
+	gpu, err := ghw.GPU()
+	if err != nil {
+		fmt.Printf("Error getting GPU info: %v", err)
+		return 0
+	}
+
+	return len(gpu.GraphicsCards)
+}
+
 func main() {
 	fmt.Printf("Lc0 client version %v\n", getExtraParams()["version"])
 
@@ -1238,19 +1285,28 @@ func main() {
 		*localHost = defaultLocalHost
 	}
 
+	var gpunum int
+	gpunum = getGpuNumber()
+	fmt.Printf("Detected %v GPU(s)\n", gpunum)
+
+	if !isFlagPassed("quiet") {
+		*quiet = gpunum > 1
+	}
+
+	if *quiet {
+		fmt.Println("quiet_mode: on")
+	} else {
+		fmt.Println("quiet_mode: off")
+	}
+
 	httpClient := &http.Client{Timeout:300 * time.Second}
 	startTime = time.Now()
+	pool := threadpool.NewThreadPool(gpunum,100)
 	for i := 0; ; i++ {
-		err := nextGame(httpClient, i)
-		if err != nil {
-			if err.Error() == "retry" {
-				time.Sleep(1 * time.Second)
-				continue
-			}
-			log.Print(err)
-			log.Print("Sleeping for 30 seconds...")
-			time.Sleep(30 * time.Second)
-			continue
+		task := &GameTask{httpClient, i}
+		pool.Execute(task)
+		if i % gpunum == 0 {
+			time.Sleep(time.Second * 10)
 		}
 	}
 }