Improvements in copy package files

dumper/packageDumper/packageDumper.go

@@ -3,11 +3,11 @@ package packageDumper
 import (
 	"database/sql"
 	"fmt"
+	"github.com/rs/zerolog/log"
 	"github.com/uyuni-project/inter-server-sync/dumper"
 	"github.com/uyuni-project/inter-server-sync/schemareader"
-	"io"
-	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 )
 
@@ -21,54 +21,69 @@ func DumpPackageFiles(db *sql.DB, schemaMetadata map[string]schemareader.Table,
 
 	exportPoint := 0
 	batchSize := 500
+	jobs := make(chan fileToCopy, batchSize)
+	results := make(chan error, batchSize)
+	for w := 1; w <= 5; w++ {
+		go worker(w, jobs, results)
+	}
+
 	for len(packageKeysData.Keys) > exportPoint {
 		upperLimit := exportPoint + batchSize
 		if upperLimit > len(packageKeysData.Keys) {
 			upperLimit = len(packageKeysData.Keys)
 		}
 		rows := dumper.GetRowsFromKeys(db, table, packageKeysData.Keys[exportPoint:upperLimit])
+
 		for _, rowPackage := range rows{
 			path := rowPackage[pathIndex]
 			source := fmt.Sprintf("%s/%s", serverDataFolder, path.Value)
 			target := fmt.Sprintf("%s/%s", outputFolder, path.Value)
-			_, error := copy(source, target)
+			jobs <- fileToCopy{source: source, target: target}
+		}
+		for a := 1; a <= len(rows); a++ {
+			error := <-results
 			if error != nil{
-				log.Fatal("could not Copy File: ", error)
-				panic(error)
+				log.Fatal().Err(error).Msg("Could not Copy File")
 			}
 		}
 		exportPoint = upperLimit
 	}
+	close(jobs)
+	close(results)
+}
+
+type fileToCopy struct {
+	source, target string
+}
+
+func worker(id int, jobs <-chan fileToCopy, results chan<- error) {
+	fmt.Println("worker", id, "started  job")
+	for j := range jobs {
+		results <- systemCopy(j.source, j.target)
+	}
 }
 
-func copy(src, dst string) (int64, error) {
+func systemCopy(src, dest string) error{
 	sourceFileStat, err := os.Stat(src)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
 	if !sourceFileStat.Mode().IsRegular() {
-		return 0, fmt.Errorf("%s is not a regular file", src)
+		return fmt.Errorf("%s is not a regular file", src)
 	}
-
-	source, err := os.Open(src)
-	if err != nil {
-		return 0, err
+	if err := os.MkdirAll(filepath.Dir(dest), 0770); err != nil {
+		return err
 	}
-	defer source.Close()
 
-	destination, err := create(dst)
+	cmd := exec.Command("cp", src, dest)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	err = cmd.Run()
 	if err != nil {
-		return 0, err
+		log.Fatal().Err(err).Msg(fmt.Sprintf("error when copy package file: %s -> %s", src, dest))
+		return err
 	}
-	defer destination.Close()
-	nBytes, err := io.Copy(destination, source)
-	return nBytes, err
+	return nil
 }
-
-func create(p string) (*os.File, error) {
-	if err := os.MkdirAll(filepath.Dir(p), 0770); err != nil {
-		return nil, err
-	}
-	return os.Create(p)
-}