diff --git a/dumper/packageDumper/packageDumper.go b/dumper/packageDumper/packageDumper.go index f95fa37..d9733dc 100644 --- a/dumper/packageDumper/packageDumper.go +++ b/dumper/packageDumper/packageDumper.go @@ -3,11 +3,11 @@ package packageDumper import ( "database/sql" "fmt" + "github.com/rs/zerolog/log" "github.com/uyuni-project/inter-server-sync/dumper" "github.com/uyuni-project/inter-server-sync/schemareader" - "io" - "log" "os" + "os/exec" "path/filepath" ) @@ -21,54 +21,69 @@ func DumpPackageFiles(db *sql.DB, schemaMetadata map[string]schemareader.Table, exportPoint := 0 batchSize := 500 + jobs := make(chan fileToCopy, batchSize) + results := make(chan error, batchSize) + for w := 1; w <= 5; w++ { + go worker(w, jobs, results) + } + for len(packageKeysData.Keys) > exportPoint { upperLimit := exportPoint + batchSize if upperLimit > len(packageKeysData.Keys) { upperLimit = len(packageKeysData.Keys) } rows := dumper.GetRowsFromKeys(db, table, packageKeysData.Keys[exportPoint:upperLimit]) + for _, rowPackage := range rows{ path := rowPackage[pathIndex] source := fmt.Sprintf("%s/%s", serverDataFolder, path.Value) target := fmt.Sprintf("%s/%s", outputFolder, path.Value) - _, error := copy(source, target) + jobs <- fileToCopy{source: source, target: target} + } + for a := 1; a <= len(rows); a++ { + error := <-results if error != nil{ - log.Fatal("could not Copy File: ", error) - panic(error) + log.Fatal().Err(error).Msg("Could not Copy File") } } exportPoint = upperLimit } + close(jobs) + close(results) +} + +type fileToCopy struct { + source, target string +} + +func worker(id int, jobs <-chan fileToCopy, results chan<- error) { + fmt.Println("worker", id, "started job") + for j := range jobs { + results <- systemCopy(j.source, j.target) + } } -func copy(src, dst string) (int64, error) { +func systemCopy(src, dest string) error{ sourceFileStat, err := os.Stat(src) if err != nil { - return 0, err + return err } if !sourceFileStat.Mode().IsRegular() { - return 0, fmt.Errorf("%s is not a regular file", src) + return fmt.Errorf("%s is not a regular file", src) } - - source, err := os.Open(src) - if err != nil { - return 0, err + if err := os.MkdirAll(filepath.Dir(dest), 0770); err != nil { + return err } - defer source.Close() - destination, err := create(dst) + cmd := exec.Command("cp", src, dest) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + err = cmd.Run() if err != nil { - return 0, err + log.Fatal().Err(err).Msg(fmt.Sprintf("error when copy package file: %s -> %s", src, dest)) + return err } - defer destination.Close() - nBytes, err := io.Copy(destination, source) - return nBytes, err + return nil } - -func create(p string) (*os.File, error) { - if err := os.MkdirAll(filepath.Dir(p), 0770); err != nil { - return nil, err - } - return os.Create(p) -} \ No newline at end of file