Skip to content

Commit

Permalink
fix: use parallel file download
Browse files Browse the repository at this point in the history
  • Loading branch information
sweatybridge committed Mar 22, 2024
1 parent aadb94f commit be126ee
Showing 1 changed file with 12 additions and 8 deletions.
20 changes: 12 additions & 8 deletions internal/bootstrap/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,10 +197,10 @@ func downloadSample(ctx context.Context, client *github.Client, templateUrl stri
repo := parts[2]
ref := parts[4]
root := strings.Join(parts[5:], "/")
opts := github.RepositoryContentGetOptions{Ref: "heads/" + ref}
opts := github.RepositoryContentGetOptions{Ref: ref}
queue := make([]string, 0)
queue = append(queue, root)
// jq := utils.NewJobQueue(5)
jq := utils.NewJobQueue(5)
for len(queue) > 0 {
contentPath := queue[0]
queue = queue[1:]
Expand All @@ -213,11 +213,15 @@ func downloadSample(ctx context.Context, client *github.Client, templateUrl stri
case "file":
path := strings.TrimPrefix(file.GetPath(), root)
hostPath := filepath.FromSlash("." + path)
content, err := file.GetContent()
if err != nil {
return errors.Errorf("failed to get content: %w", err)
}
if err := utils.WriteFile(hostPath, []byte(content), fsys); err != nil {
fileUrl := file.GetDownloadURL()
if err := jq.Put(func() error {
resp, err := http.Get(fileUrl)
if err != nil {
return errors.Errorf("failed to download content: %w", err)
}
defer resp.Body.Close()
return afero.WriteReader(fsys, hostPath, resp.Body)
}); err != nil {
return err
}
case "dir":
Expand All @@ -227,5 +231,5 @@ func downloadSample(ctx context.Context, client *github.Client, templateUrl stri
}
}
}
return nil
return jq.Collect()
}

0 comments on commit be126ee

Please sign in to comment.