Skip to content

Commit

Permalink
Merge pull request #957 from jonjohnsonjr/removed-graph
Browse files Browse the repository at this point in the history 
Parse configs in parallel in NewPackages
  • Loading branch information
@luhring
luhring authored Jun 13, 2024
2 parents c3c8c96 + 44c5464 commit dc84d81
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 64 deletions.
4 changes: 2 additions & 2 deletions pkg/cli/text.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func cmdText() *cobra.Command {
return err
}

return text(*g, pkgs, arch, textType(t), os.Stdout)
return text(g, pkgs, arch, textType(t), os.Stdout)
},
}
text.Flags().StringVarP(&dir, "dir", "d", ".", "directory to search for melange configs")
Expand Down Expand Up @@ -69,7 +69,7 @@ var textTypes = []textType{
typePackageNameAndVersion,
}

func text(g dag.Graph, pkgs *dag.Packages, arch string, t textType, w io.Writer) error {
func text(g *dag.Graph, pkgs *dag.Packages, arch string, t textType, w io.Writer) error {
filtered, err := g.Filter(dag.FilterLocal())
if err != nil {
return err
Expand Down
145 changes: 83 additions & 62 deletions pkg/dag/packages.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,17 @@ import (
"fmt"
"io/fs"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"

"chainguard.dev/apko/pkg/apk/apk"
"chainguard.dev/melange/pkg/build"
"chainguard.dev/melange/pkg/config"
"github.com/chainguard-dev/clog"
"golang.org/x/sync/errgroup"
)

const (
Expand Down Expand Up @@ -72,6 +75,7 @@ func (c Configuration) Resolved() bool {
// It does not try to determine relationships and dependencies between packages. For that,
// pass a Packages to NewGraph.
type Packages struct {
sync.Mutex
configs map[string][]*Configuration
packages map[string][]*Configuration
index map[string]*Configuration
Expand All @@ -80,6 +84,9 @@ type Packages struct {
var ErrMultipleConfigurations = fmt.Errorf("multiple configurations using the same package name")

func (p *Packages) addPackage(name string, configuration *Configuration) error {
p.Lock()
defer p.Unlock()

if _, exists := p.packages[name]; exists {
return fmt.Errorf("%s: %w", name, ErrMultipleConfigurations)
}
Expand All @@ -90,6 +97,9 @@ func (p *Packages) addPackage(name string, configuration *Configuration) error {
}

func (p *Packages) addConfiguration(name string, configuration *Configuration) error {
p.Lock()
defer p.Unlock()

p.configs[name] = append(p.configs[name], configuration)
p.index[configuration.String()] = configuration

Expand Down Expand Up @@ -151,6 +161,9 @@ func NewPackages(ctx context.Context, fsys fs.FS, dirPath, pipelineDir string) (
packages: make(map[string][]*Configuration),
index: make(map[string]*Configuration),
}

var g errgroup.Group
g.SetLimit(runtime.GOMAXPROCS(0))
err := fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
Expand Down Expand Up @@ -182,78 +195,86 @@ func NewPackages(ctx context.Context, fsys fs.FS, dirPath, pipelineDir string) (
return nil
}

p := filepath.Join(dirPath, path)
buildc, err := config.ParseConfiguration(ctx, p)
if err != nil {
return err
}
c := &Configuration{
Configuration: buildc,
Path: p,
name: buildc.Package.Name,
version: fullVersion(&buildc.Package),
pkg: buildc.Package.Name,
}

name := c.name
if name == "" {
return fmt.Errorf("no package name in %q", path)
}
if err := pkgs.addConfiguration(name, c); err != nil {
return err
}
if err := pkgs.addPackage(name, c); err != nil {
return err
}
if err := pkgs.addProvides(c, c.Package.Dependencies.Provides); err != nil {
return err
}

for i := range c.Subpackages {
subpkg := c.Subpackages[i]
name := subpkg.Name
if name == "" {
return fmt.Errorf("empty subpackage name at index %d for package %q", i, c.Package.Name)
g.Go(func() error {
p := filepath.Join(dirPath, path)
buildc, err := config.ParseConfiguration(ctx, p)
if err != nil {
return err
}
c := &Configuration{
Configuration: buildc,
Path: p,
name: name,
version: fullVersion(&buildc.Package), // subpackages have same version as origin
pkg: name,
name: buildc.Package.Name,
version: fullVersion(&buildc.Package),
pkg: buildc.Package.Name,
}

name := c.name
if name == "" {
return fmt.Errorf("no package name in %q", path)
}
if err := pkgs.addConfiguration(name, c); err != nil {
return err
}
if err := pkgs.addProvides(c, subpkg.Dependencies.Provides); err != nil {
if err := pkgs.addPackage(name, c); err != nil {
return err
}
if err := pkgs.addProvides(c, c.Package.Dependencies.Provides); err != nil {
return err
}

// TODO: resolve deps via `uses` for subpackage pipelines.
}
// Resolve all `uses` used by the pipeline. This updates the set of
// .environment.contents.packages so the next block can include those as build deps.
pctx := &build.PipelineBuild{
Build: &build.Build{
PipelineDirs: []string{pipelineDir},
Configuration: *c.Configuration,
},
Package: &c.Package,
}
for i := range c.Pipeline {
s := &build.PipelineContext{Environment: &pctx.Build.Configuration.Environment, PipelineDirs: []string{pipelineDir}, Pipeline: &c.Pipeline[i]}
if err := s.ApplyNeeds(ctx, pctx); err != nil {
return fmt.Errorf("unable to resolve needs for package %s: %w", name, err)
for i := range c.Subpackages {
subpkg := c.Subpackages[i]
name := subpkg.Name
if name == "" {
return fmt.Errorf("empty subpackage name at index %d for package %q", i, c.Package.Name)
}
c := &Configuration{
Configuration: buildc,
Path: p,
name: name,
version: fullVersion(&buildc.Package), // subpackages have same version as origin
pkg: name,
}
if err := pkgs.addConfiguration(name, c); err != nil {
return err
}
if err := pkgs.addProvides(c, subpkg.Dependencies.Provides); err != nil {
return err
}

// TODO: resolve deps via `uses` for subpackage pipelines.
}
// Resolve all `uses` used by the pipeline. This updates the set of
// .environment.contents.packages so the next block can include those as build deps.
pctx := &build.PipelineBuild{
Build: &build.Build{
PipelineDirs: []string{pipelineDir},
Configuration: *c.Configuration,
},
Package: &c.Package,
}
for i := range c.Pipeline {
s := &build.PipelineContext{Environment: &pctx.Build.Configuration.Environment, PipelineDirs: []string{pipelineDir}, Pipeline: &c.Pipeline[i]}
if err := s.ApplyNeeds(ctx, pctx); err != nil {
return fmt.Errorf("unable to resolve needs for package %s: %w", name, err)
}
c.Environment.Contents.Packages = pctx.Build.Configuration.Environment.Contents.Packages
}
c.Environment.Contents.Packages = pctx.Build.Configuration.Environment.Contents.Packages
}

return nil
})

return nil
})
if err != nil {
return nil, err
}

if err := g.Wait(); err != nil {
return nil, err
}

return pkgs, nil
}

Expand All @@ -262,7 +283,7 @@ func NewPackages(ctx context.Context, fsys fs.FS, dirPath, pipelineDir string) (
// it's not present, Config returns an empty list.
//
// Pass packageOnly=true to restruct it just to origin package names.
func (p Packages) Config(name string, packageOnly bool) []*Configuration {
func (p *Packages) Config(name string, packageOnly bool) []*Configuration {
if p.configs == nil {
// this would be unexpected
return nil
Expand Down Expand Up @@ -290,7 +311,7 @@ func (p Packages) Config(name string, packageOnly bool) []*Configuration {
return list
}

func (p Packages) ConfigByKey(key string) *Configuration {
func (p *Packages) ConfigByKey(key string) *Configuration {
if len(p.index) == 0 {
return nil
}
Expand All @@ -302,7 +323,7 @@ func (p Packages) ConfigByKey(key string) *Configuration {
}

// PkgConfig returns the melange Configuration for a given package name.
func (p Packages) PkgConfig(pkgName string) *Configuration {
func (p *Packages) PkgConfig(pkgName string) *Configuration {
for _, cfg := range p.packages[pkgName] {
if pkgName == cfg.Package.Name {
return cfg
Expand All @@ -313,7 +334,7 @@ func (p Packages) PkgConfig(pkgName string) *Configuration {

// PkgInfo returns the build.Package struct for a given package name.
// If no such package name is found in the packages, return nil package and nil error.
func (p Packages) PkgInfo(pkgName string) *config.Package {
func (p *Packages) PkgInfo(pkgName string) *config.Package {
if cfg := p.PkgConfig(pkgName); cfg != nil {
return &cfg.Package
}
Expand All @@ -322,7 +343,7 @@ func (p Packages) PkgInfo(pkgName string) *config.Package {

// Packages returns a slice of every package and subpackage available in the Packages struct,
// sorted alphabetically and then by version, with each package converted to a *apk.RepositoryPackage.
func (p Packages) Packages() []*Configuration {
func (p *Packages) Packages() []*Configuration {
allPackages := make([]*Configuration, 0, len(p.packages))
for _, byVersion := range p.packages {
allPackages = append(allPackages, byVersion...)
Expand All @@ -339,7 +360,7 @@ func (p Packages) Packages() []*Configuration {
}

// PackageNames returns a slice of the names of all packages, sorted alphabetically.
func (p Packages) PackageNames() []string {
func (p *Packages) PackageNames() []string {
allPackages := make([]string, 0, len(p.packages))
for name := range p.packages {
allPackages = append(allPackages, name)
Expand All @@ -354,7 +375,7 @@ func (p Packages) PackageNames() []string {
// If a listed element is a provides, automatically includes the origin package that provides it.
// If a listed element is a subpackage, automatically includes the origin package that contains it.
// If a listed element does not exist, returns an error.
func (p Packages) Sub(names ...string) (*Packages, error) {
func (p *Packages) Sub(names ...string) (*Packages, error) {
pkgs := &Packages{
configs: make(map[string][]*Configuration),
index: make(map[string]*Configuration),
Expand Down Expand Up @@ -392,7 +413,7 @@ func wantArch(have string, want []string) bool {
}

// WithArch returns a new Packages whose members are valid for the given arch.
func (p Packages) WithArch(arch string) (*Packages, error) {
func (p *Packages) WithArch(arch string) (*Packages, error) {
pkgs := &Packages{
configs: make(map[string][]*Configuration),
index: p.index,
Expand Down Expand Up @@ -425,7 +446,7 @@ func (p Packages) WithArch(arch string) (*Packages, error) {

// Repository provide the Packages as a apk.RepositoryWithIndex. To be used in other places that require
// using alpine/go structs instead of ours.
func (p Packages) Repository(arch string) apk.NamedIndex {
func (p *Packages) Repository(arch string) apk.NamedIndex {
repo := apk.NewRepositoryFromComponents(Local, "latest", "", arch)

// Precompute the number of packages to avoid growslice.
Expand Down

0 comments on commit dc84d81

Please sign in to comment.