From c551dc317d871459c3e64eefa6a064dea10573d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Sun, 18 Jun 2023 13:13:58 +0200 Subject: [PATCH] Select latest package from multiple archives In the future, we'd like to support different kinds of archives which have a non-empty intersection of packages. In other words, we'd like to support archives that partly override the Ubuntu archive. Currently, each sliced package is tied to an archive via the "archive" property that defaults to the default archive. Drop this link from the sliced packages to the archives and select the latest available version of the package from all configured archives. This commit, in effect, doesn't change the current behavior because we don't allow archives to define their URLs. It's only preparation for future commits that will add support for different types of archives. --- internal/setup/setup.go | 27 +++------- internal/setup/setup_test.go | 61 +++++++--------------- internal/slicer/slicer.go | 21 +++++--- internal/slicer/slicer_test.go | 94 ++++++++++++++++++++++++++++++++-- 4 files changed, 129 insertions(+), 74 deletions(-) diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 451bcc3c..04c3ce74 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -18,10 +18,9 @@ import ( // Release is a collection of package slices targeting a particular // distribution version. type Release struct { - Path string - Packages map[string]*Package - Archives map[string]*Archive - DefaultArchive string + Path string + Packages map[string]*Package + Archives map[string]*Archive } // Archive is the location from which binary packages are obtained. @@ -34,10 +33,9 @@ type Archive struct { // Package holds a collection of slices that represent parts of themselves. type Package struct { - Name string - Path string - Archive string - Slices map[string]*Slice + Name string + Path string + Slices map[string]*Slice } // Slice holds the details about a package slice. @@ -306,9 +304,6 @@ func readSlices(release *Release, baseDir, dirName string) error { if err != nil { return err } - if pkg.Archive == "" { - pkg.Archive = release.DefaultArchive - } release.Packages[pkg.Name] = pkg } @@ -326,7 +321,6 @@ type yamlArchive struct { Version string `yaml:"version"` Suites []string `yaml:"suites"` Components []string `yaml:"components"` - Default bool `yaml:"default"` } type yamlPackage struct { @@ -428,14 +422,6 @@ func parseRelease(baseDir, filePath string, data []byte) (*Release, error) { if len(details.Components) == 0 { return nil, fmt.Errorf("%s: archive %q missing components field", fileName, archiveName) } - if len(yamlVar.Archives) == 1 { - details.Default = true - } else if details.Default && release.DefaultArchive != "" { - return nil, fmt.Errorf("%s: more than one default archive: %s, %s", fileName, release.DefaultArchive, archiveName) - } - if details.Default { - release.DefaultArchive = archiveName - } release.Archives[archiveName] = &Archive{ Name: archiveName, Version: details.Version, @@ -464,7 +450,6 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro if yamlPkg.Name != pkg.Name { return nil, fmt.Errorf("%s: filename and 'package' field (%q) disagree", pkgPath, yamlPkg.Name) } - pkg.Archive = yamlPkg.Archive zeroPath := yamlPath{} for sliceName, yamlSlice := range yamlPkg.Slices { diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 7ffae74f..49b32f2a 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -60,8 +60,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -72,10 +70,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, }, @@ -103,8 +100,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -115,9 +110,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -164,8 +158,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -176,9 +168,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -424,8 +415,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -436,9 +425,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice1": { Package: "mypkg", @@ -638,8 +626,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -650,9 +636,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -677,8 +662,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -689,9 +672,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -717,8 +699,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "ubuntu", - Archives: map[string]*setup.Archive{ "ubuntu": { Name: "ubuntu", @@ -729,9 +709,8 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "ubuntu", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", Slices: map[string]*setup.Slice{ "myslice": { Package: "mypkg", @@ -755,7 +734,6 @@ var setupTests = []setupTest{{ version: 22.04 components: [main, universe] suites: [jammy] - default: true bar: version: 22.04 components: [universe] @@ -766,8 +744,6 @@ var setupTests = []setupTest{{ `, }, release: &setup.Release{ - DefaultArchive: "foo", - Archives: map[string]*setup.Archive{ "foo": { Name: "foo", @@ -784,10 +760,9 @@ var setupTests = []setupTest{{ }, Packages: map[string]*setup.Package{ "mypkg": { - Archive: "foo", - Name: "mypkg", - Path: "slices/mydir/mypkg.yaml", - Slices: map[string]*setup.Slice{}, + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{}, }, }, }, diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 2b684b5d..68a677f4 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -59,7 +59,6 @@ func Run(options *RunOptions) error { syscall.Umask(oldUmask) }() - release := options.Selection.Release targetDir := filepath.Clean(options.TargetDir) targetDirAbs := targetDir if !filepath.IsAbs(targetDirAbs) { @@ -74,15 +73,23 @@ func Run(options *RunOptions) error { for _, slice := range options.Selection.Slices { extractPackage := extract[slice.Package] if extractPackage == nil { - archiveName := release.Packages[slice.Package].Archive - archive := options.Archives[archiveName] - if archive == nil { - return fmt.Errorf("archive %q not defined", archiveName) + var selectedVersion string + var selectedArchive archive.Archive + for _, currentArchive := range options.Archives { + pkgInfo := currentArchive.Info(slice.Package) + if pkgInfo == nil { + continue + } + currentVersion := pkgInfo.Version() + if selectedVersion == "" || deb.CompareVersions(selectedVersion, currentVersion) < 0 { + selectedVersion = currentVersion + selectedArchive = currentArchive + } } - if !archive.Exists(slice.Package) { + if selectedVersion == "" { return fmt.Errorf("slice package %q missing from archive", slice.Package) } - archives[slice.Package] = archive + archives[slice.Package] = selectedArchive extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage } diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index cdb68f12..c8e8e21b 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -508,7 +508,6 @@ var slicerTests = []slicerTest{{ foo: version: 22.04 components: [main, universe] - default: true bar: version: 22.04 components: [main] @@ -562,7 +561,6 @@ var slicerTests = []slicerTest{{ version: 1 suites: [main] components: [main, universe] - default: true hadrons: version: 1 suites: [main] @@ -577,7 +575,6 @@ var slicerTests = []slicerTest{{ `, "slices/mydir/proton.yaml": ` package: proton - archive: hadrons slices: mass: contents: @@ -598,6 +595,97 @@ var slicerTests = []slicerTest{{ "/usr/share/doc/electron/": "dir 0755", "/usr/share/doc/electron/copyright": "file 0644 empty", }, +}, { + summary: "Can pick latest packages from multiple archives", + pkgs: map[string]map[string]testPackage{ + "vertebrates": { + "cheetah": testPackage{ + info: map[string]string{ + "Version": "109.4", + }, + content: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0755, "./"), + Dir(0755, "./speed/"), + Reg(0644, "./speed/cheetah", "109.4 km/h\n"), + }), + }, + "ostrich": testPackage{ + info: map[string]string{ + "Version": "100.0", + }, + content: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0755, "./"), + Dir(0755, "./speed/"), + Reg(0644, "./speed/ostrich", "100.0 km/h\n"), + }), + }, + }, + "mammals": { + "cheetah": testPackage{ + info: map[string]string{ + "Version": "120.7", + }, + content: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0755, "./"), + Dir(0755, "./speed/"), + Reg(0644, "./speed/cheetah", "120.7 km/h\n"), + }), + }, + }, + "birds": { + "ostrich": testPackage{ + info: map[string]string{ + "Version": "90.0", + }, + content: testutil.MustMakeDeb([]testutil.TarEntry{ + Dir(0755, "./"), + Dir(0755, "./speed/"), + Reg(0644, "./speed/ostrich", "90.0 km/h\n"), + }), + }, + }, + }, + slices: []setup.SliceKey{ + {"cheetah", "speed"}, + {"ostrich", "speed"}, + }, + release: map[string]string{ + "chisel.yaml": ` + format: chisel-v1 + archives: + vertebrates: + version: 1 + suites: [main] + components: [main, universe] + mammals: + version: 1 + suites: [main] + components: [main] + birds: + version: 1 + suites: [main] + components: [main] + `, + "slices/mydir/cheetah.yaml": ` + package: cheetah + slices: + speed: + contents: + /speed/cheetah: + `, + "slices/mydir/ostrich.yaml": ` + package: ostrich + slices: + speed: + contents: + /speed/ostrich: + `, + }, + result: map[string]string{ + "/speed/": "dir 0755", + "/speed/cheetah": "file 0644 e98b0879", + "/speed/ostrich": "file 0644 c8fa2806", + }, }} const defaultChiselYaml = `