From b6c88b72b1ae20d5a3a09effd8f35bf118398fdb Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Tue, 27 Aug 2024 10:35:58 +0200 Subject: [PATCH 1/2] chore: update go-yaml to latest version (#153) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d59245bd..d5b41bcb 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( golang.org/x/crypto v0.21.0 golang.org/x/term v0.18.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c - gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 + gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/go.sum b/go.sum index 154f94a3..0903cb29 100644 --- a/go.sum +++ b/go.sum @@ -90,7 +90,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc= -gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From ae52f841c4b6ac024077bff6d345a63e153fbc45 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 30 Aug 2024 15:26:40 +0200 Subject: [PATCH 2/2] feat: parse and validate generate property (#143) --- README.md | 7 +- internal/setup/setup.go | 147 +++++++++++++++----- internal/setup/setup_test.go | 253 ++++++++++++++++++++++++++++++++++- 3 files changed, 368 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 58e4fad5..4e448125 100644 --- a/README.md +++ b/README.md @@ -269,11 +269,16 @@ have additional information for identifying the kind of content to expect: which are only available for certain architectures. Example: `/usr/bin/hello: {arch: amd64}` will instruct Chisel to extract and install the "/usr/bin/hello" file only when chiselling an amd64 filesystem. + - **generate**: accepts a `manifest` value to instruct Chisel to generate the + manifest files in the directory. Example: `/var/lib/chisel/**:{generate: + manifest}`. NOTE: the provided path has to be of the form + `/slashed/path/to/dir/**` and no wildcards can appear apart from the trailing + `**`. ## TODO - [ ] Preserve ownerships when possible -- [ ] GPG signature checking for archives +- [x] GPG signature checking for archives - [ ] Use a fake server for the archive tests - [ ] Functional tests diff --git a/internal/setup/setup.go b/internal/setup/setup.go index 6c951637..b4a79710 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -60,11 +60,12 @@ type SliceScripts struct { type PathKind string const ( - DirPath PathKind = "dir" - CopyPath PathKind = "copy" - GlobPath PathKind = "glob" - TextPath PathKind = "text" - SymlinkPath PathKind = "symlink" + DirPath PathKind = "dir" + CopyPath PathKind = "copy" + GlobPath PathKind = "glob" + TextPath PathKind = "text" + SymlinkPath PathKind = "symlink" + GeneratePath PathKind = "generate" // TODO Maybe in the future, for binary support. //Base64Path PathKind = "base64" @@ -77,14 +78,22 @@ const ( UntilMutate PathUntil = "mutate" ) +type GenerateKind string + +const ( + GenerateNone GenerateKind = "" + GenerateManifest GenerateKind = "manifest" +) + type PathInfo struct { Kind PathKind Info string Mode uint - Mutable bool - Until PathUntil - Arch []string + Mutable bool + Until PathUntil + Arch []string + Generate GenerateKind } // SameContent returns whether the path has the same content properties as some @@ -95,7 +104,8 @@ func (pi *PathInfo) SameContent(other *PathInfo) bool { return (pi.Kind == other.Kind && pi.Info == other.Info && pi.Mode == other.Mode && - pi.Mutable == other.Mutable) + pi.Mutable == other.Mutable && + pi.Generate == other.Generate) } type SliceKey struct { @@ -141,10 +151,20 @@ func ReadRelease(dir string) (*Release, error) { func (r *Release) validate() error { keys := []SliceKey(nil) + + // Check for info conflicts and prepare for following checks. A conflict + // means that two slices attempt to extract different files or directories + // to the same location. + // Conflict validation is done without downloading packages which means that + // if we are extracting content from different packages to the same location + // we cannot be sure that it will be the same. On the contrary, content + // extracted from the same package will never conflict because it is + // guaranteed to be the same. + // The above also means that generated content (e.g. text files, directories + // with make:true) will always conflict with extracted content, because we + // cannot validate that they are the same without downloading the package. paths := make(map[string]*Slice) globs := make(map[string]*Slice) - - // Check for info conflicts and prepare for following checks. for _, pkg := range r.Packages { for _, new := range pkg.Slices { keys = append(keys, SliceKey{pkg.Name, new.Name}) @@ -157,36 +177,50 @@ func (r *Release) validate() error { } return fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath) } + // Note: Because for conflict resolution we only check that + // the created file would be the same and we know newInfo and + // oldInfo produce the same one, we do not have to record + // newInfo. } else { - if newInfo.Kind == GlobPath { + paths[newPath] = new + if newInfo.Kind == GeneratePath || newInfo.Kind == GlobPath { globs[newPath] = new } - paths[newPath] = new } } } } - // Check for cycles. - _, err := order(r.Packages, keys) - if err != nil { - return err - } - - // Check for glob conflicts. - for newPath, new := range globs { - for oldPath, old := range paths { - if new.Package == old.Package { + // Check for glob and generate conflicts. + for oldPath, old := range globs { + oldInfo := old.Contents[oldPath] + for newPath, new := range paths { + if oldPath == newPath { + // Identical paths have been filtered earlier. This must be the + // exact same entry. continue } + newInfo := new.Contents[newPath] + if oldInfo.Kind == GlobPath && (newInfo.Kind == GlobPath || newInfo.Kind == CopyPath) { + if new.Package == old.Package { + continue + } + } if strdist.GlobPath(newPath, oldPath) { - if old.Package > new.Package || old.Package == new.Package && old.Name > new.Name { - old, oldPath, new, newPath = new, newPath, old, oldPath + if (old.Package > new.Package) || (old.Package == new.Package && old.Name > new.Name) || + (old.Package == new.Package && old.Name == new.Name && oldPath > newPath) { + old, new = new, old + oldPath, newPath = newPath, oldPath } return fmt.Errorf("slices %s and %s conflict on %s and %s", old, new, oldPath, newPath) } } - paths[newPath] = new + } + + // Check for cycles. + _, err := order(r.Packages, keys) + if err != nil { + return err } return nil @@ -357,8 +391,9 @@ type yamlPath struct { Symlink string `yaml:"symlink"` Mutable bool `yaml:"mutable"` - Until PathUntil `yaml:"until"` - Arch yamlArch `yaml:"arch"` + Until PathUntil `yaml:"until"` + Arch yamlArch `yaml:"arch"` + Generate GenerateKind `yaml:"generate"` } // SameContent returns whether the path has the same content properties as some @@ -583,7 +618,19 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro var mutable bool var until PathUntil var arch []string - if strings.ContainsAny(contPath, "*?") { + var generate GenerateKind + if yamlPath != nil && yamlPath.Generate != "" { + zeroPathGenerate := zeroPath + zeroPathGenerate.Generate = yamlPath.Generate + if !yamlPath.SameContent(&zeroPathGenerate) || yamlPath.Until != UntilNone { + return nil, fmt.Errorf("slice %s_%s path %s has invalid generate options", + pkgName, sliceName, contPath) + } + if _, err := validateGeneratePath(contPath); err != nil { + return nil, fmt.Errorf("slice %s_%s has invalid generate path: %s", pkgName, sliceName, err) + } + kinds = append(kinds, GeneratePath) + } else if strings.ContainsAny(contPath, "*?") { if yamlPath != nil { if !yamlPath.SameContent(&zeroPath) { return nil, fmt.Errorf("slice %s_%s path %s has invalid wildcard options", @@ -595,6 +642,7 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro if yamlPath != nil { mode = yamlPath.Mode mutable = yamlPath.Mutable + generate = yamlPath.Generate if yamlPath.Dir { if !strings.HasSuffix(contPath, "/") { return nil, fmt.Errorf("slice %s_%s path %s must end in / for 'make' to be valid", @@ -644,12 +692,13 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro return nil, fmt.Errorf("slice %s_%s mutable is not a regular file: %s", pkgName, sliceName, contPath) } slice.Contents[contPath] = PathInfo{ - Kind: kinds[0], - Info: info, - Mode: mode, - Mutable: mutable, - Until: until, - Arch: arch, + Kind: kinds[0], + Info: info, + Mode: mode, + Mutable: mutable, + Until: until, + Arch: arch, + Generate: generate, } } @@ -659,6 +708,22 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro return &pkg, err } +// validateGeneratePath validates that the path follows the following format: +// - /slashed/path/to/dir/** +// +// Wildcard characters can only appear at the end as **, and the path before +// those wildcards must be a directory. +func validateGeneratePath(path string) (string, error) { + if !strings.HasSuffix(path, "/**") { + return "", fmt.Errorf("%s does not end with /**", path) + } + dirPath := strings.TrimSuffix(path, "**") + if strings.ContainsAny(dirPath, "*?") { + return "", fmt.Errorf("%s contains wildcard characters in addition to trailing **", path) + } + return dirPath, nil +} + func stripBase(baseDir, path string) string { // Paths must be clean for this to work correctly. return strings.TrimPrefix(path, baseDir+string(filepath.Separator)) @@ -691,9 +756,17 @@ func Select(release *Release, slices []SliceKey) (*Selection, error) { } return nil, fmt.Errorf("slices %s and %s conflict on %s", old, new, newPath) } - continue + } else { + paths[newPath] = new + } + // An invalid "generate" value should only throw an error if that + // particular slice is selected. Hence, the check is here. + switch newInfo.Generate { + case GenerateNone, GenerateManifest: + default: + return nil, fmt.Errorf("slice %s has invalid 'generate' for path %s: %q, consider an update if available", + new, newPath, newInfo.Generate) } - paths[newPath] = new } } diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index 05bfd39c..bec02017 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -1309,7 +1309,258 @@ var setupTests = []setupTest{{ /dir/file: {text: "foo"} `, }, - // TODO this should be an error because the content does not match. + relerror: `slices test-package_myslice1 and test-package_myslice2 conflict on /dir/\*\* and /dir/file`, +}, { + summary: "Specify generate: manifest", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /dir/**: {generate: "manifest"} + `, + }, + release: &setup.Release{ + DefaultArchive: "ubuntu", + + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Archive: "ubuntu", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{ + "myslice": { + Package: "mypkg", + Name: "myslice", + Contents: map[string]setup.PathInfo{ + "/dir/**": {Kind: "generate", Generate: "manifest"}, + }, + }, + }, + }, + }, + }, + selslices: []setup.SliceKey{{"mypkg", "myslice"}}, + selection: &setup.Selection{ + Slices: []*setup.Slice{{ + Package: "mypkg", + Name: "myslice", + Contents: map[string]setup.PathInfo{ + "/dir/**": {Kind: "generate", Generate: "manifest"}, + }, + }}, + }, +}, { + summary: "Can specify generate with bogus value but cannot select those slices", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /dir/**: {generate: "foo"} + `, + }, + release: &setup.Release{ + DefaultArchive: "ubuntu", + + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Archive: "ubuntu", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{ + "myslice": { + Package: "mypkg", + Name: "myslice", + Contents: map[string]setup.PathInfo{ + "/dir/**": {Kind: "generate", Generate: "foo"}, + }, + }, + }, + }, + }, + }, + selslices: []setup.SliceKey{{"mypkg", "myslice"}}, + selerror: `slice mypkg_myslice has invalid 'generate' for path /dir/\*\*: "foo", consider an update if available`, +}, { + summary: "Paths with generate: manifest must have trailing /**", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /path/: {generate: "manifest"} + `, + }, + relerror: `slice mypkg_myslice has invalid generate path: /path/ does not end with /\*\*`, +}, { + summary: "Paths with generate: manifest must not have any other wildcard except the trailing **", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /pat*h/to/dir/**: {generate: "manifest"} + `, + }, + relerror: `slice mypkg_myslice has invalid generate path: /pat\*h/to/dir/\*\* contains wildcard characters in addition to trailing \*\*`, +}, { + summary: "Same paths conflict if one is generate and the other is not", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /path/**: {generate: "manifest"} + `, + "slices/mydir/mypkg2.yaml": ` + package: mypkg2 + slices: + myslice: + contents: + /path/**: + `, + }, + relerror: `slices mypkg_myslice and mypkg2_myslice conflict on /path/\*\*`, +}, { + summary: "Generate paths can be the same across packages", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /path/**: {generate: manifest} + `, + "slices/mydir/mypkg2.yaml": ` + package: mypkg2 + slices: + myslice: + contents: + /path/**: {generate: manifest} + `, + }, + release: &setup.Release{ + DefaultArchive: "ubuntu", + + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy"}, + Components: []string{"main", "universe"}, + PubKeys: []*packet.PublicKey{testKey.PubKey}, + }, + }, + Packages: map[string]*setup.Package{ + "mypkg": { + Archive: "ubuntu", + Name: "mypkg", + Path: "slices/mydir/mypkg.yaml", + Slices: map[string]*setup.Slice{ + "myslice": { + Package: "mypkg", + Name: "myslice", + Contents: map[string]setup.PathInfo{ + "/path/**": {Kind: "generate", Generate: "manifest"}, + }, + }, + }, + }, + "mypkg2": { + Archive: "ubuntu", + Name: "mypkg2", + Path: "slices/mydir/mypkg2.yaml", + Slices: map[string]*setup.Slice{ + "myslice": { + Package: "mypkg2", + Name: "myslice", + Contents: map[string]setup.PathInfo{ + "/path/**": {Kind: "generate", Generate: "manifest"}, + }, + }, + }, + }, + }, + }, +}, { + summary: "Generate paths cannot conflict with any other path", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /path/**: {generate: manifest} + /path/file: + `, + }, + relerror: `slices mypkg_myslice and mypkg_myslice conflict on /path/\*\* and /path/file`, +}, { + summary: "Generate paths cannot conflict with any other path across slices", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice1: + contents: + /path/file: + myslice2: + contents: + /path/**: {generate: manifest} + `, + }, + relerror: `slices mypkg_myslice1 and mypkg_myslice2 conflict on /path/file and /path/\*\*`, +}, { + summary: "Generate paths conflict with other generate paths", + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice1: + contents: + /path/subdir/**: {generate: manifest} + myslice2: + contents: + /path/**: {generate: manifest} + `, + }, + relerror: `slices mypkg_myslice1 and mypkg_myslice2 conflict on /path/subdir/\*\* and /path/\*\*`, +}, { + summary: `No other options in "generate" paths`, + input: map[string]string{ + "slices/mydir/mypkg.yaml": ` + package: mypkg + slices: + myslice: + contents: + /path/**: {generate: "manifest", until: mutate} + `, + }, + relerror: `slice mypkg_myslice path /path/\*\* has invalid generate options`, }} var defaultChiselYaml = `