diff --git a/cmd/chisel/cmd_help.go b/cmd/chisel/cmd_help.go index a0ebd273..d164d99f 100644 --- a/cmd/chisel/cmd_help.go +++ b/cmd/chisel/cmd_help.go @@ -155,7 +155,7 @@ type helpCategory struct { var helpCategories = []helpCategory{{ Label: "Basic", Description: "general operations", - Commands: []string{"find", "help", "version"}, + Commands: []string{"find", "info", "help", "version"}, }, { Label: "Action", Description: "make things happen", diff --git a/cmd/chisel/cmd_info.go b/cmd/chisel/cmd_info.go new file mode 100644 index 00000000..6c53e4e3 --- /dev/null +++ b/cmd/chisel/cmd_info.go @@ -0,0 +1,137 @@ +package main + +import ( + "fmt" + "strconv" + "strings" + + "github.com/jessevdk/go-flags" + "gopkg.in/yaml.v3" + + "github.com/canonical/chisel/internal/setup" +) + +var shortInfoHelp = "Show information about package slices" +var longInfoHelp = ` +The info command shows detailed information about package slices. + +It accepts a whitespace-separated list of strings. The list can be +composed of package names, slice names, or a combination of both. The +default output format is YAML. When multiple arguments are provided, +the output is a list of YAML documents separated by a "---" line. + +Slice definitions are shown verbatim according to their definition in +the selected release. For example, globs are not expanded. +` + +var infoDescs = map[string]string{ + "release": "Chisel release name or directory (e.g. ubuntu-22.04)", +} + +type infoCmd struct { + Release string `long:"release" value-name:""` + + Positional struct { + Queries []string `positional-arg-name:"" required:"yes"` + } `positional-args:"yes"` +} + +func init() { + addCommand("info", shortInfoHelp, longInfoHelp, func() flags.Commander { return &infoCmd{} }, infoDescs, nil) +} + +func (cmd *infoCmd) Execute(args []string) error { + if len(args) > 0 { + return ErrExtraArgs + } + + release, err := obtainRelease(cmd.Release) + if err != nil { + return err + } + + packages, notFound := selectPackageSlices(release, cmd.Positional.Queries) + + for i, pkg := range packages { + data, err := yaml.Marshal(pkg) + if err != nil { + return err + } + if i > 0 { + fmt.Fprintln(Stdout, "---") + } + fmt.Fprint(Stdout, string(data)) + } + + if len(notFound) > 0 { + for i := range notFound { + notFound[i] = strconv.Quote(notFound[i]) + } + return fmt.Errorf("no slice definitions found for: " + strings.Join(notFound, ", ")) + } + + return nil +} + +// selectPackageSlices takes in a release and a list of query strings +// of package names and/or slice names, and returns a list of packages +// containing the found slices. It also returns a list of query +// strings that were not found. +func selectPackageSlices(release *setup.Release, queries []string) (packages []*setup.Package, notFound []string) { + var pkgOrder []string + pkgSlices := make(map[string][]string) + allPkgSlices := make(map[string]bool) + + sliceExists := func(key setup.SliceKey) bool { + pkg, ok := release.Packages[key.Package] + if !ok { + return false + } + _, ok = pkg.Slices[key.Slice] + return ok + } + for _, query := range queries { + var pkg, slice string + if strings.Contains(query, "_") { + key, err := setup.ParseSliceKey(query) + if err != nil || !sliceExists(key) { + notFound = append(notFound, query) + continue + } + pkg, slice = key.Package, key.Slice + } else { + if _, ok := release.Packages[query]; !ok { + notFound = append(notFound, query) + continue + } + pkg = query + } + if len(pkgSlices[pkg]) == 0 && !allPkgSlices[pkg] { + pkgOrder = append(pkgOrder, pkg) + } + if slice == "" { + allPkgSlices[pkg] = true + } else { + pkgSlices[pkg] = append(pkgSlices[pkg], slice) + } + } + + for _, pkgName := range pkgOrder { + var pkg *setup.Package + if allPkgSlices[pkgName] { + pkg = release.Packages[pkgName] + } else { + releasePkg := release.Packages[pkgName] + pkg = &setup.Package{ + Name: releasePkg.Name, + Archive: releasePkg.Archive, + Slices: make(map[string]*setup.Slice), + } + for _, sliceName := range pkgSlices[pkgName] { + pkg.Slices[sliceName] = releasePkg.Slices[sliceName] + } + } + packages = append(packages, pkg) + } + return packages, notFound +} diff --git a/cmd/chisel/cmd_info_test.go b/cmd/chisel/cmd_info_test.go new file mode 100644 index 00000000..c78a477c --- /dev/null +++ b/cmd/chisel/cmd_info_test.go @@ -0,0 +1,234 @@ +package main_test + +import ( + "os" + "path/filepath" + "strings" + + . "gopkg.in/check.v1" + + chisel "github.com/canonical/chisel/cmd/chisel" + "github.com/canonical/chisel/internal/testutil" +) + +type infoTest struct { + summary string + input map[string]string + query []string + err string + stdout string +} + +var infoTests = []infoTest{{ + summary: "A single slice inspection", + input: infoRelease, + query: []string{"mypkg1_myslice1"}, + stdout: ` + package: mypkg1 + archive: ubuntu + slices: + myslice1: + contents: + /dir/file: {} + `, +}, { + summary: "A single package inspection", + input: infoRelease, + query: []string{"mypkg2"}, + stdout: ` + package: mypkg2 + archive: ubuntu + slices: + myslice: + contents: + /dir/another-file: {} + `, +}, { + summary: "Multiple slices within the same package", + input: infoRelease, + query: []string{"mypkg1_myslice2", "mypkg1_myslice1"}, + stdout: ` + package: mypkg1 + archive: ubuntu + slices: + myslice1: + contents: + /dir/file: {} + myslice2: + essential: + - mypkg1_myslice1 + - mypkg2_myslice + `, +}, { + summary: "Packages and slices", + input: infoRelease, + query: []string{"mypkg1_myslice1", "mypkg2", "mypkg1_myslice2"}, + stdout: ` + package: mypkg1 + archive: ubuntu + slices: + myslice1: + contents: + /dir/file: {} + myslice2: + essential: + - mypkg1_myslice1 + - mypkg2_myslice + --- + package: mypkg2 + archive: ubuntu + slices: + myslice: + contents: + /dir/another-file: {} + `, +}, { + summary: "Package and its slices", + input: infoRelease, + query: []string{"mypkg1_myslice1", "mypkg1"}, + stdout: ` + package: mypkg1 + archive: ubuntu + slices: + myslice1: + contents: + /dir/file: {} + myslice2: + essential: + - mypkg1_myslice1 + - mypkg2_myslice + `, +}, { + summary: "Same slice appearing multiple times", + input: infoRelease, + query: []string{"mypkg1_myslice1", "mypkg1_myslice1", "mypkg1_myslice1"}, + stdout: ` + package: mypkg1 + archive: ubuntu + slices: + myslice1: + contents: + /dir/file: {} + `, +}, { + summary: "No slices found", + input: infoRelease, + query: []string{"foo", "bar_foo"}, + err: `no slice definitions found for: "foo", "bar_foo"`, +}, { + summary: "Some slices found, others not found", + input: infoRelease, + query: []string{"foo", "mypkg1_myslice1", "bar_foo"}, + stdout: ` + package: mypkg1 + archive: ubuntu + slices: + myslice1: + contents: + /dir/file: {} + /dir/sub-dir/: {make: true, mode: 0644} + `, + err: `no slice definitions found for: "foo", "bar_foo"`, +}, { + summary: "No args", + input: infoRelease, + err: "the required argument ` (at least 1 argument)` was not provided", +}, { + summary: "Empty, whitespace args", + input: infoRelease, + query: []string{"", " "}, + err: `no slice definitions found for: "", " "`, +}, { + summary: "Ignore invalid slice names", + input: infoRelease, + query: []string{"foo_bar_foo", "a_b", "7_c", "a_b c", "a_b x_y"}, + err: `no slice definitions found for: "foo_bar_foo", "a_b", "7_c", "a_b c", "a_b x_y"`, +}} + +var testKey = testutil.PGPKeys["key1"] + +var defaultChiselYaml = ` + format: chisel-v1 + archives: + ubuntu: + version: 22.04 + components: [main, universe] + v1-public-keys: [test-key] + v1-public-keys: + test-key: + id: ` + testKey.ID + ` + armor: |` + "\n" + testutil.PrefixEachLine(testKey.PubKeyArmor, "\t\t\t\t\t\t") + +var infoRelease = map[string]string{ + "chisel.yaml": string(defaultChiselYaml), + "slices/mypkg1.yaml": ` + package: mypkg1 + essential: + - mypkg1_myslice1 + slices: + myslice1: + contents: + /dir/file: + myslice2: + essential: + - mypkg2_myslice + `, + "slices/mypkg2.yaml": ` + package: mypkg2 + slices: + myslice: + contents: + /dir/another-file: + `, + "slices/mypkg3.yaml": ` + package: mypkg3 + essential: + - mypkg1_myslice1 + slices: + myslice: + essential: + - mypkg2_myslice + contents: + /dir/other-file: + /dir/glob*: + /dir/sub-dir/: {make: true, mode: 0644} + /dir/copy: {copy: /dir/file} + /dir/symlink: {symlink: /dir/file} + /dir/mutable: {text: TODO, mutable: true, arch: riscv64} + /dir/arch-specific*: {arch: [amd64,arm64,i386]} + /dir/until: {until: mutate} + /dir/unfolded: + copy: /dir/file + mode: 0644 + mutate: | + # Test multi-line string. + content.write("/dir/mutable", foo) + `, +} + +func (s *ChiselSuite) TestInfoCommand(c *C) { + for _, test := range infoTests { + c.Logf("Summary: %s", test.summary) + + s.ResetStdStreams() + + dir := c.MkDir() + for path, data := range test.input { + fpath := filepath.Join(dir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + c.Assert(err, IsNil) + } + test.query = append([]string{"info", "--release", dir}, test.query...) + + _, err := chisel.Parser().ParseArgs(test.query) + if test.err != "" { + c.Assert(err, ErrorMatches, test.err) + continue + } + c.Assert(err, IsNil) + test.stdout = string(testutil.Reindent(test.stdout)) + c.Assert(s.Stdout(), Equals, strings.TrimSpace(test.stdout)+"\n") + } +} diff --git a/internal/setup/setup.go b/internal/setup/setup.go index b4a79710..e604786d 100644 --- a/internal/setup/setup.go +++ b/internal/setup/setup.go @@ -44,6 +44,12 @@ type Package struct { Slices map[string]*Slice } +func (p *Package) MarshalYAML() (interface{}, error) { + return packageToYAML(p) +} + +var _ yaml.Marshaler = (*Package)(nil) + // Slice holds the details about a package slice. type Slice struct { Package string @@ -378,24 +384,36 @@ type yamlArchive struct { type yamlPackage struct { Name string `yaml:"package"` - Archive string `yaml:"archive"` - Essential []string `yaml:"essential"` - Slices map[string]yamlSlice `yaml:"slices"` + Archive string `yaml:"archive,omitempty"` + Essential []string `yaml:"essential,omitempty"` + Slices map[string]yamlSlice `yaml:"slices,omitempty"` } type yamlPath struct { - Dir bool `yaml:"make"` - Mode uint `yaml:"mode"` - Copy string `yaml:"copy"` - Text *string `yaml:"text"` - Symlink string `yaml:"symlink"` - Mutable bool `yaml:"mutable"` + Dir bool `yaml:"make,omitempty"` + Mode yamlMode `yaml:"mode,omitempty"` + Copy string `yaml:"copy,omitempty"` + Text *string `yaml:"text,omitempty"` + Symlink string `yaml:"symlink,omitempty"` + Mutable bool `yaml:"mutable,omitempty"` + Until PathUntil `yaml:"until,omitempty"` + Arch yamlArch `yaml:"arch,omitempty"` + Generate GenerateKind `yaml:"generate,omitempty"` +} - Until PathUntil `yaml:"until"` - Arch yamlArch `yaml:"arch"` - Generate GenerateKind `yaml:"generate"` +func (yp *yamlPath) MarshalYAML() (interface{}, error) { + type flowPath *yamlPath + node := &yaml.Node{} + err := node.Encode(flowPath(yp)) + if err != nil { + return nil, err + } + node.Style |= yaml.FlowStyle + return node, nil } +var _ yaml.Marshaler = (*yamlPath)(nil) + // SameContent returns whether the path has the same content properties as some // other path. In other words, the resulting file/dir entry is the same. The // Mutable flag must also match, as that's a common agreement that the actual @@ -410,16 +428,16 @@ func (yp *yamlPath) SameContent(other *yamlPath) bool { } type yamlArch struct { - list []string + List []string } func (ya *yamlArch) UnmarshalYAML(value *yaml.Node) error { var s string var l []string if value.Decode(&s) == nil { - ya.list = []string{s} + ya.List = []string{s} } else if value.Decode(&l) == nil { - ya.list = l + ya.List = l } else { return fmt.Errorf("cannot decode arch") } @@ -427,10 +445,35 @@ func (ya *yamlArch) UnmarshalYAML(value *yaml.Node) error { return nil } +func (ya yamlArch) MarshalYAML() (interface{}, error) { + if len(ya.List) == 1 { + return ya.List[0], nil + } + return ya.List, nil +} + +var _ yaml.Marshaler = yamlArch{} + +type yamlMode uint + +func (ym yamlMode) MarshalYAML() (interface{}, error) { + // Workaround for marshalling integers in octal format. + // Ref: https://github.com/go-yaml/yaml/issues/420. + node := &yaml.Node{} + err := node.Encode(uint(ym)) + if err != nil { + return nil, err + } + node.Value = fmt.Sprintf("0%o", ym) + return node, nil +} + +var _ yaml.Marshaler = yamlMode(0) + type yamlSlice struct { - Essential []string `yaml:"essential"` - Contents map[string]*yamlPath `yaml:"contents"` - Mutate string `yaml:"mutate"` + Essential []string `yaml:"essential,omitempty"` + Contents map[string]*yamlPath `yaml:"contents,omitempty"` + Mutate string `yaml:"mutate,omitempty"` } type yamlPubKey struct { @@ -640,7 +683,7 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro kinds = append(kinds, GlobPath) } if yamlPath != nil { - mode = yamlPath.Mode + mode = uint(yamlPath.Mode) mutable = yamlPath.Mutable generate = yamlPath.Generate if yamlPath.Dir { @@ -671,7 +714,7 @@ func parsePackage(baseDir, pkgName, pkgPath string, data []byte) (*Package, erro default: return nil, fmt.Errorf("slice %s_%s has invalid 'until' for path %s: %q", pkgName, sliceName, contPath, until) } - arch = yamlPath.Arch.list + arch = yamlPath.Arch.List for _, s := range arch { if deb.ValidateArch(s) != nil { return nil, fmt.Errorf("slice %s_%s has invalid 'arch' for path %s: %q", pkgName, sliceName, contPath, s) @@ -772,3 +815,68 @@ func Select(release *Release, slices []SliceKey) (*Selection, error) { return selection, nil } + +// pathInfoToYAML converts a PathInfo object to a yamlPath object. +// The returned object takes pointers to the given PathInfo object. +func pathInfoToYAML(pi *PathInfo) (*yamlPath, error) { + path := &yamlPath{ + Mode: yamlMode(pi.Mode), + Mutable: pi.Mutable, + Until: pi.Until, + Arch: yamlArch{List: pi.Arch}, + } + switch pi.Kind { + case DirPath: + path.Dir = true + case CopyPath: + path.Copy = pi.Info + case TextPath: + path.Text = &pi.Info + case SymlinkPath: + path.Symlink = pi.Info + case GlobPath: + // Nothing more needs to be done for this type. + default: + return nil, fmt.Errorf("internal error: unrecognised PathInfo type: %s", pi.Kind) + } + return path, nil +} + +// sliceToYAML converts a Slice object to a yamlSlice object. +func sliceToYAML(s *Slice) (*yamlSlice, error) { + slice := &yamlSlice{ + Essential: make([]string, 0, len(s.Essential)), + Contents: make(map[string]*yamlPath, len(s.Contents)), + Mutate: s.Scripts.Mutate, + } + for _, key := range s.Essential { + slice.Essential = append(slice.Essential, key.String()) + } + for path, info := range s.Contents { + // TODO remove the following line after upgrading to Go 1.22 or higher. + info := info + yamlPath, err := pathInfoToYAML(&info) + if err != nil { + return nil, err + } + slice.Contents[path] = yamlPath + } + return slice, nil +} + +// packageToYAML converts a Package object to a yamlPackage object. +func packageToYAML(p *Package) (*yamlPackage, error) { + pkg := &yamlPackage{ + Name: p.Name, + Archive: p.Archive, + Slices: make(map[string]yamlSlice, len(p.Slices)), + } + for name, slice := range p.Slices { + yamlSlice, err := sliceToYAML(slice) + if err != nil { + return nil, err + } + pkg.Slices[name] = *yamlSlice + } + return pkg, nil +} diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go index bec02017..2d146585 100644 --- a/internal/setup/setup_test.go +++ b/internal/setup/setup_test.go @@ -7,6 +7,7 @@ import ( "golang.org/x/crypto/openpgp/packet" . "gopkg.in/check.v1" + "gopkg.in/yaml.v3" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/testutil" @@ -1648,6 +1649,159 @@ func runParseReleaseTests(c *C, tests []setupTest) { } } +func (s *S) TestPackageMarshalYAML(c *C) { + for _, test := range setupTests { + c.Logf("Summary: %s", test.summary) + + if test.relerror == "" || test.release == nil { + continue + } + + data, ok := test.input["chisel.yaml"] + if !ok { + data = defaultChiselYaml + } + + dir := c.MkDir() + // Write chisel.yaml. + fpath := filepath.Join(dir, "chisel.yaml") + err := os.WriteFile(fpath, testutil.Reindent(data), 0644) + c.Assert(err, IsNil) + // Write the packages YAML. + for _, pkg := range test.release.Packages { + fpath = filepath.Join(dir, pkg.Path) + err = os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + pkgData, err := yaml.Marshal(pkg) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(string(pkgData)), 0644) + c.Assert(err, IsNil) + } + + release, err := setup.ReadRelease(dir) + c.Assert(err, IsNil) + + release.Path = "" + c.Assert(release, DeepEquals, test.release) + } +} + +func (s *S) TestPackageYAMLFormat(c *C) { + var tests = []struct { + summary string + input map[string]string + expected map[string]string + }{{ + summary: "Basic slice", + input: map[string]string{ + "slices/mypkg.yaml": ` + package: mypkg + archive: ubuntu + slices: + myslice: + contents: + /dir/file: {} + `, + }, + }, { + summary: "All types of paths", + input: map[string]string{ + "slices/mypkg.yaml": ` + package: mypkg + archive: ubuntu + slices: + myslice: + contents: + /dir/arch-specific*: {arch: [amd64, arm64, i386]} + /dir/copy: {copy: /dir/file} + /dir/empty-file: {text: ""} + /dir/glob*: {} + /dir/mutable: {text: TODO, mutable: true, arch: riscv64} + /dir/other-file: {} + /dir/sub-dir/: {make: true, mode: 0644} + /dir/symlink: {symlink: /dir/file} + /dir/until: {until: mutate} + mutate: | + # Test multi-line string. + content.write("/dir/mutable", foo) + `, + }, + }, { + summary: "Global and per-slice essentials", + input: map[string]string{ + "slices/mypkg.yaml": ` + package: mypkg + archive: ubuntu + essential: + - mypkg_myslice3 + slices: + myslice1: + essential: + - mypkg_myslice2 + contents: + /dir/file1: {} + myslice2: + contents: + /dir/file2: {} + myslice3: + contents: + /dir/file3: {} + `, + }, + expected: map[string]string{ + "slices/mypkg.yaml": ` + package: mypkg + archive: ubuntu + slices: + myslice1: + essential: + - mypkg_myslice3 + - mypkg_myslice2 + contents: + /dir/file1: {} + myslice2: + essential: + - mypkg_myslice3 + contents: + /dir/file2: {} + myslice3: + contents: + /dir/file3: {} + `, + }, + }} + + for _, test := range tests { + c.Logf("Summary: %s", test.summary) + + if _, ok := test.input["chisel.yaml"]; !ok { + test.input["chisel.yaml"] = defaultChiselYaml + } + + dir := c.MkDir() + for path, data := range test.input { + fpath := filepath.Join(dir, path) + err := os.MkdirAll(filepath.Dir(fpath), 0755) + c.Assert(err, IsNil) + err = os.WriteFile(fpath, testutil.Reindent(data), 0644) + c.Assert(err, IsNil) + } + + release, err := setup.ReadRelease(dir) + c.Assert(err, IsNil) + + if test.expected == nil { + test.expected = test.input + } + for _, pkg := range release.Packages { + data, err := yaml.Marshal(pkg) + c.Assert(err, IsNil) + expected := string(testutil.Reindent(test.expected[pkg.Path])) + c.Assert(strings.TrimSpace(string(data)), Equals, strings.TrimSpace(expected)) + } + } +} + var sliceKeyTests = []struct { input string expected setup.SliceKey diff --git a/tests/info/task.yaml b/tests/info/task.yaml new file mode 100644 index 00000000..395b53e1 --- /dev/null +++ b/tests/info/task.yaml @@ -0,0 +1,28 @@ +summary: Chisel can show detailed information about slices + +execute: | + # Install dependencies. + apt update && apt install -y wget + wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\ + chmod +x /usr/bin/yq + + # Single slice. + chisel info --release ${OS}-${RELEASE} base-passwd_data > file.yaml + yq file.yaml + grep -q "/etc/group: {text: FIXME, mutable: true}" file.yaml + ! grep -q "/usr/share/doc/base-passwd/copyright" file.yaml + + # Multiple slices. + chisel info --release ${OS}-${RELEASE} base-passwd_data base-passwd_copyright > file.yaml + yq file.yaml + grep -q "/etc/group: {text: FIXME, mutable: true}" file.yaml + grep -q "/usr/share/doc/base-passwd/copyright" file.yaml + + # Whole package. + chisel info --release ${OS}-${RELEASE} base-passwd > file.yaml + yq file.yaml + grep -q "/etc/group: {text: FIXME, mutable: true}" file.yaml + grep -q "/usr/share/doc/base-passwd/copyright" file.yaml + + # Non-existing. + ! chisel info --release ${OS}-${RELEASE} does-not-exist