diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 4224ed31..06c43920 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -3,11 +3,6 @@ package main import ( "github.com/jessevdk/go-flags" - "fmt" - "os" - "regexp" - "strings" - "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" @@ -18,10 +13,13 @@ var shortCutHelp = "Cut a tree with selected slices" var longCutHelp = ` The cut command uses the provided selection of package slices to create a new filesystem tree in the root location. + +By default it fetches the slices for the same Ubuntu version as the +current host, unless the --release flag is used. ` var cutDescs = map[string]string{ - "release": "Chisel release directory", + "release": "Chisel release name or directory (e.g. ubuntu-22.04)", "root": "Root for generated content", "arch": "Package architecture", } @@ -54,25 +52,7 @@ func (cmd *cmdCut) Execute(args []string) error { sliceKeys[i] = sliceKey } - var release *setup.Release - var err error - if strings.Contains(cmd.Release, "/") { - release, err = setup.ReadRelease(cmd.Release) - } else { - var label, version string - if cmd.Release == "" { - label, version, err = readReleaseInfo() - } else { - label, version, err = parseReleaseInfo(cmd.Release) - } - if err != nil { - return err - } - release, err = setup.FetchRelease(&setup.FetchOptions{ - Label: label, - Version: version, - }) - } + release, err := obtainRelease(cmd.Release) if err != nil { return err } @@ -106,35 +86,3 @@ func (cmd *cmdCut) Execute(args []string) error { }) return err } - -// TODO These need testing, and maybe moving into a common file. - -var releaseExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})-([0-9]+(?:\.?[0-9])+)$`) - -func parseReleaseInfo(release string) (label, version string, err error) { - match := releaseExp.FindStringSubmatch(release) - if match == nil { - return "", "", fmt.Errorf("invalid release reference: %q", release) - } - return match[1], match[2], nil -} - -func readReleaseInfo() (label, version string, err error) { - data, err := os.ReadFile("/etc/lsb-release") - if err == nil { - const labelPrefix = "DISTRIB_ID=" - const versionPrefix = "DISTRIB_RELEASE=" - for _, line := range strings.Split(string(data), "\n") { - switch { - case strings.HasPrefix(line, labelPrefix): - label = strings.ToLower(line[len(labelPrefix):]) - case strings.HasPrefix(line, versionPrefix): - version = line[len(versionPrefix):] - } - if label != "" && version != "" { - return label, version, nil - } - } - } - return "", "", fmt.Errorf("cannot infer release via /etc/lsb-release, see the --release option") -} diff --git a/cmd/chisel/cmd_find.go b/cmd/chisel/cmd_find.go new file mode 100644 index 00000000..91c3f7f7 --- /dev/null +++ b/cmd/chisel/cmd_find.go @@ -0,0 +1,136 @@ +package main + +import ( + "fmt" + "sort" + "strings" + "text/tabwriter" + + "github.com/jessevdk/go-flags" + + "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/strdist" +) + +var shortFindHelp = "Find existing slices" +var longFindHelp = ` +The find command queries the slice definitions for matching slices. +Globs (* and ?) are allowed in the query. + +By default it fetches the slices for the same Ubuntu version as the +current host, unless the --release flag is used. +` + +var findDescs = map[string]string{ + "release": "Chisel release name or directory (e.g. ubuntu-22.04)", +} + +type cmdFind struct { + Release string `long:"release" value-name:""` + + Positional struct { + Query []string `positional-arg-name:"" required:"yes"` + } `positional-args:"yes"` +} + +func init() { + addCommand("find", shortFindHelp, longFindHelp, func() flags.Commander { return &cmdFind{} }, findDescs, nil) +} + +func (cmd *cmdFind) Execute(args []string) error { + if len(args) > 0 { + return ErrExtraArgs + } + + release, err := obtainRelease(cmd.Release) + if err != nil { + return err + } + + slices, err := findSlices(release, cmd.Positional.Query) + if err != nil { + return err + } + if len(slices) == 0 { + fmt.Fprintf(Stderr, "No matching slices for \"%s\"\n", strings.Join(cmd.Positional.Query, " ")) + return nil + } + + w := tabWriter() + fmt.Fprintf(w, "Slice\tSummary\n") + for _, s := range slices { + fmt.Fprintf(w, "%s\t%s\n", s, "-") + } + w.Flush() + + return nil +} + +// match reports whether a slice (partially) matches the query. +func match(slice *setup.Slice, query string) bool { + var term string + switch { + case strings.HasPrefix(query, "_"): + query = strings.TrimPrefix(query, "_") + term = slice.Name + case strings.Contains(query, "_"): + term = slice.String() + default: + term = slice.Package + } + query = strings.ReplaceAll(query, "**", "⁑") + return strdist.Distance(term, query, distWithGlobs, 0) <= 1 +} + +// findSlices returns slices from the provided release that match all of the +// query strings (AND). +func findSlices(release *setup.Release, query []string) (slices []*setup.Slice, err error) { + slices = []*setup.Slice{} + for _, pkg := range release.Packages { + for _, slice := range pkg.Slices { + if slice == nil { + continue + } + allMatch := true + for _, term := range query { + if !match(slice, term) { + allMatch = false + break + } + } + if allMatch { + slices = append(slices, slice) + } + } + } + sort.Slice(slices, func(i, j int) bool { + return slices[i].String() < slices[j].String() + }) + return slices, nil +} + +func tabWriter() *tabwriter.Writer { + return tabwriter.NewWriter(Stdout, 5, 3, 2, ' ', 0) +} + +// distWithGlobs encodes the standard Levenshtein distance with support for +// "*", "?" and "**". However, because it works on runes "**" has to be encoded +// as "⁑" in the strings. +// +// Supported wildcards: +// +// ? - Any one character +// * - Any zero or more characters +// ⁑ - Any zero or more characters +func distWithGlobs(ar, br rune) strdist.Cost { + if ar == '⁑' || br == '⁑' { + return strdist.Cost{SwapAB: 0, DeleteA: 0, InsertB: 0} + } + if ar == '*' || br == '*' { + return strdist.Cost{SwapAB: 0, DeleteA: 0, InsertB: 0} + } + if ar == '?' || br == '?' { + return strdist.Cost{SwapAB: 0, DeleteA: 1, InsertB: 1} + } + return strdist.StandardCost(ar, br) +} diff --git a/cmd/chisel/cmd_find_test.go b/cmd/chisel/cmd_find_test.go new file mode 100644 index 00000000..7d7c0d7a --- /dev/null +++ b/cmd/chisel/cmd_find_test.go @@ -0,0 +1,140 @@ +package main_test + +import ( + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/testutil" + + chisel "github.com/canonical/chisel/cmd/chisel" +) + +type findTest struct { + summary string + release *setup.Release + query []string + result []*setup.Slice +} + +func makeSamplePackage(pkg string, slices []string) *setup.Package { + slicesMap := map[string]*setup.Slice{} + for _, slice := range slices { + slicesMap[slice] = &setup.Slice{ + Package: pkg, + Name: slice, + } + } + return &setup.Package{ + Name: pkg, + Path: "slices/" + pkg, + Archive: "ubuntu", + Slices: slicesMap, + } +} + +var sampleRelease = &setup.Release{ + DefaultArchive: "ubuntu", + + Archives: map[string]*setup.Archive{ + "ubuntu": { + Name: "ubuntu", + Version: "22.04", + Suites: []string{"jammy", "jammy-security"}, + Components: []string{"main", "other"}, + }, + }, + Packages: map[string]*setup.Package{ + "openjdk-8-jdk": makeSamplePackage("openjdk-8-jdk", []string{"bins", "config", "core", "libs", "utils"}), + "python3.10": makeSamplePackage("python3.10", []string{"bins", "config", "core", "libs", "utils"}), + }, +} + +var findTests = []findTest{{ + summary: "Search by package name", + release: sampleRelease, + query: []string{"python3.10"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["bins"], + sampleRelease.Packages["python3.10"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["core"], + sampleRelease.Packages["python3.10"].Slices["libs"], + sampleRelease.Packages["python3.10"].Slices["utils"], + }, +}, { + summary: "Search by slice name", + release: sampleRelease, + query: []string{"_config"}, + result: []*setup.Slice{ + sampleRelease.Packages["openjdk-8-jdk"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["config"], + }, +}, { + summary: "Slice search without leading underscore", + release: sampleRelease, + query: []string{"config"}, + result: []*setup.Slice{}, +}, { + summary: "Check distance greater than one", + release: sampleRelease, + query: []string{"python3."}, + result: []*setup.Slice{}, +}, { + summary: "Check glob matching (*)", + release: sampleRelease, + query: []string{"python3.*_bins"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["bins"], + }, +}, { + summary: "Check glob matching (?)", + release: sampleRelease, + query: []string{"python3.1?_co*"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["core"], + }, +}, { + summary: "Check no matching slice", + release: sampleRelease, + query: []string{"foo_bar"}, + result: []*setup.Slice{}, +}, { + summary: "Several terms all match", + release: sampleRelease, + query: []string{"python*", "_co*"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["config"], + sampleRelease.Packages["python3.10"].Slices["core"], + }, +}, { + summary: "Distance of one in each term", + release: sampleRelease, + query: []string{"python3.1", "_lib"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["libs"], + }, +}, { + summary: "Query with underscore is matched against full name", + release: sampleRelease, + query: []string{"python3.1_libs"}, + result: []*setup.Slice{ + sampleRelease.Packages["python3.10"].Slices["libs"], + }, +}, { + summary: "Several terms, one does not match", + release: sampleRelease, + query: []string{"python", "slice"}, + result: []*setup.Slice{}, +}} + +func (s *ChiselSuite) TestFindSlices(c *C) { + for _, test := range findTests { + c.Logf("Summary: %s", test.summary) + + for _, query := range testutil.Permutations(test.query) { + slices, err := chisel.FindSlices(test.release, query) + c.Assert(err, IsNil) + c.Assert(slices, DeepEquals, test.result) + } + } +} diff --git a/cmd/chisel/cmd_help.go b/cmd/chisel/cmd_help.go index a6314f49..a0ebd273 100644 --- a/cmd/chisel/cmd_help.go +++ b/cmd/chisel/cmd_help.go @@ -155,7 +155,7 @@ type helpCategory struct { var helpCategories = []helpCategory{{ Label: "Basic", Description: "general operations", - Commands: []string{"help", "version"}, + Commands: []string{"find", "help", "version"}, }, { Label: "Action", Description: "make things happen", diff --git a/cmd/chisel/export_test.go b/cmd/chisel/export_test.go index dd852b44..a99a7831 100644 --- a/cmd/chisel/export_test.go +++ b/cmd/chisel/export_test.go @@ -17,3 +17,5 @@ func FakeIsStdinTTY(t bool) (restore func()) { isStdinTTY = oldIsStdinTTY } } + +var FindSlices = findSlices diff --git a/cmd/chisel/helpers.go b/cmd/chisel/helpers.go new file mode 100644 index 00000000..7506888e --- /dev/null +++ b/cmd/chisel/helpers.go @@ -0,0 +1,71 @@ +package main + +import ( + "fmt" + "os" + "regexp" + "strings" + + "github.com/canonical/chisel/internal/setup" +) + +// TODO These need testing + +var releaseExp = regexp.MustCompile(`^([a-z](?:-?[a-z0-9]){2,})-([0-9]+(?:\.?[0-9])+)$`) + +func parseReleaseInfo(release string) (label, version string, err error) { + match := releaseExp.FindStringSubmatch(release) + if match == nil { + return "", "", fmt.Errorf("invalid release reference: %q", release) + } + return match[1], match[2], nil +} + +func readReleaseInfo() (label, version string, err error) { + data, err := os.ReadFile("/etc/lsb-release") + if err == nil { + const labelPrefix = "DISTRIB_ID=" + const versionPrefix = "DISTRIB_RELEASE=" + for _, line := range strings.Split(string(data), "\n") { + switch { + case strings.HasPrefix(line, labelPrefix): + label = strings.ToLower(line[len(labelPrefix):]) + case strings.HasPrefix(line, versionPrefix): + version = line[len(versionPrefix):] + } + if label != "" && version != "" { + return label, version, nil + } + } + } + return "", "", fmt.Errorf("cannot infer release via /etc/lsb-release, see the --release option") +} + +// obtainRelease returns the Chisel release information matching the provided string, +// fetching it if necessary. The provided string should be either: +// * "-", +// * the path to a directory containing a previously fetched release, +// * "" and Chisel will attempt to read the release label from the host. +func obtainRelease(releaseStr string) (release *setup.Release, err error) { + if strings.Contains(releaseStr, "/") { + release, err = setup.ReadRelease(releaseStr) + } else { + var label, version string + if releaseStr == "" { + label, version, err = readReleaseInfo() + } else { + label, version, err = parseReleaseInfo(releaseStr) + } + if err != nil { + return nil, err + } + release, err = setup.FetchRelease(&setup.FetchOptions{ + Label: label, + Version: version, + }) + } + if err != nil { + return nil, err + } + return release, nil +} diff --git a/internal/strdist/strdist.go b/internal/strdist/strdist.go index eb0c1dca..f205bfcf 100644 --- a/internal/strdist/strdist.go +++ b/internal/strdist/strdist.go @@ -102,8 +102,8 @@ func Distance(a, b string, f CostFunc, cut int64) int64 { // Supported wildcards: // // ? - Any one character, except for / -// * - Any zero or more characters, execept for / -// ** - Any zero or more characrers, including / +// * - Any zero or more characters, except for / +// ** - Any zero or more characters, including / func GlobPath(a, b string) bool { a = strings.ReplaceAll(a, "**", "⁑") b = strings.ReplaceAll(b, "**", "⁑") diff --git a/tests/find/task.yaml b/tests/find/task.yaml new file mode 100644 index 00000000..283d7bc3 --- /dev/null +++ b/tests/find/task.yaml @@ -0,0 +1,14 @@ +summary: Chisel can find slice by slice name, package name or a combination + +execute: | + find() { + fullname=$1 + shift + query=$@ + chisel find --release ${OS}-${RELEASE} $query | grep $fullname + } + + find "ca-certificates_data" "ca-certificates_data" + find "ca-certificates_data" "ca-certificates" "_data" + find "ca-certificates_data" "_data" "ca-certificates" + ! find "ca-certificates_data" "ca-certificates" "foo"