diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 2a00b718..aa68c322 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -10,6 +10,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/db" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" ) @@ -98,13 +99,19 @@ func (cmd *cmdCut) Execute(args []string) error { archives[archiveName] = openArchive } - return slicer.Run(&slicer.RunOptions{ + dbw := db.New() + + err = slicer.Run(&slicer.RunOptions{ Selection: selection, Archives: archives, TargetDir: cmd.RootDir, + AddToDB: dbw.Add, }) + if err != nil { + return err + } - return printVersions() + return db.Save(dbw, cmd.RootDir) } // TODO These need testing, and maybe moving into a common file. diff --git a/internal/slicer/fakedb_test.go b/internal/slicer/fakedb_test.go new file mode 100644 index 00000000..f2930216 --- /dev/null +++ b/internal/slicer/fakedb_test.go @@ -0,0 +1,162 @@ +package slicer_test + +import ( + "fmt" + "io" + "sort" + "strings" + + "github.com/canonical/chisel/internal/db" +) + +// fakeDB is used to compare a list of DB objects created by the slicer against +// a list of expected DB objects. We don't care about the order in which slicer +// creates DB objects. In real usage, they will be reordered by the jsonwall +// database anyway. We only care about the set of objects created. So we record +// the created objects and put them into fakeDB and put the expected objects +// into another fakeDB. Then, we compare both sets as sorted lists obtained +// from fakeDB.values(). +// +// Since DB object types are not ordered nor comparable (Path has pointers), we +// keep different types of objects in different slices and sort these slices +// with a comparison function appropriate for each type. + +type fakeDB struct { + packages []db.Package + slices []db.Slice + paths []db.Path + contents []db.Content +} + +func (p *fakeDB) add(value any) error { + switch v := value.(type) { + case db.Package: + p.packages = append(p.packages, v) + case db.Slice: + p.slices = append(p.slices, v) + case db.Path: + p.paths = append(p.paths, v) + case db.Content: + p.contents = append(p.contents, v) + default: + return fmt.Errorf("invalid DB type %T", v) + } + return nil +} + +func (p *fakeDB) values() []any { + sort.Slice(p.packages, func(i, j int) bool { + x1 := p.packages[i].Name + x2 := p.packages[j].Name + return x1 < x2 + }) + sort.Slice(p.slices, func(i, j int) bool { + x1 := p.slices[i].Name + x2 := p.slices[j].Name + return x1 < x2 + }) + sort.Slice(p.paths, func(i, j int) bool { + x1 := p.paths[i].Path + x2 := p.paths[j].Path + return x1 < x2 + }) + sort.Slice(p.contents, func(i, j int) bool { + x1 := p.contents[i].Slice + x2 := p.contents[j].Slice + y1 := p.contents[i].Path + y2 := p.contents[j].Path + return x1 < x2 || (x1 == x2 && y1 < y2) + }) + i := 0 + vals := make([]any, len(p.packages)+len(p.slices)+len(p.paths)+len(p.contents)) + for _, v := range p.packages { + vals[i] = v + i++ + } + for _, v := range p.slices { + vals[i] = v + i++ + } + for _, v := range p.paths { + vals[i] = v + i++ + } + for _, v := range p.contents { + vals[i] = v + i++ + } + return vals +} + +func (p *fakeDB) dumpValues(w io.Writer) { + for _, v := range p.values() { + switch t := v.(type) { + case db.Package: + fmt.Fprintln(w, "db.Package{") + fmt.Fprintf(w, "\tName: %#v,\n", t.Name) + fmt.Fprintf(w, "\tVersion: %#v,\n", t.Version) + if t.SHA256 != "" { + fmt.Fprintf(w, "\tSHA256: %#v,\n", t.SHA256) + } + if t.Arch != "" { + fmt.Fprintf(w, "\tArch: %#v,\n", t.Arch) + } + fmt.Fprintln(w, "},") + case db.Slice: + fmt.Fprintln(w, "db.Slice{") + fmt.Fprintf(w, "\tName: %#v,\n", t.Name) + fmt.Fprintln(w, "},") + case db.Path: + fmt.Fprintln(w, "db.Path{") + fmt.Fprintf(w, "\tPath: %#v,\n", t.Path) + fmt.Fprintf(w, "\tMode: %#o,\n", t.Mode) + fmt.Fprintf(w, "\tSlices: %#v,\n", t.Slices) + if t.SHA256 != nil { + fmt.Fprint(w, "\tSHA256: &[...]byte{") + for i, b := range t.SHA256 { + if i%8 == 0 { + fmt.Fprint(w, "\n\t\t") + } else { + fmt.Fprint(w, " ") + } + fmt.Fprintf(w, "%#02x,", b) + } + fmt.Fprintln(w, "\n\t},") + } + if t.FinalSHA256 != nil { + fmt.Fprint(w, "\tFinalSHA256: &[...]byte{") + for i, b := range t.FinalSHA256 { + if i%8 == 0 { + fmt.Fprint(w, "\n\t\t") + } else { + fmt.Fprint(w, " ") + } + fmt.Fprintf(w, "%#02x,", b) + } + fmt.Fprintln(w, "\n\t},") + } + if t.Size != 0 { + fmt.Fprintf(w, "\tSize: %d,\n", t.Size) + } + if t.Link != "" { + fmt.Fprintf(w, "\tLink: %#v,\n", t.Link) + } + fmt.Fprintln(w, "},") + case db.Content: + fmt.Fprintln(w, "db.Content{") + fmt.Fprintf(w, "\tSlice: %#v,\n", t.Slice) + fmt.Fprintf(w, "\tPath: %#v,\n", t.Path) + fmt.Fprintln(w, "},") + default: + panic(fmt.Sprintf("invalid DB value %#v", v)) + } + } +} + +func (p *fakeDB) dump() string { + var buf strings.Builder + fmt.Fprintln(&buf, "-----BEGIN DB DUMP-----") + p.dumpValues(&buf) + fmt.Fprintln(&buf, "-----END DB DUMP-----") + return buf.String() +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 85b8fd91..20de6ead 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -11,16 +11,20 @@ import ( "syscall" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/db" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" ) +type AddToDB func(value any) error + type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive TargetDir string + AddToDB AddToDB } func Run(options *RunOptions) error { @@ -30,6 +34,11 @@ func Run(options *RunOptions) error { pathInfos := make(map[string]setup.PathInfo) knownPaths := make(map[string]bool) + addToDB := options.AddToDB + if addToDB == nil { + addToDB = func(value any) error { return nil } + } + knownPaths["/"] = true // addKnownPath path adds path and all its directory parent paths into @@ -68,6 +77,11 @@ func Run(options *RunOptions) error { // Build information to process the selection. for _, slice := range options.Selection.Slices { + pkgSlice := slice.String() + if err := addToDB(db.Slice{pkgSlice}); err != nil { + return fmt.Errorf("cannot write slice to db: %w", err) + } + extractPackage := extract[slice.Package] if extractPackage == nil { archiveName := release.Packages[slice.Package].Archive @@ -81,6 +95,17 @@ func Run(options *RunOptions) error { archives[slice.Package] = archive extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage + + pkgInfo := archive.Info(slice.Package) + dbPackage := db.Package{ + slice.Package, + pkgInfo.Version(), + pkgInfo.SHA256(), + pkgInfo.Arch(), + } + if err := addToDB(dbPackage); err != nil { + return fmt.Errorf("cannot write package to db: %w", err) + } } arch := archives[slice.Package].Options().Arch copyrightPath := "/usr/share/doc/" + slice.Package + "/copyright" diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 9433c27e..43c2d6e2 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -12,6 +12,7 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/db" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" "github.com/canonical/chisel/internal/testutil" @@ -36,6 +37,7 @@ type slicerTest struct { slices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) result map[string]string + db []any error string } @@ -104,6 +106,15 @@ var slicerTests = []slicerTest{{ "/etc/dir/sub/": "dir 01777", "/etc/passwd": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Glob extraction", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -121,6 +132,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/": "dir 0755", "/usr/bin/hello": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Create new file under extracted directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -138,6 +158,15 @@ var slicerTests = []slicerTest{{ "/tmp/": "dir 01777", // This is the magic. "/tmp/new": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Create new nested file under extracted directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -156,6 +185,15 @@ var slicerTests = []slicerTest{{ "/tmp/new/": "dir 0755", "/tmp/new/sub": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Create new directory under extracted directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -173,6 +211,15 @@ var slicerTests = []slicerTest{{ "/tmp/": "dir 01777", // This is the magic. "/tmp/new/": "dir 0755", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Conditional architecture", arch: "amd64", @@ -200,6 +247,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/hello1": "file 0775 eaf29575", "/usr/bin/hello3": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: write a file", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -218,6 +274,15 @@ var slicerTests = []slicerTest{{ "/tmp/": "dir 01777", "/tmp/file1": "file 0644 d98cf53e", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: read a file", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -240,6 +305,15 @@ var slicerTests = []slicerTest{{ "/foo/": "dir 0755", "/foo/file2": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: use 'until' to remove file after mutate", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -261,6 +335,15 @@ var slicerTests = []slicerTest{{ "/foo/": "dir 0755", "/foo/file2": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: use 'until' to remove wildcard after mutate", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -278,6 +361,15 @@ var slicerTests = []slicerTest{{ "/usr/": "dir 0755", "/etc/": "dir 0755", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: 'until' does not remove non-empty directories", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -296,6 +388,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/": "dir 0755", "/usr/bin/hallo": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: cannot write non-mutable files", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -342,6 +443,18 @@ var slicerTests = []slicerTest{{ content.read("/usr/bin/hello") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice1", + }, + db.Slice{ + Name: "base-files_myslice2", + }, + }, }, { summary: "Relative content root directory must not error", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -362,6 +475,15 @@ var slicerTests = []slicerTest{{ opts.TargetDir, err = filepath.Rel(dir, opts.TargetDir) c.Assert(err, IsNil) }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Can list parent directories of normal paths", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -381,6 +503,15 @@ var slicerTests = []slicerTest{{ content.list("/x/y") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Cannot list unselected directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -425,6 +556,15 @@ var slicerTests = []slicerTest{{ content.list("/usr/bin") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Cannot list directories not matched by glob", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -465,6 +605,25 @@ var slicerTests = []slicerTest{{ /etc/ssl/openssl.cnf: `, }, + db: []any{ + db.Package{ + Name: "copyright-symlink-libssl3", + Version: "1.0", + }, + db.Package{ + Name: "copyright-symlink-openssl", + Version: "1.0", + }, + db.Slice{ + Name: "copyright-symlink-libssl3_libs", + }, + db.Slice{ + Name: "copyright-symlink-openssl_bins", + }, + db.Slice{ + Name: "copyright-symlink-openssl_config", + }, + }, }, { summary: "Can list unclean directory paths", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -484,6 +643,15 @@ var slicerTests = []slicerTest{{ content.list("/x/./././y") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Cannot read directories", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -528,6 +696,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/": "dir 0755", "/usr/bin/hello": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Custom archives with custom packages", pkgs: map[string]map[string]testPackage{ @@ -599,6 +776,22 @@ var slicerTests = []slicerTest{{ "/usr/share/doc/electron/": "dir 0755", "/usr/share/doc/electron/copyright": "file 0644 empty", }, + db: []any{ + db.Package{ + Name: "electron", + Version: "1.0", + }, + db.Package{ + Name: "proton", + Version: "1.0", + }, + db.Slice{ + Name: "electron_mass", + }, + db.Slice{ + Name: "proton_mass", + }, + }, }} const defaultChiselYaml = ` @@ -711,11 +904,15 @@ func (s *S) TestRun(c *C) { archives[name] = archive } + var obtainedDB = &fakeDB{} + var expectedDB = &fakeDB{} + targetDir := c.MkDir() options := slicer.RunOptions{ Selection: selection, Archives: archives, TargetDir: targetDir, + AddToDB: obtainedDB.add, } if test.hackopt != nil { test.hackopt(c, &options) @@ -745,5 +942,12 @@ func (s *S) TestRun(c *C) { } c.Assert(testutil.TreeDump(targetDir), DeepEquals, result) } + + //c.Log(obtainedDB.dump()) + for _, v := range test.db { + err := expectedDB.add(v) + c.Assert(err, IsNil) + } + c.Assert(obtainedDB.values(), DeepEquals, expectedDB.values()) } }