From c2fc95e830f3f266f83a3741c3e2d301431a220a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Virtus?= Date: Fri, 16 Jun 2023 06:06:49 +0200 Subject: [PATCH] slicer: Record packages and slices in DB This is the first commit that adds support for the DB into the slicer and the cut command. The database is created and saved in the cut command and populated in slicer. Currently, only packages and slices are recorded in the DB. Recording of paths is added in the forthcoming commit because it is more intricate. Slicer test is extended to allow testing of created DB objects. A list of expected DB objects is added to each existing test case. These lists are currently quite boring as most test cases use only the embedded base-files package, but it'll be extended with path entries in the forthcoming commit. --- cmd/chisel/cmd_cut.go | 11 +- internal/slicer/fakedb_test.go | 162 ++++++++++++++++++++++++++ internal/slicer/slicer.go | 25 ++++ internal/slicer/slicer_test.go | 204 +++++++++++++++++++++++++++++++++ 4 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 internal/slicer/fakedb_test.go diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 2a00b718..aa68c322 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -10,6 +10,7 @@ import ( "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/cache" + "github.com/canonical/chisel/internal/db" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" ) @@ -98,13 +99,19 @@ func (cmd *cmdCut) Execute(args []string) error { archives[archiveName] = openArchive } - return slicer.Run(&slicer.RunOptions{ + dbw := db.New() + + err = slicer.Run(&slicer.RunOptions{ Selection: selection, Archives: archives, TargetDir: cmd.RootDir, + AddToDB: dbw.Add, }) + if err != nil { + return err + } - return printVersions() + return db.Save(dbw, cmd.RootDir) } // TODO These need testing, and maybe moving into a common file. diff --git a/internal/slicer/fakedb_test.go b/internal/slicer/fakedb_test.go new file mode 100644 index 00000000..f2930216 --- /dev/null +++ b/internal/slicer/fakedb_test.go @@ -0,0 +1,162 @@ +package slicer_test + +import ( + "fmt" + "io" + "sort" + "strings" + + "github.com/canonical/chisel/internal/db" +) + +// fakeDB is used to compare a list of DB objects created by the slicer against +// a list of expected DB objects. We don't care about the order in which slicer +// creates DB objects. In real usage, they will be reordered by the jsonwall +// database anyway. We only care about the set of objects created. So we record +// the created objects and put them into fakeDB and put the expected objects +// into another fakeDB. Then, we compare both sets as sorted lists obtained +// from fakeDB.values(). +// +// Since DB object types are not ordered nor comparable (Path has pointers), we +// keep different types of objects in different slices and sort these slices +// with a comparison function appropriate for each type. + +type fakeDB struct { + packages []db.Package + slices []db.Slice + paths []db.Path + contents []db.Content +} + +func (p *fakeDB) add(value any) error { + switch v := value.(type) { + case db.Package: + p.packages = append(p.packages, v) + case db.Slice: + p.slices = append(p.slices, v) + case db.Path: + p.paths = append(p.paths, v) + case db.Content: + p.contents = append(p.contents, v) + default: + return fmt.Errorf("invalid DB type %T", v) + } + return nil +} + +func (p *fakeDB) values() []any { + sort.Slice(p.packages, func(i, j int) bool { + x1 := p.packages[i].Name + x2 := p.packages[j].Name + return x1 < x2 + }) + sort.Slice(p.slices, func(i, j int) bool { + x1 := p.slices[i].Name + x2 := p.slices[j].Name + return x1 < x2 + }) + sort.Slice(p.paths, func(i, j int) bool { + x1 := p.paths[i].Path + x2 := p.paths[j].Path + return x1 < x2 + }) + sort.Slice(p.contents, func(i, j int) bool { + x1 := p.contents[i].Slice + x2 := p.contents[j].Slice + y1 := p.contents[i].Path + y2 := p.contents[j].Path + return x1 < x2 || (x1 == x2 && y1 < y2) + }) + i := 0 + vals := make([]any, len(p.packages)+len(p.slices)+len(p.paths)+len(p.contents)) + for _, v := range p.packages { + vals[i] = v + i++ + } + for _, v := range p.slices { + vals[i] = v + i++ + } + for _, v := range p.paths { + vals[i] = v + i++ + } + for _, v := range p.contents { + vals[i] = v + i++ + } + return vals +} + +func (p *fakeDB) dumpValues(w io.Writer) { + for _, v := range p.values() { + switch t := v.(type) { + case db.Package: + fmt.Fprintln(w, "db.Package{") + fmt.Fprintf(w, "\tName: %#v,\n", t.Name) + fmt.Fprintf(w, "\tVersion: %#v,\n", t.Version) + if t.SHA256 != "" { + fmt.Fprintf(w, "\tSHA256: %#v,\n", t.SHA256) + } + if t.Arch != "" { + fmt.Fprintf(w, "\tArch: %#v,\n", t.Arch) + } + fmt.Fprintln(w, "},") + case db.Slice: + fmt.Fprintln(w, "db.Slice{") + fmt.Fprintf(w, "\tName: %#v,\n", t.Name) + fmt.Fprintln(w, "},") + case db.Path: + fmt.Fprintln(w, "db.Path{") + fmt.Fprintf(w, "\tPath: %#v,\n", t.Path) + fmt.Fprintf(w, "\tMode: %#o,\n", t.Mode) + fmt.Fprintf(w, "\tSlices: %#v,\n", t.Slices) + if t.SHA256 != nil { + fmt.Fprint(w, "\tSHA256: &[...]byte{") + for i, b := range t.SHA256 { + if i%8 == 0 { + fmt.Fprint(w, "\n\t\t") + } else { + fmt.Fprint(w, " ") + } + fmt.Fprintf(w, "%#02x,", b) + } + fmt.Fprintln(w, "\n\t},") + } + if t.FinalSHA256 != nil { + fmt.Fprint(w, "\tFinalSHA256: &[...]byte{") + for i, b := range t.FinalSHA256 { + if i%8 == 0 { + fmt.Fprint(w, "\n\t\t") + } else { + fmt.Fprint(w, " ") + } + fmt.Fprintf(w, "%#02x,", b) + } + fmt.Fprintln(w, "\n\t},") + } + if t.Size != 0 { + fmt.Fprintf(w, "\tSize: %d,\n", t.Size) + } + if t.Link != "" { + fmt.Fprintf(w, "\tLink: %#v,\n", t.Link) + } + fmt.Fprintln(w, "},") + case db.Content: + fmt.Fprintln(w, "db.Content{") + fmt.Fprintf(w, "\tSlice: %#v,\n", t.Slice) + fmt.Fprintf(w, "\tPath: %#v,\n", t.Path) + fmt.Fprintln(w, "},") + default: + panic(fmt.Sprintf("invalid DB value %#v", v)) + } + } +} + +func (p *fakeDB) dump() string { + var buf strings.Builder + fmt.Fprintln(&buf, "-----BEGIN DB DUMP-----") + p.dumpValues(&buf) + fmt.Fprintln(&buf, "-----END DB DUMP-----") + return buf.String() +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 85b8fd91..20de6ead 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -11,16 +11,20 @@ import ( "syscall" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/db" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" ) +type AddToDB func(value any) error + type RunOptions struct { Selection *setup.Selection Archives map[string]archive.Archive TargetDir string + AddToDB AddToDB } func Run(options *RunOptions) error { @@ -30,6 +34,11 @@ func Run(options *RunOptions) error { pathInfos := make(map[string]setup.PathInfo) knownPaths := make(map[string]bool) + addToDB := options.AddToDB + if addToDB == nil { + addToDB = func(value any) error { return nil } + } + knownPaths["/"] = true // addKnownPath path adds path and all its directory parent paths into @@ -68,6 +77,11 @@ func Run(options *RunOptions) error { // Build information to process the selection. for _, slice := range options.Selection.Slices { + pkgSlice := slice.String() + if err := addToDB(db.Slice{pkgSlice}); err != nil { + return fmt.Errorf("cannot write slice to db: %w", err) + } + extractPackage := extract[slice.Package] if extractPackage == nil { archiveName := release.Packages[slice.Package].Archive @@ -81,6 +95,17 @@ func Run(options *RunOptions) error { archives[slice.Package] = archive extractPackage = make(map[string][]deb.ExtractInfo) extract[slice.Package] = extractPackage + + pkgInfo := archive.Info(slice.Package) + dbPackage := db.Package{ + slice.Package, + pkgInfo.Version(), + pkgInfo.SHA256(), + pkgInfo.Arch(), + } + if err := addToDB(dbPackage); err != nil { + return fmt.Errorf("cannot write package to db: %w", err) + } } arch := archives[slice.Package].Options().Arch copyrightPath := "/usr/share/doc/" + slice.Package + "/copyright" diff --git a/internal/slicer/slicer_test.go b/internal/slicer/slicer_test.go index 9433c27e..43c2d6e2 100644 --- a/internal/slicer/slicer_test.go +++ b/internal/slicer/slicer_test.go @@ -12,6 +12,7 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/archive" + "github.com/canonical/chisel/internal/db" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" "github.com/canonical/chisel/internal/testutil" @@ -36,6 +37,7 @@ type slicerTest struct { slices []setup.SliceKey hackopt func(c *C, opts *slicer.RunOptions) result map[string]string + db []any error string } @@ -104,6 +106,15 @@ var slicerTests = []slicerTest{{ "/etc/dir/sub/": "dir 01777", "/etc/passwd": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Glob extraction", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -121,6 +132,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/": "dir 0755", "/usr/bin/hello": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Create new file under extracted directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -138,6 +158,15 @@ var slicerTests = []slicerTest{{ "/tmp/": "dir 01777", // This is the magic. "/tmp/new": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Create new nested file under extracted directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -156,6 +185,15 @@ var slicerTests = []slicerTest{{ "/tmp/new/": "dir 0755", "/tmp/new/sub": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Create new directory under extracted directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -173,6 +211,15 @@ var slicerTests = []slicerTest{{ "/tmp/": "dir 01777", // This is the magic. "/tmp/new/": "dir 0755", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Conditional architecture", arch: "amd64", @@ -200,6 +247,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/hello1": "file 0775 eaf29575", "/usr/bin/hello3": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: write a file", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -218,6 +274,15 @@ var slicerTests = []slicerTest{{ "/tmp/": "dir 01777", "/tmp/file1": "file 0644 d98cf53e", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: read a file", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -240,6 +305,15 @@ var slicerTests = []slicerTest{{ "/foo/": "dir 0755", "/foo/file2": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: use 'until' to remove file after mutate", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -261,6 +335,15 @@ var slicerTests = []slicerTest{{ "/foo/": "dir 0755", "/foo/file2": "file 0644 5b41362b", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: use 'until' to remove wildcard after mutate", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -278,6 +361,15 @@ var slicerTests = []slicerTest{{ "/usr/": "dir 0755", "/etc/": "dir 0755", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: 'until' does not remove non-empty directories", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -296,6 +388,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/": "dir 0755", "/usr/bin/hallo": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Script: cannot write non-mutable files", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -342,6 +443,18 @@ var slicerTests = []slicerTest{{ content.read("/usr/bin/hello") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice1", + }, + db.Slice{ + Name: "base-files_myslice2", + }, + }, }, { summary: "Relative content root directory must not error", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -362,6 +475,15 @@ var slicerTests = []slicerTest{{ opts.TargetDir, err = filepath.Rel(dir, opts.TargetDir) c.Assert(err, IsNil) }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Can list parent directories of normal paths", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -381,6 +503,15 @@ var slicerTests = []slicerTest{{ content.list("/x/y") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Cannot list unselected directory", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -425,6 +556,15 @@ var slicerTests = []slicerTest{{ content.list("/usr/bin") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Cannot list directories not matched by glob", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -465,6 +605,25 @@ var slicerTests = []slicerTest{{ /etc/ssl/openssl.cnf: `, }, + db: []any{ + db.Package{ + Name: "copyright-symlink-libssl3", + Version: "1.0", + }, + db.Package{ + Name: "copyright-symlink-openssl", + Version: "1.0", + }, + db.Slice{ + Name: "copyright-symlink-libssl3_libs", + }, + db.Slice{ + Name: "copyright-symlink-openssl_bins", + }, + db.Slice{ + Name: "copyright-symlink-openssl_config", + }, + }, }, { summary: "Can list unclean directory paths", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -484,6 +643,15 @@ var slicerTests = []slicerTest{{ content.list("/x/./././y") `, }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Cannot read directories", slices: []setup.SliceKey{{"base-files", "myslice"}}, @@ -528,6 +696,15 @@ var slicerTests = []slicerTest{{ "/usr/bin/": "dir 0755", "/usr/bin/hello": "file 0775 eaf29575", }, + db: []any{ + db.Package{ + Name: "base-files", + Version: "1.0", + }, + db.Slice{ + Name: "base-files_myslice", + }, + }, }, { summary: "Custom archives with custom packages", pkgs: map[string]map[string]testPackage{ @@ -599,6 +776,22 @@ var slicerTests = []slicerTest{{ "/usr/share/doc/electron/": "dir 0755", "/usr/share/doc/electron/copyright": "file 0644 empty", }, + db: []any{ + db.Package{ + Name: "electron", + Version: "1.0", + }, + db.Package{ + Name: "proton", + Version: "1.0", + }, + db.Slice{ + Name: "electron_mass", + }, + db.Slice{ + Name: "proton_mass", + }, + }, }} const defaultChiselYaml = ` @@ -711,11 +904,15 @@ func (s *S) TestRun(c *C) { archives[name] = archive } + var obtainedDB = &fakeDB{} + var expectedDB = &fakeDB{} + targetDir := c.MkDir() options := slicer.RunOptions{ Selection: selection, Archives: archives, TargetDir: targetDir, + AddToDB: obtainedDB.add, } if test.hackopt != nil { test.hackopt(c, &options) @@ -745,5 +942,12 @@ func (s *S) TestRun(c *C) { } c.Assert(testutil.TreeDump(targetDir), DeepEquals, result) } + + //c.Log(obtainedDB.dump()) + for _, v := range test.db { + err := expectedDB.add(v) + c.Assert(err, IsNil) + } + c.Assert(obtainedDB.values(), DeepEquals, expectedDB.values()) } }