From 7fa6ee965832231bf6f054b7ce2fd5044e8b2844 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Fri, 27 Sep 2024 17:19:08 +0200 Subject: [PATCH] move report and generateManifest to manifest package --- internal/manifest/manifest.go | 165 ++++++++++++++++ internal/manifest/manifest_test.go | 186 ++++++++++++++++++- internal/{slicer => manifest}/report.go | 8 +- internal/{slicer => manifest}/report_test.go | 32 ++-- internal/slicer/slicer.go | 176 +----------------- 5 files changed, 383 insertions(+), 184 deletions(-) rename internal/{slicer => manifest}/report.go (96%) rename internal/{slicer => manifest}/report_test.go (91%) diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index 724e4d2e..7d3d7b78 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -3,10 +3,16 @@ package manifest import ( "fmt" "io" + "io/fs" + "os" "path/filepath" "slices" + "sort" "strings" + "github.com/klauspost/compress/zstd" + + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/jsonwall" "github.com/canonical/chisel/internal/setup" ) @@ -176,6 +182,68 @@ func LocateManifestSlices(slices []*setup.Slice, manifestFileName string) map[st return manifestSlices } +type GenerateManifestsOptions struct { + PackageInfo []*archive.PackageInfo + Selection []*setup.Slice + Report *Report + TargetDir string + Filename string + Mode os.FileMode +} + +func GenerateManifests(options *GenerateManifestsOptions) error { + manifestSlices := LocateManifestSlices(options.Selection, options.Filename) + if len(manifestSlices) == 0 { + // Nothing to do. + return nil + } + dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{ + Schema: Schema, + }) + + err := manifestAddPackages(dbw, options.PackageInfo) + if err != nil { + return err + } + + err = manifestAddSlices(dbw, options.Selection) + if err != nil { + return err + } + + err = manifestAddReport(dbw, options.Report.Entries) + if err != nil { + return err + } + + err = manifestAddManifestPaths(dbw, options.Mode, manifestSlices) + if err != nil { + return err + } + + files := []io.Writer{} + for relPath := range manifestSlices { + logf("Generating manifest at %s...", relPath) + absPath := filepath.Join(options.TargetDir, relPath) + if err := os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { + return err + } + file, err := os.OpenFile(absPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, options.Mode) + if err != nil { + return err + } + files = append(files, file) + defer file.Close() + } + w, err := zstd.NewWriter(io.MultiWriter(files...)) + if err != nil { + return err + } + defer w.Close() + _, err = dbw.WriteTo(w) + return err +} + type prefixable interface { Path | Content | Package | Slice } @@ -198,3 +266,100 @@ func iteratePrefix[T prefixable](manifest *Manifest, prefix *T, onMatch func(*T) } return nil } + +func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo) error { + for _, info := range infos { + err := dbw.Add(&Package{ + Kind: "package", + Name: info.Name, + Version: info.Version, + Digest: info.SHA256, + Arch: info.Arch, + }) + if err != nil { + return err + } + } + return nil +} + +func manifestAddSlices(dbw *jsonwall.DBWriter, slices []*setup.Slice) error { + for _, slice := range slices { + err := dbw.Add(&Slice{ + Kind: "slice", + Name: slice.String(), + }) + if err != nil { + return err + } + } + return nil +} + +func manifestAddReport(dbw *jsonwall.DBWriter, entries map[string]ReportEntry) error { + for _, entry := range entries { + sliceNames := []string{} + for slice := range entry.Slices { + err := dbw.Add(&Content{ + Kind: "content", + Slice: slice.String(), + Path: entry.Path, + }) + if err != nil { + return err + } + sliceNames = append(sliceNames, slice.String()) + } + sort.Strings(sliceNames) + err := dbw.Add(&Path{ + Kind: "path", + Path: entry.Path, + Mode: fmt.Sprintf("0%o", unixPerm(entry.Mode)), + Slices: sliceNames, + Hash: entry.Hash, + FinalHash: entry.FinalHash, + Size: uint64(entry.Size), + Link: entry.Link, + }) + if err != nil { + return err + } + } + return nil +} + +func manifestAddManifestPaths(dbw *jsonwall.DBWriter, manifestMode os.FileMode, manifestSlices map[string][]*setup.Slice) error { + for path, slices := range manifestSlices { + sliceNames := []string{} + for _, slice := range slices { + err := dbw.Add(&Content{ + Kind: "content", + Slice: slice.String(), + Path: path, + }) + if err != nil { + return err + } + sliceNames = append(sliceNames, slice.String()) + } + sort.Strings(sliceNames) + err := dbw.Add(&Path{ + Kind: "path", + Path: path, + Mode: fmt.Sprintf("0%o", unixPerm(manifestMode)), + Slices: sliceNames, + }) + if err != nil { + return err + } + } + return nil +} + +func unixPerm(mode fs.FileMode) (perm uint32) { + perm = uint32(mode.Perm()) + if mode&fs.ModeSticky != 0 { + perm |= 01000 + } + return perm +} diff --git a/internal/manifest/manifest_test.go b/internal/manifest/manifest_test.go index 8924e82b..2c5ee102 100644 --- a/internal/manifest/manifest_test.go +++ b/internal/manifest/manifest_test.go @@ -1,15 +1,20 @@ package manifest_test import ( + "io/fs" "os" "path" + "path/filepath" "slices" "strings" + "github.com/klauspost/compress/zstd" . "gopkg.in/check.v1" + "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/manifest" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/internal/testutil" ) type manifestContents struct { @@ -19,7 +24,7 @@ type manifestContents struct { Contents []*manifest.Content } -var manifestTests = []struct { +var readManifestTests = []struct { summary string input string mfest *manifestContents @@ -125,7 +130,7 @@ var manifestTests = []struct { }} func (s *S) TestManifestReadValidate(c *C) { - for _, test := range manifestTests { + for _, test := range readManifestTests { c.Logf("Summary: %s", test.summary) // Reindent the jsonwall to remove leading tabs in each line. @@ -266,6 +271,183 @@ func (s *S) TestLocateManifestSlices(c *C) { } } +func (s *S) TestGenerateManifests(c *C) { + slice1 := &setup.Slice{ + Package: "package1", + Name: "slice1", + Contents: map[string]setup.PathInfo{ + "/dir/**": { + Kind: "generate", + Generate: "manifest", + }, + "/other-dir/**": { + Kind: "generate", + Generate: "manifest", + }, + }, + } + slice2 := &setup.Slice{ + Package: "package2", + Name: "slice2", + } + report := &manifest.Report{ + Root: "/", + Entries: map[string]manifest.ReportEntry{ + "/file": { + Path: "/file", + Mode: 0456, + Hash: "hash", + Size: 1234, + Slices: map[*setup.Slice]bool{slice1: true}, + FinalHash: "final-hash", + }, + "/link": { + Path: "/link", + Mode: 0567, + Link: "/target", + Slices: map[*setup.Slice]bool{slice1: true, slice2: true}, + }, + }, + } + packageInfo := []*archive.PackageInfo{{ + Name: "package1", + Version: "v1", + Arch: "a1", + SHA256: "s1", + }, { + Name: "package2", + Version: "v2", + Arch: "a2", + SHA256: "s2", + }} + + expectedLocations := []string{ + "/dir/manifest.wall", + "/other-dir/manifest.wall", + } + expected := &manifestContents{ + Paths: []*manifest.Path{{ + Kind: "path", + Path: "/dir/manifest.wall", + Mode: "0645", + Slices: []string{"package1_slice1"}, + }, { + Kind: "path", + Path: "/file", + Mode: "0456", + Slices: []string{"package1_slice1"}, + Size: 1234, + Hash: "hash", + FinalHash: "final-hash", + }, { + Kind: "path", + Path: "/link", + Link: "/target", + Mode: "0567", + Slices: []string{"package1_slice1", "package2_slice2"}, + }, { + Kind: "path", + Path: "/other-dir/manifest.wall", + Mode: "0645", + Slices: []string{"package1_slice1"}, + }}, + Packages: []*manifest.Package{{ + Kind: "package", + Name: "package1", + Version: "v1", + Digest: "s1", + Arch: "a1", + }, { + Kind: "package", + Name: "package2", + Version: "v2", + Digest: "s2", + Arch: "a2", + }}, + Slices: []*manifest.Slice{{ + Kind: "slice", + Name: "package1_slice1", + }, { + Kind: "slice", + Name: "package2_slice2", + }}, + Contents: []*manifest.Content{{ + Kind: "content", + Slice: "package1_slice1", + Path: "/dir/manifest.wall", + }, { + Kind: "content", + Slice: "package1_slice1", + Path: "/file", + }, { + Kind: "content", + Slice: "package1_slice1", + Path: "/link", + }, { + Kind: "content", + Slice: "package1_slice1", + Path: "/other-dir/manifest.wall", + }, { + Kind: "content", + Slice: "package2_slice2", + Path: "/link", + }}, + } + + tmpDir := c.MkDir() + options := &manifest.GenerateManifestsOptions{ + PackageInfo: packageInfo, + Selection: []*setup.Slice{slice1, slice2}, + Report: report, + TargetDir: tmpDir, + Filename: "manifest.wall", + Mode: 0645, + } + found := map[string]bool{} + err := manifest.GenerateManifests(options) + c.Assert(err, IsNil) + err = filepath.WalkDir(tmpDir, func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + c.Assert(expectedLocations, testutil.Contains, strings.TrimPrefix(path, tmpDir)) + found[path] = true + file, err := os.Open(path) + c.Assert(err, IsNil) + reader, err := zstd.NewReader(file) + c.Assert(err, IsNil) + mfest, err := manifest.Read(reader) + c.Assert(err, IsNil) + err = manifest.Validate(mfest) + c.Assert(err, IsNil) + contents := dumpManifestContents(c, mfest) + c.Assert(contents, DeepEquals, expected) + return nil + }) + c.Assert(err, IsNil) + c.Assert(found, HasLen, len(expectedLocations)) +} + +func (s *S) TestGenerateNoManifests(c *C) { + tmpDir := c.MkDir() + options := &manifest.GenerateManifestsOptions{ + PackageInfo: nil, + Selection: nil, + Report: nil, + TargetDir: tmpDir, + Filename: "manifest.wall", + Mode: 0645, + } + err := manifest.GenerateManifests(options) + c.Assert(err, IsNil) + err = filepath.WalkDir(tmpDir, func(path string, d fs.DirEntry, err error) error { + // If there is any regular file it means that a manifest was generated. + c.Assert(d.IsDir(), Equals, true) + return nil + }) + c.Assert(err, IsNil) +} + func dumpManifestContents(c *C, mfest *manifest.Manifest) *manifestContents { var slices []*manifest.Slice err := mfest.IterateSlices("", func(slice *manifest.Slice) error { diff --git a/internal/slicer/report.go b/internal/manifest/report.go similarity index 96% rename from internal/slicer/report.go rename to internal/manifest/report.go index 8897f518..fed4a80f 100644 --- a/internal/slicer/report.go +++ b/internal/manifest/report.go @@ -1,4 +1,4 @@ -package slicer +package manifest import ( "fmt" @@ -35,8 +35,12 @@ func NewReport(root string) (*Report, error) { if !filepath.IsAbs(root) { return nil, fmt.Errorf("cannot use relative path for report root: %q", root) } + root = filepath.Clean(root) + if !strings.HasSuffix(root, "/") { + root = root + "/" + } report := &Report{ - Root: filepath.Clean(root) + "/", + Root: root, Entries: make(map[string]ReportEntry), } return report, nil diff --git a/internal/slicer/report_test.go b/internal/manifest/report_test.go similarity index 91% rename from internal/slicer/report_test.go rename to internal/manifest/report_test.go index 762b35ad..4345fa39 100644 --- a/internal/slicer/report_test.go +++ b/internal/manifest/report_test.go @@ -1,4 +1,4 @@ -package slicer_test +package manifest_test import ( "io/fs" @@ -6,8 +6,8 @@ import ( . "gopkg.in/check.v1" "github.com/canonical/chisel/internal/fsutil" + "github.com/canonical/chisel/internal/manifest" "github.com/canonical/chisel/internal/setup" - "github.com/canonical/chisel/internal/slicer" ) var oneSlice = &setup.Slice{ @@ -64,13 +64,13 @@ var reportTests = []struct { add []sliceAndEntry mutate []*fsutil.Entry // indexed by path. - expected map[string]slicer.ReportEntry + expected map[string]manifest.ReportEntry // error after adding the last [sliceAndEntry]. err string }{{ summary: "Regular directory", add: []sliceAndEntry{{entry: sampleDir, slice: oneSlice}}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -83,7 +83,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, {entry: sampleDir, slice: otherSlice}, }, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -93,7 +93,7 @@ var reportTests = []struct { }, { summary: "Regular file", add: []sliceAndEntry{{entry: sampleFile, slice: oneSlice}}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -105,7 +105,7 @@ var reportTests = []struct { }, { summary: "Regular file link", add: []sliceAndEntry{{entry: sampleLink, slice: oneSlice}}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-link": { Path: "/example-link", Mode: 0777, @@ -120,7 +120,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, {entry: sampleFile, slice: otherSlice}, }, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -141,7 +141,7 @@ var reportTests = []struct { {entry: sampleFile, slice: oneSlice}, {entry: sampleFile, slice: oneSlice}, }, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -225,7 +225,7 @@ var reportTests = []struct { {entry: sampleDir, slice: oneSlice}, }, mutate: []*fsutil.Entry{&sampleFileMutated}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-dir/": { Path: "/example-dir/", Mode: fs.ModeDir | 0654, @@ -247,7 +247,7 @@ var reportTests = []struct { {entry: sampleFile, slice: oneSlice}, }, mutate: []*fsutil.Entry{&sampleFile}, - expected: map[string]slicer.ReportEntry{ + expected: map[string]manifest.ReportEntry{ "/example-file": { Path: "/example-file", Mode: 0777, @@ -272,7 +272,7 @@ var reportTests = []struct { func (s *S) TestReport(c *C) { for _, test := range reportTests { var err error - report, err := slicer.NewReport("/base/") + report, err := manifest.NewReport("/base/") c.Assert(err, IsNil) for _, si := range test.add { err = report.Add(si.slice, &si.entry) @@ -290,6 +290,12 @@ func (s *S) TestReport(c *C) { } func (s *S) TestRootRelativePath(c *C) { - _, err := slicer.NewReport("../base/") + _, err := manifest.NewReport("../base/") c.Assert(err, ErrorMatches, `cannot use relative path for report root: "../base/"`) } + +func (s *S) TestRootOnlySlash(c *C) { + report, err := manifest.NewReport("/") + c.Assert(err, IsNil) + c.Assert(report.Root, Equals, "/") +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 89c7abc8..01de2ced 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -13,18 +13,15 @@ import ( "strings" "syscall" - "github.com/klauspost/compress/zstd" - "github.com/canonical/chisel/internal/archive" "github.com/canonical/chisel/internal/deb" "github.com/canonical/chisel/internal/fsutil" - "github.com/canonical/chisel/internal/jsonwall" "github.com/canonical/chisel/internal/manifest" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" ) -const manifestFileName = "manifest.wall" +const manifestFilename = "manifest.wall" const manifestMode fs.FileMode = 0644 type RunOptions struct { @@ -171,7 +168,7 @@ func Run(options *RunOptions) error { // listed as until: mutate in all the slices that reference them. knownPaths := map[string]pathData{} addKnownPath(knownPaths, "/", pathData{}) - report, err := NewReport(targetDir) + report, err := manifest.NewReport(targetDir) if err != nil { return fmt.Errorf("internal error: cannot create report: %w", err) } @@ -331,11 +328,13 @@ func Run(options *RunOptions) error { return err } - err = generateManifests(&generateManifestsOptions{ - packageInfo: pkgInfos, - selection: options.Selection.Slices, - report: report, - targetDir: targetDir, + err = manifest.GenerateManifests(&manifest.GenerateManifestsOptions{ + PackageInfo: pkgInfos, + Selection: options.Selection.Slices, + Report: report, + TargetDir: targetDir, + Filename: manifestFilename, + Mode: manifestMode, }) return err } @@ -434,160 +433,3 @@ func createFile(targetPath string, pathInfo setup.PathInfo) (*fsutil.Entry, erro MakeParents: true, }) } - -type generateManifestsOptions struct { - packageInfo []*archive.PackageInfo - selection []*setup.Slice - report *Report - targetDir string -} - -func generateManifests(options *generateManifestsOptions) error { - manifestSlices := manifest.LocateManifestSlices(options.selection, manifestFileName) - if len(manifestSlices) == 0 { - // Nothing to do. - return nil - } - dbw := jsonwall.NewDBWriter(&jsonwall.DBWriterOptions{ - Schema: manifest.Schema, - }) - - err := manifestAddPackages(dbw, options.packageInfo) - if err != nil { - return err - } - - err = manifestAddSlices(dbw, options.selection) - if err != nil { - return err - } - - err = manifestAddReport(dbw, options.report.Entries) - if err != nil { - return err - } - - err = manifestAddManifestPaths(dbw, manifestSlices) - if err != nil { - return err - } - - files := []io.Writer{} - for relPath := range manifestSlices { - logf("Generating manifest at %s...", relPath) - absPath := filepath.Join(options.targetDir, relPath) - if err := os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { - return err - } - file, err := os.OpenFile(absPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, manifestMode) - if err != nil { - return err - } - files = append(files, file) - defer file.Close() - } - w, err := zstd.NewWriter(io.MultiWriter(files...)) - if err != nil { - return err - } - defer w.Close() - _, err = dbw.WriteTo(w) - return err -} - -func manifestAddPackages(dbw *jsonwall.DBWriter, infos []*archive.PackageInfo) error { - for _, info := range infos { - err := dbw.Add(&manifest.Package{ - Kind: "package", - Name: info.Name, - Version: info.Version, - Digest: info.SHA256, - Arch: info.Arch, - }) - if err != nil { - return err - } - } - return nil -} - -func manifestAddSlices(dbw *jsonwall.DBWriter, slices []*setup.Slice) error { - for _, slice := range slices { - err := dbw.Add(&manifest.Slice{ - Kind: "slice", - Name: slice.String(), - }) - if err != nil { - return err - } - } - return nil -} - -func manifestAddReport(dbw *jsonwall.DBWriter, entries map[string]ReportEntry) error { - for _, entry := range entries { - sliceNames := []string{} - for slice := range entry.Slices { - err := dbw.Add(&manifest.Content{ - Kind: "content", - Slice: slice.String(), - Path: entry.Path, - }) - if err != nil { - return err - } - sliceNames = append(sliceNames, slice.String()) - } - sort.Strings(sliceNames) - err := dbw.Add(&manifest.Path{ - Kind: "path", - Path: entry.Path, - Mode: fmt.Sprintf("0%o", unixPerm(entry.Mode)), - Slices: sliceNames, - Hash: entry.Hash, - FinalHash: entry.FinalHash, - Size: uint64(entry.Size), - Link: entry.Link, - }) - if err != nil { - return err - } - } - return nil -} - -func manifestAddManifestPaths(dbw *jsonwall.DBWriter, manifestSlices map[string][]*setup.Slice) error { - for path, slices := range manifestSlices { - sliceNames := []string{} - for _, slice := range slices { - err := dbw.Add(&manifest.Content{ - Kind: "content", - Slice: slice.String(), - Path: path, - }) - if err != nil { - return err - } - sliceNames = append(sliceNames, slice.String()) - } - sort.Strings(sliceNames) - err := dbw.Add(&manifest.Path{ - Kind: "path", - Path: path, - Mode: fmt.Sprintf("0%o", unixPerm(manifestMode)), - Slices: sliceNames, - }) - if err != nil { - return err - } - } - return nil -} - -func unixPerm(mode fs.FileMode) (perm uint32) { - perm = uint32(mode.Perm()) - if mode&fs.ModeSticky != 0 { - perm |= 01000 - } - return perm -}