From 2cea8307b6416f52d2b7d2760c5ceae6915fc420 Mon Sep 17 00:00:00 2001 From: Alberto Carretero Date: Sat, 14 Sep 2024 17:04:45 +0200 Subject: [PATCH] feat: internal manifest package (#144) Add Manifest abstraction on top of the jsonwall format. --- internal/manifest/log.go | 53 ++++++++ internal/manifest/manifest.go | 182 +++++++++++++++++++++++++ internal/manifest/manifest_test.go | 208 +++++++++++++++++++++++++++++ internal/manifest/suite_test.go | 25 ++++ 4 files changed, 468 insertions(+) create mode 100644 internal/manifest/log.go create mode 100644 internal/manifest/manifest.go create mode 100644 internal/manifest/manifest_test.go create mode 100644 internal/manifest/suite_test.go diff --git a/internal/manifest/log.go b/internal/manifest/log.go new file mode 100644 index 00000000..7fe0f91c --- /dev/null +++ b/internal/manifest/log.go @@ -0,0 +1,53 @@ +package manifest + +import ( + "fmt" + "sync" +) + +// Avoid importing the log type information unnecessarily. There's a small cost +// associated with using an interface rather than the type. Depending on how +// often the logger is plugged in, it would be worth using the type instead. +type log_Logger interface { + Output(calldepth int, s string) error +} + +var globalLoggerLock sync.Mutex +var globalLogger log_Logger +var globalDebug bool + +// Specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger log_Logger) { + globalLoggerLock.Lock() + globalLogger = logger + globalLoggerLock.Unlock() +} + +// Enable the delivery of debug messages to the logger. Only meaningful +// if a logger is also set. +func SetDebug(debug bool) { + globalLoggerLock.Lock() + globalDebug = debug + globalLoggerLock.Unlock() +} + +// logf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf. +func logf(format string, args ...interface{}) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} + +// debugf sends to the logger registered via SetLogger the string resulting +// from running format and args through Sprintf, but only if debugging was +// enabled via SetDebug. +func debugf(format string, args ...interface{}) { + globalLoggerLock.Lock() + defer globalLoggerLock.Unlock() + if globalDebug && globalLogger != nil { + globalLogger.Output(2, fmt.Sprintf(format, args...)) + } +} diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go new file mode 100644 index 00000000..fc8a1366 --- /dev/null +++ b/internal/manifest/manifest.go @@ -0,0 +1,182 @@ +package manifest + +import ( + "fmt" + "io" + "slices" + + "github.com/canonical/chisel/internal/jsonwall" + "github.com/canonical/chisel/internal/setup" +) + +const schema = "1.0" + +type Package struct { + Kind string `json:"kind"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + Digest string `json:"sha256,omitempty"` + Arch string `json:"arch,omitempty"` +} + +type Slice struct { + Kind string `json:"kind"` + Name string `json:"name,omitempty"` +} + +type Path struct { + Kind string `json:"kind"` + Path string `json:"path,omitempty"` + Mode string `json:"mode,omitempty"` + Slices []string `json:"slices,omitempty"` + Hash string `json:"sha256,omitempty"` + FinalHash string `json:"final_sha256,omitempty"` + Size uint64 `json:"size,omitempty"` + Link string `json:"link,omitempty"` +} + +type Content struct { + Kind string `json:"kind"` + Slice string `json:"slice,omitempty"` + Path string `json:"path,omitempty"` +} + +type Manifest struct { + db *jsonwall.DB +} + +// Read loads a Manifest without performing any validation. The data is assumed +// to be both valid jsonwall and a valid Manifest (see Validate). +func Read(reader io.Reader) (manifest *Manifest, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("cannot read manifest: %s", err) + } + }() + + db, err := jsonwall.ReadDB(reader) + if err != nil { + return nil, err + } + mfestSchema := db.Schema() + if mfestSchema != schema { + return nil, fmt.Errorf("unknown schema version %q", mfestSchema) + } + + manifest = &Manifest{db: db} + return manifest, nil +} + +func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { + return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) +} + +func (manifest *Manifest) IteratePackages(onMatch func(*Package) error) (err error) { + return iteratePrefix(manifest, &Package{Kind: "package"}, onMatch) +} + +func (manifest *Manifest) IterateSlices(pkgName string, onMatch func(*Slice) error) (err error) { + return iteratePrefix(manifest, &Slice{Kind: "slice", Name: pkgName}, onMatch) +} + +func (manifest *Manifest) IterateContents(slice string, onMatch func(*Content) error) (err error) { + return iteratePrefix(manifest, &Content{Kind: "content", Slice: slice}, onMatch) +} + +// Validate checks that the Manifest is valid. Note that to do that it has to +// load practically the whole manifest into memory and unmarshall all the +// entries. +func Validate(manifest *Manifest) (err error) { + defer func() { + if err != nil { + err = fmt.Errorf("invalid manifest: %s", err) + } + }() + + pkgExist := map[string]bool{} + err = manifest.IteratePackages(func(pkg *Package) error { + pkgExist[pkg.Name] = true + return nil + }) + if err != nil { + return err + } + + sliceExist := map[string]bool{} + err = manifest.IterateSlices("", func(slice *Slice) error { + sk, err := setup.ParseSliceKey(slice.Name) + if err != nil { + return err + } + if !pkgExist[sk.Package] { + return fmt.Errorf("package %q not found in packages", sk.Package) + } + sliceExist[slice.Name] = true + return nil + }) + if err != nil { + return err + } + + pathToSlices := map[string][]string{} + err = manifest.IterateContents("", func(content *Content) error { + if !sliceExist[content.Slice] { + return fmt.Errorf("slice %s not found in slices", content.Slice) + } + if !slices.Contains(pathToSlices[content.Path], content.Slice) { + pathToSlices[content.Path] = append(pathToSlices[content.Path], content.Slice) + } + return nil + }) + if err != nil { + return err + } + + done := map[string]bool{} + err = manifest.IteratePaths("", func(path *Path) error { + pathSlices, ok := pathToSlices[path.Path] + if !ok { + return fmt.Errorf("path %s has no matching entry in contents", path.Path) + } + slices.Sort(pathSlices) + slices.Sort(path.Slices) + if !slices.Equal(pathSlices, path.Slices) { + return fmt.Errorf("path %s and content have diverging slices: %q != %q", path.Path, path.Slices, pathSlices) + } + done[path.Path] = true + return nil + }) + if err != nil { + return err + } + + if len(done) != len(pathToSlices) { + for path := range pathToSlices { + return fmt.Errorf("content path %s has no matching entry in paths", path) + } + } + return nil +} + +type prefixable interface { + Path | Content | Package | Slice +} + +func iteratePrefix[T prefixable](manifest *Manifest, prefix *T, onMatch func(*T) error) error { + iter, err := manifest.db.IteratePrefix(prefix) + if err != nil { + return err + } + for iter.Next() { + var val T + err := iter.Get(&val) + if err != nil { + return fmt.Errorf("cannot read manifest: %s", err) + } + err = onMatch(&val) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/manifest/manifest_test.go b/internal/manifest/manifest_test.go new file mode 100644 index 00000000..1da510da --- /dev/null +++ b/internal/manifest/manifest_test.go @@ -0,0 +1,208 @@ +package manifest_test + +import ( + "os" + "path" + "slices" + "strings" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/manifest" +) + +type manifestContents struct { + Paths []*manifest.Path + Packages []*manifest.Package + Slices []*manifest.Slice + Contents []*manifest.Content +} + +var manifestTests = []struct { + summary string + input string + mfest *manifestContents + valError string + readError string +}{{ + summary: "All types", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":13} + {"kind":"content","slice":"pkg1_manifest","path":"/manifest/manifest.wall"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/file"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/foo/bar/"} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/link/file"} + {"kind":"content","slice":"pkg2_myotherslice","path":"/dir/foo/bar/"} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + {"kind":"package","name":"pkg2","version":"v2","sha256":"hash2","arch":"arch2"} + {"kind":"path","path":"/dir/file","mode":"0644","slices":["pkg1_myslice"],"sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","final_sha256":"8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6","size":21} + {"kind":"path","path":"/dir/foo/bar/","mode":"01777","slices":["pkg2_myotherslice","pkg1_myslice"]} + {"kind":"path","path":"/dir/link/file","mode":"0644","slices":["pkg1_myslice"],"link":"/dir/file"} + {"kind":"path","path":"/manifest/manifest.wall","mode":"0644","slices":["pkg1_manifest"]} + {"kind":"slice","name":"pkg1_manifest"} + {"kind":"slice","name":"pkg1_myslice"} + {"kind":"slice","name":"pkg2_myotherslice"} + `, + mfest: &manifestContents{ + Paths: []*manifest.Path{ + {Kind: "path", Path: "/dir/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, Hash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", FinalHash: "8067926c032c090867013d14fb0eb21ae858344f62ad07086fd32375845c91a6", Size: 0x15, Link: ""}, + {Kind: "path", Path: "/dir/foo/bar/", Mode: "01777", Slices: []string{"pkg2_myotherslice", "pkg1_myslice"}, Hash: "", FinalHash: "", Size: 0x0, Link: ""}, + {Kind: "path", Path: "/dir/link/file", Mode: "0644", Slices: []string{"pkg1_myslice"}, Hash: "", FinalHash: "", Size: 0x0, Link: "/dir/file"}, + {Kind: "path", Path: "/manifest/manifest.wall", Mode: "0644", Slices: []string{"pkg1_manifest"}, Hash: "", FinalHash: "", Size: 0x0, Link: ""}, + }, + Packages: []*manifest.Package{ + {Kind: "package", Name: "pkg1", Version: "v1", Digest: "hash1", Arch: "arch1"}, + {Kind: "package", Name: "pkg2", Version: "v2", Digest: "hash2", Arch: "arch2"}, + }, + Slices: []*manifest.Slice{ + {Kind: "slice", Name: "pkg1_manifest"}, + {Kind: "slice", Name: "pkg1_myslice"}, + {Kind: "slice", Name: "pkg2_myotherslice"}, + }, + Contents: []*manifest.Content{ + {Kind: "content", Slice: "pkg1_manifest", Path: "/manifest/manifest.wall"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/file"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/foo/bar/"}, + {Kind: "content", Slice: "pkg1_myslice", Path: "/dir/link/file"}, + {Kind: "content", Slice: "pkg2_myotherslice", Path: "/dir/foo/bar/"}, + }, + }, +}, { + summary: "Slice not found", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"content","slice":"pkg1_manifest","path":"/manifest/manifest.wall"} + `, + valError: `invalid manifest: slice pkg1_manifest not found in slices`, +}, { + summary: "Package not found", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"slice","name":"pkg1_manifest"} + `, + valError: `invalid manifest: package "pkg1" not found in packages`, +}, { + summary: "Path not found in contents", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"path","path":"/dir/","mode":"01777","slices":["pkg1_myslice"]} + `, + valError: `invalid manifest: path /dir/ has no matching entry in contents`, +}, { + summary: "Content and path have different slices", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":3} + {"kind":"content","slice":"pkg1_myotherslice","path":"/dir/"} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + {"kind":"path","path":"/dir/","mode":"01777","slices":["pkg1_myslice"]} + {"kind":"slice","name":"pkg1_myotherslice"} + `, + valError: `invalid manifest: path /dir/ and content have diverging slices: \["pkg1_myslice"\] != \["pkg1_myotherslice"\]`, +}, { + summary: "Content not found in paths", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":3} + {"kind":"content","slice":"pkg1_myslice","path":"/dir/"} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + {"kind":"slice","name":"pkg1_myslice"} + `, + valError: `invalid manifest: content path /dir/ has no matching entry in paths`, +}, { + summary: "Malformed jsonwall", + input: ` + {"jsonwall":"1.0","schema":"1.0","count":1} + {"kind":"content", "not valid json" + `, + valError: `invalid manifest: cannot read manifest: unexpected end of JSON input`, +}, { + summary: "Unknown schema", + input: ` + {"jsonwall":"1.0","schema":"2.0","count":1} + {"kind":"package","name":"pkg1","version":"v1","sha256":"hash1","arch":"arch1"} + `, + readError: `cannot read manifest: unknown schema version "2.0"`, +}} + +func (s *S) TestRun(c *C) { + for _, test := range manifestTests { + c.Logf("Summary: %s", test.summary) + + // Reindent the jsonwall to remove leading tabs in each line. + lines := strings.Split(strings.TrimSpace(test.input), "\n") + trimmedLines := make([]string, 0, len(lines)) + for _, line := range lines { + trimmedLines = append(trimmedLines, strings.TrimLeft(line, "\t")) + } + test.input = strings.Join(trimmedLines, "\n") + // Assert that the jsonwall is valid, for the test to be meaningful. + slices.Sort(trimmedLines) + orderedInput := strings.Join(trimmedLines, "\n") + c.Assert(test.input, DeepEquals, orderedInput, Commentf("input jsonwall lines should be ordered")) + + tmpDir := c.MkDir() + manifestPath := path.Join(tmpDir, "manifest.wall") + w, err := os.OpenFile(manifestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + c.Assert(err, IsNil) + _, err = w.Write([]byte(test.input)) + c.Assert(err, IsNil) + w.Close() + + r, err := os.OpenFile(manifestPath, os.O_RDONLY, 0644) + c.Assert(err, IsNil) + defer r.Close() + + mfest, err := manifest.Read(r) + if test.readError != "" { + c.Assert(err, ErrorMatches, test.readError) + continue + } + c.Assert(err, IsNil) + err = manifest.Validate(mfest) + if test.valError != "" { + c.Assert(err, ErrorMatches, test.valError) + continue + } + c.Assert(err, IsNil) + if test.mfest != nil { + c.Assert(dumpManifestContents(c, mfest), DeepEquals, test.mfest) + } + } +} + +func dumpManifestContents(c *C, mfest *manifest.Manifest) *manifestContents { + var slices []*manifest.Slice + err := mfest.IterateSlices("", func(slice *manifest.Slice) error { + slices = append(slices, slice) + return nil + }) + c.Assert(err, IsNil) + + var pkgs []*manifest.Package + err = mfest.IteratePackages(func(pkg *manifest.Package) error { + pkgs = append(pkgs, pkg) + return nil + }) + c.Assert(err, IsNil) + + var paths []*manifest.Path + err = mfest.IteratePaths("", func(path *manifest.Path) error { + paths = append(paths, path) + return nil + }) + c.Assert(err, IsNil) + + var contents []*manifest.Content + err = mfest.IterateContents("", func(content *manifest.Content) error { + contents = append(contents, content) + return nil + }) + c.Assert(err, IsNil) + + mc := manifestContents{ + Paths: paths, + Packages: pkgs, + Slices: slices, + Contents: contents, + } + return &mc +} diff --git a/internal/manifest/suite_test.go b/internal/manifest/suite_test.go new file mode 100644 index 00000000..9450841d --- /dev/null +++ b/internal/manifest/suite_test.go @@ -0,0 +1,25 @@ +package manifest_test + +import ( + "testing" + + . "gopkg.in/check.v1" + + "github.com/canonical/chisel/internal/slicer" +) + +func Test(t *testing.T) { TestingT(t) } + +type S struct{} + +var _ = Suite(&S{}) + +func (s *S) SetUpTest(c *C) { + slicer.SetDebug(true) + slicer.SetLogger(c) +} + +func (s *S) TearDownTest(c *C) { + slicer.SetDebug(false) + slicer.SetLogger(nil) +}