From 738a742bb6befbb02b7f1d965bea52e38ae3fd78 Mon Sep 17 00:00:00 2001 From: plastikfan Date: Fri, 27 Oct 2023 23:21:47 +0100 Subject: [PATCH] feat(storage): add core virtual file system (#347) --- .gitignore | 2 + .vscode/settings.json | 14 + .../Nephilim/Mourning Sun/info.requiem.txt | 1 + go.mod | 1 + go.sum | 2 + xfs/storage/mem-fs.go | 120 ++++++ xfs/storage/native-fs.go | 106 +++++ xfs/storage/storage-defs.go | 94 +++++ xfs/storage/storage-suite_test.go | 13 + xfs/storage/virtual-fs_test.go | 395 ++++++++++++++++++ 10 files changed, 748 insertions(+) create mode 100644 Test/data/storage/Nephilim/Mourning Sun/info.requiem.txt create mode 100644 xfs/storage/mem-fs.go create mode 100644 xfs/storage/native-fs.go create mode 100644 xfs/storage/storage-defs.go create mode 100644 xfs/storage/storage-suite_test.go create mode 100644 xfs/storage/virtual-fs_test.go diff --git a/.gitignore b/.gitignore index 30b5be3..c234cd7 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ i18n/temp/* dist/ MUSICO/ +shroud.txt +__A/ diff --git a/.vscode/settings.json b/.vscode/settings.json index bece236..5b70354 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,10 +2,13 @@ "cSpell.words": [ "argh", "Assisable", + "avfs", + "beezledub", "bodyclose", "booter", "bootstrapper", "chardata", + "cobrass", "deadcode", "deepcopy", "depguard", @@ -16,6 +19,7 @@ "exportloopref", "extendio", "Fastward", + "faydeaudeau", "fieldalignment", "Flacs", "forloc", @@ -27,7 +31,9 @@ "gofmt", "Goid", "goimports", + "gola", "goleak", + "gomega", "gomnd", "gosec", "gosimple", @@ -42,21 +48,29 @@ "linters", "logr", "lorax", + "memfs", "mohae", "MUSICO", "nakedret", "natefinch", "navi", + "newname", + "newpath", "nicksnyder", "nolint", "nolintlint", + "oldname", + "oldpath", "onecontext", + "onsi", "prealloc", "rabbitweed", "repath", "Resumer", "rxgo", + "samber", "sidewalk", + "snivilised", "staticcheck", "structcheck", "stylecheck", diff --git a/Test/data/storage/Nephilim/Mourning Sun/info.requiem.txt b/Test/data/storage/Nephilim/Mourning Sun/info.requiem.txt new file mode 100644 index 0000000..4349514 --- /dev/null +++ b/Test/data/storage/Nephilim/Mourning Sun/info.requiem.txt @@ -0,0 +1 @@ +requiem-content \ No newline at end of file diff --git a/go.mod b/go.mod index 877cb91..1107c89 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( ) require ( + github.com/avfs/avfs v0.33.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20230406165453-00490a63f317 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect diff --git a/go.sum b/go.sum index ac1385d..7122ae4 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/avfs/avfs v0.33.0 h1:5WQXbUbr6VS7aani39ZN2Vrd/s3wLnyih1Sc4ExWTxs= +github.com/avfs/avfs v0.33.0/go.mod h1:Q59flcFRYe9KYkNMfrLUJney3yeKGQpcWRyxsDBW7vI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/xfs/storage/mem-fs.go b/xfs/storage/mem-fs.go new file mode 100644 index 0000000..8c61c7a --- /dev/null +++ b/xfs/storage/mem-fs.go @@ -0,0 +1,120 @@ +package storage + +import ( + "fmt" + "io/fs" + "os" + + "github.com/avfs/avfs/vfs/memfs" + "github.com/pkg/errors" +) + +type memFS struct { + backend VirtualBackend + mfs *memfs.MemFS +} + +func UseMemFS() VirtualFS { + return &memFS{ + backend: "mem", + mfs: memfs.New(), + } +} + +func (ms *memFS) Backend() VirtualBackend { + return ms.backend +} + +// interface ExistsInFS + +func (ms *memFS) FileExists(path string) bool { + result := false + if info, err := ms.mfs.Lstat(path); err == nil { + result = !info.IsDir() + } + + return result +} + +func (ms *memFS) DirectoryExists(path string) bool { + result := false + if info, err := ms.mfs.Lstat(path); err == nil { + result = info.IsDir() + } + + return result +} + +// end: interface ExistsInFS + +// interface ReadOnlyVirtualFS + +func (ms *memFS) Lstat(path string) (fs.FileInfo, error) { + return ms.mfs.Lstat(path) +} + +func (ms *memFS) Stat(path string) (fs.FileInfo, error) { + return ms.mfs.Stat(path) +} + +func (ms *memFS) ReadFile(name string) ([]byte, error) { + return ms.mfs.ReadFile(name) +} + +func (ms *memFS) ReadDir(name string) ([]os.DirEntry, error) { + return ms.mfs.ReadDir(name) +} + +// end: interface ReadOnlyVirtualFS + +// interface WriteToFS + +func (ms *memFS) Chmod(name string, mode os.FileMode) error { + return ms.mfs.Chmod(name, mode) +} + +func (ms *memFS) Chown(name string, uid, gid int) error { + return ms.mfs.Chown(name, uid, gid) +} + +func (ms *memFS) Create(name string) (*os.File, error) { + f, err := ms.mfs.Create(name) + + if file, ok := f.(*os.File); ok { + return file, err + } + + return nil, errors.Wrap(err, + fmt.Sprintf("file '%v' creation in '%v' failed", name, ms.backend), + ) +} + +func (ms *memFS) Link(oldname, newname string) error { + return ms.mfs.Link(oldname, newname) +} + +func (ms *memFS) Mkdir(name string, perm fs.FileMode) error { + return ms.mfs.Mkdir(name, perm) +} + +func (ms *memFS) MkdirAll(path string, perm os.FileMode) error { + return ms.mfs.MkdirAll(path, perm) +} + +func (ms *memFS) Remove(name string) error { + return ms.mfs.Remove(name) +} + +func (ms *memFS) RemoveAll(path string) error { + return ms.mfs.RemoveAll(path) +} + +func (ms *memFS) Rename(oldpath, newpath string) error { + return ms.mfs.Rename(oldpath, newpath) +} + +func (ms *memFS) WriteFile(name string, data []byte, perm os.FileMode) error { + return ms.mfs.WriteFile(name, data, perm) +} + +// end: interface WriteToFS diff --git a/xfs/storage/native-fs.go b/xfs/storage/native-fs.go new file mode 100644 index 0000000..95ae3ee --- /dev/null +++ b/xfs/storage/native-fs.go @@ -0,0 +1,106 @@ +package storage + +import ( + "io/fs" + "os" +) + +type nativeFS struct { + backend VirtualBackend +} + +func UseNativeFS() VirtualFS { + return &nativeFS{ + backend: "native", + } +} + +func (ns *nativeFS) Backend() VirtualBackend { + return ns.backend +} + +// interface ExistsInFS + +func (ns *nativeFS) FileExists(path string) bool { + result := false + if info, err := os.Lstat(path); err == nil { + result = !info.IsDir() + } + + return result +} + +func (ns *nativeFS) DirectoryExists(path string) bool { + result := false + if info, err := os.Lstat(path); err == nil { + result = info.IsDir() + } + + return result +} + +// end: interface ExistsInFS + +// interface ReadOnlyVirtualFS + +func (ns *nativeFS) Lstat(path string) (fs.FileInfo, error) { + return os.Lstat(path) +} + +func (ns *nativeFS) Stat(path string) (fs.FileInfo, error) { + return os.Stat(path) +} + +func (ns *nativeFS) ReadFile(name string) ([]byte, error) { + return os.ReadFile(name) +} + +func (ns *nativeFS) ReadDir(name string) ([]os.DirEntry, error) { + return os.ReadDir(name) +} + +// end: interface ReadOnlyVirtualFS + +func (ns *nativeFS) Chmod(name string, mode os.FileMode) error { + return os.Chmod(name, mode) +} + +func (ns *nativeFS) Chown(name string, uid, gid int) error { + return os.Chown(name, uid, gid) +} + +func (ns *nativeFS) Create(name string) (*os.File, error) { + return os.Create(name) +} + +// interface WriteToFS + +func (ns *nativeFS) Link(oldname, newname string) error { + return os.Link(oldname, newname) +} + +func (ns *nativeFS) Mkdir(name string, perm fs.FileMode) error { + return os.Mkdir(name, perm) +} + +func (ns *nativeFS) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (ns *nativeFS) Remove(name string) error { + return os.Remove(name) +} + +func (ns *nativeFS) RemoveAll(path string) error { + return os.RemoveAll(path) +} + +func (ns *nativeFS) Rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} + +func (ns *nativeFS) WriteFile(name string, data []byte, perm os.FileMode) error { + return os.WriteFile(name, data, perm) +} + +// end: interface WriteToFS diff --git a/xfs/storage/storage-defs.go b/xfs/storage/storage-defs.go new file mode 100644 index 0000000..913450a --- /dev/null +++ b/xfs/storage/storage-defs.go @@ -0,0 +1,94 @@ +package storage + +import ( + "io/fs" + "os" +) + +type filepathAPI interface { + // Intended only for those filepath methods that actually affect the + // filesystem. Eg: there is no point in replicating methods like + // filepath.Join here they are just path helpers that do not read/write + // to the filesystem. + // Currently, there is no requirement for using any filepath methods + // with the golang generator, hence nothing is defined here. We may + // want to replicate this filesystem model in other contexts, so this + // will serve as a reminder in the intended use of this interface. +} + +// ExistsInFS contains methods that check the existence of file system items. +type ExistsInFS interface { + // FileExists does file exist at the path specified + FileExists(path string) bool + + // DirectoryExists does directory exist at the path specified + DirectoryExists(path string) bool +} + +type ReadFromFS interface { + // Lstat, see https://pkg.go.dev/os#Lstat + Lstat(path string) (fs.FileInfo, error) + + // Lstat, see https://pkg.go.dev/os#Stat + Stat(path string) (fs.FileInfo, error) + + // ReadFile, see https://pkg.go.dev/os#ReadFile + ReadFile(name string) ([]byte, error) + + // ReadDir, see https://pkg.go.dev/os#ReadDir + ReadDir(name string) ([]os.DirEntry, error) +} + +// WriteToFS contains methods that perform mutative operations on the file system. +type WriteToFS interface { + + // Chmod, see https://pkg.go.dev/os#Chmod + Chmod(name string, mode os.FileMode) error + + // Chown, https://pkg.go.dev/os#Chown + Chown(name string, uid, gid int) error + + // Create, see https://pkg.go.dev/os#Create + Create(name string) (*os.File, error) + + // Link, see https://pkg.go.dev/os#Link + Link(oldname, newname string) error + + // Mkdir, see https://pkg.go.dev/os#Mkdir + Mkdir(name string, perm fs.FileMode) error + + // MkdirAll, see https://pkg.go.dev/os#MkdirAll + MkdirAll(path string, perm os.FileMode) error + + // Remove, see https://pkg.go.dev/os#Remove + Remove(name string) error + + // RemoveAll, see https://pkg.go.dev/os#RemoveAll + RemoveAll(path string) error + + // Rename, see https://pkg.go.dev/os#Rename + Rename(oldpath, newpath string) error + + // WriteFile, see https://pkg.go.dev/os#WriteFile + WriteFile(name string, data []byte, perm os.FileMode) error +} + +// ReadOnlyVirtualFS provides read-only access to the file system. +type ReadOnlyVirtualFS interface { + filepathAPI + ExistsInFS + ReadFromFS +} + +// VirtualFS is a facade over the native file system, which include read +// and write access. +type VirtualFS interface { + filepathAPI + ExistsInFS + ReadFromFS + WriteToFS + + Backend() VirtualBackend +} + +type VirtualBackend string diff --git a/xfs/storage/storage-suite_test.go b/xfs/storage/storage-suite_test.go new file mode 100644 index 0000000..8c58349 --- /dev/null +++ b/xfs/storage/storage-suite_test.go @@ -0,0 +1,13 @@ +package storage_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestStorage(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Storage Suite") +} diff --git a/xfs/storage/virtual-fs_test.go b/xfs/storage/virtual-fs_test.go new file mode 100644 index 0000000..6a40cd7 --- /dev/null +++ b/xfs/storage/virtual-fs_test.go @@ -0,0 +1,395 @@ +package storage_test + +import ( + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/snivilised/extendio/i18n" + "github.com/snivilised/extendio/xfs/storage" +) + +type virtualTE struct { + message string + should string + fn func(vfs storage.VirtualFS, isNative bool) +} + +func (v *virtualTE) action(vfs storage.VirtualFS, isNative bool) { + v.fn(vfs, isNative) +} + +var ( + faydeaudeau = os.FileMode(0o777) + beezledub = os.FileMode(0o666) +) + +func reason(backend storage.VirtualBackend, message string, actual, expected any) string { + return fmt.Sprintf("🔥 [%v:%v] expected '%v' to be '%v'", + backend, message, actual, expected, + ) +} + +type setupFile struct { + path string + data []byte +} + +func setupDirectory(fs storage.VirtualFS, directoryPath string) { + if e := fs.MkdirAll(directoryPath, faydeaudeau); e != nil { + Fail(e.Error()) + } +} + +func setupFiles(fs storage.VirtualFS, directoryPath string, files ...*setupFile) { + setupDirectory(fs, directoryPath) + + for _, f := range files { + if e := fs.WriteFile(f.path, f.data, beezledub); e != nil { + Fail(e.Error()) + } + } +} + +var _ = Describe("virtual-fs", Ordered, func() { + var ( + mfs storage.VirtualFS + nfs storage.VirtualFS + root, requiem string + ) + + BeforeAll(func() { + if current, err := os.Getwd(); err == nil { + resolved := filepath.Join( + current, "..", "..", "Test", "data", "storage", "Nephilim", "Mourning Sun", + ) + + var err error + root, err = filepath.Abs(resolved) + + if err != nil { + Fail("failed to resolve root path") + } + + requiem = filepath.Join(root, "info.requiem.txt") + } + }) + + BeforeEach(func() { + mfs = storage.UseMemFS() + nfs = storage.UseNativeFS() + + if err := Use(func(o *UseOptions) { + o.Tag = DefaultLanguage.Get() + }); err != nil { + Fail(err.Error()) + } + }) + + DescribeTable("vfs", + func(entry *virtualTE) { + entry.action(mfs, false) + entry.action(nfs, true) + }, + + func(entry *virtualTE) string { + return fmt.Sprintf("🧪 ===> given: '%v', should: '%v'", + entry.message, entry.should, + ) + }, + + // --- ExistsInFS + + Entry(nil, &virtualTE{ + message: "FileExists", + should: "return correct existence status", + fn: func(vfs storage.VirtualFS, isNative bool) { + if !isNative { + setupFiles(vfs, root, &setupFile{ + path: requiem, + data: []byte("foo-bar"), + }) + } + actual := vfs.FileExists(requiem) + + Expect(actual).To(BeTrue(), + reason(vfs.Backend(), "file exists return error", actual, true), + ) + }, + }), + + Entry(nil, &virtualTE{ + message: "DirectoryExists", + should: "return correct existence status", + fn: func(vfs storage.VirtualFS, isNative bool) { + if !isNative { + setupDirectory(vfs, root) + } + + actual := vfs.DirectoryExists(root) + + Expect(actual).To(BeTrue(), + reason(vfs.Backend(), "directory exists return error", actual, true), + ) + }, + }), + + // --- end: ExistsInFS + + // --- ReadOnlyVirtualFS + + Entry(nil, &virtualTE{ + message: "Lstat", + should: "return correct file info", + fn: func(vfs storage.VirtualFS, isNative bool) { + if !isNative { + setupFiles(vfs, root, &setupFile{ + path: requiem, + data: []byte("requiem-content"), + }) + } + info, err := vfs.Lstat(requiem) + Expect(err).Error().To(BeNil()) + + expected := "info.requiem.txt" + actual := info.Name() + Expect(actual).To(Equal(expected), + reason(vfs.Backend(), "lstat return correct name", actual, expected), + ) + }, + }), + + Entry(nil, &virtualTE{ + message: "Stat", + should: "return correct file info", + fn: func(vfs storage.VirtualFS, isNative bool) { + if !isNative { + setupFiles(vfs, root, &setupFile{ + path: requiem, + data: []byte("requiem-content"), + }) + } + info, err := vfs.Stat(requiem) + Expect(err).Error().To(BeNil()) + + expected := "info.requiem.txt" + actual := info.Name() + Expect(actual).To(Equal(expected), + reason(vfs.Backend(), "lstat return correct name", actual, expected), + ) + }, + }), + + Entry(nil, &virtualTE{ + message: "ReadFile", + should: "return correct file content", + fn: func(vfs storage.VirtualFS, isNative bool) { + expected := "requiem-content" + + if !isNative { + setupFiles(vfs, root, &setupFile{ + path: requiem, + data: []byte(expected), + }) + } + content, err := vfs.ReadFile(requiem) + actual := string(content) + + Expect(actual).To(Equal(expected), + reason(vfs.Backend(), "read file return content", actual, expected), + ) + Expect(err).Error().To(BeNil()) + }, + }), + + Entry(nil, &virtualTE{ + message: "ReadDir", + should: "return correct read status", + fn: func(vfs storage.VirtualFS, isNative bool) { + expected := "requiem-content" + + if !isNative { + setupFiles(vfs, root, &setupFile{ + path: requiem, + data: []byte(expected), + }) + } + actual, err := vfs.ReadDir(root) + + Expect(actual).To(HaveLen(1), + reason(vfs.Backend(), "read directory return content", actual, expected), + ) + Expect(err).Error().To(BeNil()) + }, + }), + + // --- end: ReadOnlyVirtualFS + + // --- WriteToFS + + Entry(nil, &virtualTE{ + message: "Create", + should: "create file", + fn: func(vfs storage.VirtualFS, isNative bool) { + path := filepath.Join(root, "shroud.txt") + + if !isNative { + setupDirectory(vfs, root) + } + + file, err := vfs.Create(path) + if err == nil { + defer file.Close() + } + + defer func() { + _ = vfs.Remove(path) + }() + + Expect(err).Error().To(BeNil(), + reason(vfs.Backend(), "create file return error", err, nil), + ) + }, + }), + + // Chmod + // Chown + // Link + + Entry(nil, &virtualTE{ + message: "Mkdir", + should: "create all directory segments in path", + fn: func(vfs storage.VirtualFS, isNative bool) { + if isNative { + return // bypass due to potential of access denied in native-fs + } + + setupDirectory(vfs, root) + + path := filepath.Join(root, "__A") + actual := vfs.Mkdir(path, beezledub) + + Expect(actual).Error().To(BeNil(), + reason(vfs.Backend(), "Mkdir return error", actual, nil), + ) + Expect(vfs.DirectoryExists(path)).To(BeTrue()) + }, + }), + + Entry(nil, &virtualTE{ + message: "MkdirAll", + should: "create all directory segments in path", + fn: func(vfs storage.VirtualFS, isNative bool) { + if isNative { + return // bypass due to potential of access denied in native-fs + } + + setupDirectory(vfs, root) + + path := filepath.Join(root, "__A", "__B", "__C") + actual := vfs.MkdirAll(path, beezledub) + + Expect(actual).Error().To(BeNil(), + reason(vfs.Backend(), "MkdirAll return error", actual, nil), + ) + Expect(vfs.DirectoryExists(path)).To(BeTrue()) + }, + }), + + Entry(nil, &virtualTE{ + message: "Remove", + should: "remove file at path", + fn: func(vfs storage.VirtualFS, isNative bool) { + path := filepath.Join(root, "shroud.txt") + setupFiles(vfs, root, &setupFile{ + path: path, + data: []byte("foo-bar"), + }) + + actual := vfs.Remove(path) + + Expect(actual).Error().To(BeNil(), + reason(vfs.Backend(), "remove file return error", actual, nil), + ) + }, + }), + + Entry(nil, &virtualTE{ + message: "RemoveAll", + should: "remove all at path", + fn: func(vfs storage.VirtualFS, isNative bool) { + path := filepath.Join(root, "__A") + + setupFiles(vfs, path, + &setupFile{ + path: filepath.Join(path, "x.txt"), + data: []byte("x-content"), + }, + &setupFile{ + path: filepath.Join(path, "y.txt"), + data: []byte("y-content"), + }, + ) + + actual := vfs.RemoveAll(path) + + Expect(actual).Error().To(BeNil(), + reason(vfs.Backend(), "remove all at path return error", actual, nil), + ) + }, + }), + + Entry(nil, &virtualTE{ + message: "Rename", + should: "rename file at path", + fn: func(vfs storage.VirtualFS, isNative bool) { + path := filepath.Join(root, "shroud.txt") + destination := filepath.Join(root, "renamed-shroud.txt") + setupFiles(vfs, root, &setupFile{ + path: path, + data: []byte("foo-bar"), + }) + + actual := vfs.Rename(path, destination) + + if isNative { + defer func() { + _ = vfs.Remove(destination) + }() + } + + Expect(actual).Error().To(BeNil(), + reason(vfs.Backend(), "rename return error", actual, nil), + ) + Expect(vfs.FileExists(destination)).To(BeTrue()) + }, + }), + + Entry(nil, &virtualTE{ + message: "WriteFile", + should: "write file to path", + fn: func(vfs storage.VirtualFS, isNative bool) { + setupDirectory(vfs, root) + path := filepath.Join(root, "shroud.txt") + + content := []byte("Mourning Sun") + actual := vfs.WriteFile(path, content, beezledub) + + if isNative { + defer func() { + _ = vfs.Remove(path) + }() + } + + Expect(actual).Error().To(BeNil(), + reason(vfs.Backend(), "write file return error", actual, nil), + ) + }, + }), + + // --- end: WriteToFS + ) +})