From 82aa7eee78b4b86cb26614c79179f138cd276105 Mon Sep 17 00:00:00 2001 From: Mathias Lang Date: Sun, 14 Jan 2024 15:25:47 +0100 Subject: [PATCH] Improve unittest framework to have a virtual FS This gets the unittest framework closer to the actual behavior, and allows us to mock basic FS operations such as mkdir and writeFile. Further improvements can be made to support more operations, and to have package setup done before `TestDub` instantiation. --- source/dub/packagemanager.d | 42 ++++- source/dub/test/base.d | 282 ++++++++++++++++++++++++++++++--- source/dub/test/dependencies.d | 24 +-- source/dub/test/other.d | 6 +- source/dub/test/subpackages.d | 39 +++++ 5 files changed, 350 insertions(+), 43 deletions(-) create mode 100644 source/dub/test/subpackages.d diff --git a/source/dub/packagemanager.d b/source/dub/packagemanager.d index 9f13b2c47..549dae491 100644 --- a/source/dub/packagemanager.d +++ b/source/dub/packagemanager.d @@ -9,7 +9,7 @@ module dub.packagemanager; import dub.dependency; import dub.internal.utils; -import dub.internal.vibecompat.core.file; +import dub.internal.vibecompat.core.file : FileInfo; import dub.internal.vibecompat.data.json; import dub.internal.vibecompat.inet.path; import dub.internal.logging; @@ -760,6 +760,8 @@ class PackageManager { Package store(NativePath src, PlacementLocation dest, in PackageName name, in Version vers) { + import dub.internal.vibecompat.core.file; + assert(!name.sub.length, "Cannot store a subpackage, use main package instead"); NativePath dstpath = this.getPackagePath(dest, name, vers.toString()); ensureDirectory(dstpath.parentPath()); @@ -779,6 +781,7 @@ class PackageManager { private Package store_(NativePath src, NativePath destination, in PackageName name, in Version vers) { + import dub.internal.vibecompat.core.file; import std.range : walkLength; logDebug("Placing package '%s' version '%s' to location '%s' from file '%s'", @@ -1033,6 +1036,8 @@ symlink_exit: /// .svn folders) Hash hashPackage(Package pack) { + import dub.internal.vibecompat.core.file; + string[] ignored_directories = [".git", ".dub", ".svn"]; // something from .dub_ignore or what? string[] ignored_files = []; @@ -1088,6 +1093,23 @@ symlink_exit: } } } + + /// Used for dependency injection in `Location` + protected bool existsDirectory(NativePath path) + { + static import dub.internal.vibecompat.core.file; + return dub.internal.vibecompat.core.file.existsDirectory(path); + } + + /// Ditto + protected alias IterateDirDg = int delegate(scope int delegate(ref FileInfo)); + + /// Ditto + protected IterateDirDg iterateDirectory(NativePath path) + { + static import dub.internal.vibecompat.core.file; + return dub.internal.vibecompat.core.file.iterateDirectory(path); + } } deprecated(OverrideDepMsg) @@ -1229,6 +1251,8 @@ package struct Location { void loadOverrides() { + import dub.internal.vibecompat.core.file; + this.overrides = null; auto ovrfilepath = this.packagePath ~ LocalOverridesFilename; if (existsFile(ovrfilepath)) { @@ -1248,6 +1272,8 @@ package struct Location { private void writeOverrides() { + import dub.internal.vibecompat.core.file; + Json[] newlist; foreach (ovr; this.overrides) { auto jovr = Json.emptyObject; @@ -1266,6 +1292,8 @@ package struct Location { private void writeLocalPackageList() { + import dub.internal.vibecompat.core.file; + Json[] newlist; foreach (p; this.searchPath) { auto entry = Json.emptyObject; @@ -1291,6 +1319,8 @@ package struct Location { // load locally defined packages void scanLocalPackages(bool refresh, PackageManager manager) { + import dub.internal.vibecompat.core.file; + NativePath list_path = this.packagePath; Package[] packs; NativePath[] paths; @@ -1372,7 +1402,7 @@ package struct Location { void scanPackageFolder(NativePath path, PackageManager mgr, Package[] existing_packages) { - if (!path.existsDirectory()) + if (!mgr.existsDirectory(path)) return; void loadInternal (NativePath pack_path, NativePath packageFile) @@ -1396,7 +1426,7 @@ package struct Location { } logDebug("iterating dir %s", path.toNativeString()); - try foreach (pdir; iterateDirectory(path)) { + try foreach (pdir; mgr.iterateDirectory(path)) { logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name); if (!pdir.isDirectory) continue; @@ -1417,10 +1447,10 @@ package struct Location { // This is the most common code path else if (mgr.isManagedPath(path)) { // Iterate over versions of a package - foreach (versdir; iterateDirectory(pack_path)) { + foreach (versdir; mgr.iterateDirectory(pack_path)) { if (!versdir.isDirectory) continue; auto vers_path = pack_path ~ versdir.name ~ (pdir.name ~ "/"); - if (!vers_path.existsDirectory()) continue; + if (!mgr.existsDirectory(vers_path)) continue; packageFile = Package.findPackageFile(vers_path); loadInternal(vers_path, packageFile); } @@ -1480,7 +1510,7 @@ package struct Location { string versStr = vers.toString(); const path = this.getPackagePath(name, versStr); - if (!path.existsDirectory()) + if (!mgr.existsDirectory(path)) return null; logDiagnostic("Lazily loading package %s:%s from %s", name.main, vers, path); diff --git a/source/dub/test/base.d b/source/dub/test/base.d index 215e3e88f..a76cbac9c 100644 --- a/source/dub/test/base.d +++ b/source/dub/test/base.d @@ -51,15 +51,19 @@ version (unittest): import std.array; public import std.algorithm; +import std.exception; import std.format; +import std.string; import dub.data.settings; public import dub.dependency; public import dub.dub; public import dub.package_; +import dub.internal.vibecompat.core.file : FileInfo; import dub.packagemanager; import dub.packagesuppliers.packagesupplier; import dub.project; +import dub.recipe.io : parsePackageRecipe; /// Example of a simple unittest for a project with a single dependency unittest @@ -79,9 +83,9 @@ unittest scope dub = new TestDub(); // Let the `PackageManager` know about the `b` package - dub.addTestPackage(b, Version("1.0.0"), PackageFormat.sdl); + dub.addTestPackage("b", Version("1.0.0"), b, PackageFormat.sdl); // And about our main package - auto mainPackage = dub.addTestPackage(a, Version("1.0.0")); + auto mainPackage = dub.addTestPackage("a", Version("1.0.0"), a); // `Dub.loadPackage` will set this package as the project // While not required, it follows the common Dub use case. dub.loadPackage(mainPackage); @@ -216,7 +220,6 @@ public class TestDub : Dub */ public Package makeTestPackage(string str, Version vers, PackageFormat fmt = PackageFormat.json) { - import dub.recipe.io; final switch (fmt) { case PackageFormat.json: auto recipe = parsePackageRecipe(str, "dub.json"); @@ -229,11 +232,12 @@ public class TestDub : Dub } } - /// Ditto - public Package addTestPackage(string str, Version vers, PackageFormat fmt = PackageFormat.json) - { - return this.packageManager.add(this.makeTestPackage(str, vers, fmt)); - } + /// Ditto + public Package addTestPackage(string name, Version vers, string content, + PackageFormat fmt = PackageFormat.json) + { + return this.packageManager.add(PackageName(name), vers, content, fmt); + } } /** @@ -272,12 +276,15 @@ package class TestPackageManager : PackageManager { /// List of all SCM packages that can be fetched by this instance protected Package[Repository] scm; + /// The virtual filesystem that this PackageManager acts on + protected FSEntry fs; this() { NativePath local = NativePath(TestDub.ProjectPath); NativePath user = TestDub.Paths.userSettings; NativePath system = TestDub.Paths.systemSettings; + this.fs = new FSEntry(); super(local, user, system, false); } @@ -305,10 +312,20 @@ package class TestPackageManager : PackageManager * * Note: Deprecated `refresh(bool)` does IO, but it's deprecated */ - public override void refresh() - { - // Do nothing - } + public override void refresh() + { + // Local packages are not yet implemented + version (none) { + foreach (ref repository; this.m_repositories) + repository.scanLocalPackages(false, this); + } + this.m_internal.scan(this, false); + foreach (ref repository; this.m_repositories) + repository.scan(this, false); + + // Removed override loading usually done here as they are deprecated + this.m_initialized = true; + } /** * Loads a `Package` @@ -319,9 +336,37 @@ package class TestPackageManager : PackageManager protected override Package load(NativePath path, NativePath recipe = NativePath.init, Package parent = null, string version_ = null, StrictMode mode = StrictMode.Ignore) - { - assert(0, "`TestPackageManager.load` is not implemented"); - } + { + import dub.internal.utils : stripUTF8Bom; + if (recipe.empty) + recipe = this.findPackageFile(path); + + enforce(!recipe.empty, + "No package file found in %s, expected one of %s" + .format(path.toNativeString(), + packageInfoFiles.map!(f => cast(string)f.filename).join("/"))); + + const PackageName parent_name = parent + ? PackageName(parent.name) : PackageName.init; + + string text = stripUTF8Bom(cast(string)this.fs.readFile(recipe)); + auto content = parsePackageRecipe(text, recipe.toNativeString(), + parent_name, null, mode); + + auto ret = new Package(content, path, parent, version_); + ret.m_infoFile = recipe; + return ret; + } + + /// Reimplementation of `Package.findPackageFile` + public NativePath findPackageFile(NativePath directory) + { + foreach (file; packageInfoFiles) { + auto filename = directory ~ file.filename; + if (this.fs.existsFile(filename)) return filename; + } + return NativePath.init; + } /** * Re-Implementation of `loadSCMPackage`. @@ -352,8 +397,8 @@ package class TestPackageManager : PackageManager protected Package loadSCMRepository(in PackageName name, in Repository repo) { if (auto prepo = repo in this.scm) { - this.add(*prepo); - return *prepo; + this.addPackages(this.m_internal.fromPath, *prepo); + return *prepo; } return null; } @@ -365,12 +410,31 @@ package class TestPackageManager : PackageManager * function used by `TestDub`, but could be generalized once IO has been * abstracted away from this class. */ - public Package add(Package pkg) + public Package add(in PackageName pkg, in Version vers, string content, + PackageFormat fmt, PlacementLocation loc = PlacementLocation.user) { - // See `PackageManager.addPackages` for inspiration. - assert(!pkg.subPackages.length, "Subpackages are not yet supported"); - this.m_internal.fromPath ~= pkg; - return pkg; + import dub.recipe.io : serializePackageRecipe; + + auto path = this.getPackagePath(loc, pkg, vers.toString()); + this.fs.mkdir(path); + + final switch (fmt) { + case PackageFormat.json: + path ~= "dub.json"; + break; + case PackageFormat.sdl: + path ~= "dub.sdl"; + break; + } + + auto recipe = parsePackageRecipe(content, path.toNativeString()); + recipe.version_ = vers.toString(); + auto app = appender!string(); + serializePackageRecipe(app, recipe, path.toNativeString()); + this.fs.writeFile(path, app.data()); + + this.refresh(); + return this.getPackage(pkg, vers, loc); } /// Add a reachable SCM package to this `PackageManager` @@ -378,6 +442,32 @@ package class TestPackageManager : PackageManager { this.scm[repo] = pkg; } + + /// + protected override bool existsDirectory(NativePath path) + { + return this.fs.existsDirectory(path); + } + + /// + protected override IterateDirDg iterateDirectory(NativePath path) + { + enforce(this.fs.existsDirectory(path), + path.toNativeString() ~ " does not exists or is not a directory"); + auto dir = this.fs.lookup(path); + int iterator(scope int delegate(ref FileInfo) del) { + foreach (c; dir.children) { + FileInfo fi; + fi.name = c.name; + fi.size = (c.type == FSEntry.Type.Directory) ? 0 : c.content.length; + fi.isDirectory = (c.type == FSEntry.Type.Directory); + if (auto res = del(fi)) + return res; + } + return 0; + } + return &iterator; + } } /** @@ -442,3 +532,151 @@ public class MockPackageSupplier : PackageSupplier assert(0, this.url ~ " - searchPackages not implemented for: " ~ query); } } + +/// An abstract filesystem representation +public class FSEntry +{ + /// Type of file system entry + public enum Type { + Directory, + File, + } + + /// Ditto + protected Type type; + /// The name of this node + protected string name; + /// The parent of this entry (can be null for the root) + protected FSEntry parent; + union { + /// Children for this FSEntry (with type == Directory) + protected FSEntry[] children; + /// Content for this FDEntry (with type == File) + protected ubyte[] content; + } + + /// Creates a new FSEntry + private this (FSEntry p, Type t, string n) + { + this.type = t; + this.parent = p; + this.name = n; + } + + /// Create the root of the filesystem, only usable from this module + private this (bool initialize = true) + { + this.type = Type.Directory; + + if (initialize) { + /// Create the base structure + this.mkdir(TestDub.Paths.temp); + this.mkdir(TestDub.Paths.systemSettings); + this.mkdir(TestDub.Paths.userSettings); + this.mkdir(TestDub.Paths.userPackages); + this.mkdir(TestDub.Paths.cache); + + this.mkdir(NativePath(TestDub.ProjectPath)); + } + } + + /// Get a direct children node, returns `null` if it can't be found + protected FSEntry lookup(string name) + { + assert(!name.canFind('/')); + foreach (c; this.children) + if (c.name == name) + return c; + return null; + } + + /// Returns: A path relative to `this.path` + protected NativePath relativePath(NativePath path) + { + assert(!path.absolute() || path.startsWith(this.path), + "Calling relativePath with a differently rooted path"); + return path.absolute() ? path.relativeTo(this.path) : path; + } + + /// Get an arbitrarily nested children node + protected FSEntry lookup(NativePath path) + { + auto relp = this.relativePath(path); + if (relp.length == 0) + return this; + if (auto c = this.lookup(relp.nodes[0].name)) + return relp.length > 1 ? c.lookup(path[1 .. $]) : c; + return null; + } + + /// Returns: The `path` of this FSEntry + public NativePath path() const + { + if (this.parent is null) + return NativePath("/"); + auto thisPath = this.parent.path ~ this.name; + thisPath.endsWithSlash = (this.type == Type.Directory); + return thisPath; + } + + /// Implements `mkdir -p`, returns the created directory + public FSEntry mkdir (NativePath path) + { + auto relp = this.relativePath(path); + // Check if the child already exists + auto child = this.lookup(relp.nodes[0].name); + if (child is null) { + child = new FSEntry(this, Type.Directory, relp.nodes[0].name); + this.children ~= child; + } + // Recurse if needed + return relp.length > 1 ? child.mkdir(relp[1 .. $]) : child; + } + + /// Checks the existence of a file + public bool existsFile (NativePath path) + { + auto entry = this.lookup(path); + return entry !is null && entry.type == Type.File; + } + + /// Checks the existence of a directory + public bool existsDirectory (NativePath path) + { + auto entry = this.lookup(path); + return entry !is null && entry.type == Type.Directory; + } + + /// Reads a file, returns the content as `ubyte[]` + public ubyte[] readFile (NativePath path) + { + auto entry = this.lookup(path); + enforce(entry.type == Type.File, "Trying to read a directory"); + return entry.content.dup; + } + + /// Write to this file + public void writeFile (NativePath path, const(char)[] data) + { + this.writeFile(path, data.representation); + } + + /// Ditto + public void writeFile (NativePath path, const(ubyte)[] data) + { + if (auto file = this.lookup(path)) { + enforce(file.type == Type.File, + "Trying to write to directory: " ~ path.toNativeString()); + file.content = data.dup; + } else { + auto parentPath = path.parentPath(); + auto parent = this.lookup(parentPath); + enforce(parent !is null, "No such directory: " ~ parentPath.toNativeString()); + enforce(parent.type == Type.Directory, + "Parent path is not a directory: " ~ parentPath.toNativeString()); + auto file = new FSEntry(parent, Type.File, path.head.name()); + file.content = data.dup; + parent.children ~= file; + } + } +} diff --git a/source/dub/test/dependencies.d b/source/dub/test/dependencies.d index d9f78ed65..da5104078 100644 --- a/source/dub/test/dependencies.d +++ b/source/dub/test/dependencies.d @@ -39,9 +39,9 @@ dependency "c" version="*" const c = `name "c"`; scope dub = new TestDub(); - dub.addTestPackage(c, Version("1.0.0"), PackageFormat.sdl); - dub.addTestPackage(b, Version("1.0.0"), PackageFormat.sdl); - dub.loadPackage(dub.addTestPackage(a, Version("1.0.0"), PackageFormat.sdl)); + dub.addTestPackage(`c`, Version("1.0.0"), c, PackageFormat.sdl); + dub.addTestPackage(`b`, Version("1.0.0"), b, PackageFormat.sdl); + dub.loadPackage(dub.addTestPackage(`a`, Version("1.0.0"), a, PackageFormat.sdl)); dub.upgrade(UpgradeOptions.select); @@ -63,9 +63,9 @@ dependency "c" version="*" const c = `name "c"`; scope dub = new TestDub(); - dub.addTestPackage(c, Version("1.0.0"), PackageFormat.sdl); - dub.addTestPackage(b, Version("1.0.0"), PackageFormat.sdl); - dub.loadPackage(dub.addTestPackage(a, Version("1.0.0"), PackageFormat.sdl)); + dub.addTestPackage(`c`, Version("1.0.0"), c, PackageFormat.sdl); + dub.addTestPackage(`b`, Version("1.0.0"), b, PackageFormat.sdl); + dub.loadPackage(dub.addTestPackage(`a`, Version("1.0.0"), a, PackageFormat.sdl)); dub.upgrade(UpgradeOptions.select); @@ -91,10 +91,10 @@ dependency "d" version="*" const d = `name "d"`; scope dub = new TestDub(); - dub.addTestPackage(d, Version("1.0.0"), PackageFormat.sdl); - dub.addTestPackage(c, Version("1.0.0"), PackageFormat.sdl); - dub.addTestPackage(b, Version("1.0.0"), PackageFormat.sdl); - dub.loadPackage(dub.addTestPackage(a, Version("1.0.0"), PackageFormat.sdl)); + dub.addTestPackage(`d`, Version("1.0.0"), d, PackageFormat.sdl); + dub.addTestPackage(`c`, Version("1.0.0"), c, PackageFormat.sdl); + dub.addTestPackage(`b`, Version("1.0.0"), b, PackageFormat.sdl); + dub.loadPackage(dub.addTestPackage(`a`, Version("1.0.0"), a, PackageFormat.sdl)); dub.upgrade(UpgradeOptions.select); @@ -113,7 +113,7 @@ dependency "b" version="*" `; scope dub = new TestDub(); - dub.loadPackage(dub.addTestPackage(a, Version("1.0.0"), PackageFormat.sdl)); + dub.loadPackage(dub.addTestPackage(`a`, Version("1.0.0"), a, PackageFormat.sdl)); try dub.upgrade(UpgradeOptions.select); @@ -125,7 +125,7 @@ dependency "b" version="*" assert(dub.project.getDependency("no", true) is null, "Returned unexpected dependency"); // Add the missing dependency to our PackageManager - dub.addTestPackage(`name "b"`, Version("1.0.0"), PackageFormat.sdl); + dub.addTestPackage(`b`, Version("1.0.0"), `name "b"`, PackageFormat.sdl); dub.upgrade(UpgradeOptions.select); assert(dub.project.hasAllDependencies(), "project have missing dependencies"); assert(dub.project.getDependency("b", true), "Missing 'b' dependency"); diff --git a/source/dub/test/other.d b/source/dub/test/other.d index 9407fb2c7..1288f49a1 100644 --- a/source/dub/test/other.d +++ b/source/dub/test/other.d @@ -31,20 +31,20 @@ unittest // Invalid URL, valid hash const a = Template.format("a", "git+https://nope.nope", ValidHash); try - dub.loadPackage(dub.addTestPackage(a, Version("1.0.0"))); + dub.loadPackage(dub.addTestPackage(`a`, Version("1.0.0"), a)); catch (Exception exc) assert(exc.message.canFind("Unable to fetch")); // Valid URL, invalid hash const b = Template.format("b", ValidURL, "invalid"); try - dub.loadPackage(dub.addTestPackage(b, Version("1.0.0"))); + dub.loadPackage(dub.addTestPackage(`b`, Version("1.0.0"), b)); catch (Exception exc) assert(exc.message.canFind("Unable to fetch")); // Valid URL, valid hash const c = Template.format("c", ValidURL, ValidHash); - dub.loadPackage(dub.addTestPackage(c, Version("1.0.0"))); + dub.loadPackage(dub.addTestPackage(`c`, Version("1.0.0"), c)); assert(dub.project.hasAllDependencies()); assert(dub.project.getDependency("dep1", true), "Missing 'dep1' dependency"); } diff --git a/source/dub/test/subpackages.d b/source/dub/test/subpackages.d new file mode 100644 index 000000000..552608aee --- /dev/null +++ b/source/dub/test/subpackages.d @@ -0,0 +1,39 @@ +/******************************************************************************* + + Test for subpackages + + Subpackages are packages that are part of a 'main' packages. Their version + is that of their main (parent) package. They are referenced using a column, + e.g. `mainpkg:subpkg`. Nested subpackages are disallowed. + +*******************************************************************************/ + +module dub.test.subpackages; + +version(unittest): + +import dub.test.base; + +/// Test of the PackageManager APIs +unittest +{ + const a = `{ "name": "a", "dependencies": { "b:a": "~>1.0", "b:b": "~>1.0" } }`; + const b = `{ "name": "b", "subPackages": [ { "name": "a" }, { "name": "b" } ] }`; + + scope dub = new TestDub(); + dub.addTestPackage(`b`, Version("1.0.0"), b); + auto mainPackage = dub.addTestPackage(`a`, Version("1.0.0"), a); + dub.loadPackage(mainPackage); + dub.upgrade(UpgradeOptions.select); + + assert(dub.project.hasAllDependencies(), "project has missing dependencies"); + assert(dub.project.getDependency("b:b", true), "Missing 'b:b' dependency"); + assert(dub.project.getDependency("b:a", true), "Missing 'b:a' dependency"); + assert(dub.project.getDependency("no", true) is null, "Returned unexpected dependency"); + + assert(dub.packageManager().getPackage(PackageName("b:a"), Version("1.0.0")).name == "b:a"); + assert(dub.packageManager().getPackage(PackageName("b:b"), Version("1.0.0")).name == "b:b"); + assert(dub.packageManager().getPackage(PackageName("b"), Version("1.0.0")).name == "b"); + + assert(!dub.packageManager().getPackage(PackageName("b:b"), Version("1.1.0"))); +}