Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speed up configuration resolution #2905

Merged
merged 10 commits into from
May 10, 2024
Merged

Speed up configuration resolution #2905

Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
aafa300
Cache various package information to reduce GC allocations in getPack…
s-ludwig May 3, 2024
c7bbf76
Eliminate edges in the configuration graph in-place.
s-ludwig May 3, 2024
cd2a4dc
Avoid re-allocating the AA for isReachableByAllParentPacks for every …
s-ludwig May 3, 2024
e6e1b18
Avoid re-allocating allconfigs_path in determineAllConfigs.
s-ludwig May 3, 2024
5c05b53
Use package indices instead of names or Package instances to identify…
s-ludwig May 3, 2024
f36c0d8
Avoid linear time haveConfig.
s-ludwig May 3, 2024
449d412
Avoid linear lookup and name allocations in Project.getDependency.
s-ludwig May 3, 2024
e479113
Reduce allocation overhead in determineDependencyConfigs.
s-ludwig May 3, 2024
8b91535
Avoid linear time edge list construction.
s-ludwig May 3, 2024
f622afd
Avoid visiting packages more than once in checkPacksRec.
s-ludwig May 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 39 additions & 17 deletions source/dub/project.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -643,16 +643,33 @@ class Project {
/// Returns a map with the configuration for all packages in the dependency tree.
string[string] getPackageConfigs(in BuildPlatform platform, string config, bool allow_non_library = true)
const {
atilaneves marked this conversation as resolved.
Show resolved Hide resolved
import std.typecons : Rebindable, rebindable;

struct Vertex { string pack, config; }
struct Edge { size_t from, to; }

Vertex[] configs;
Edge[] edges;
string[][string] parents;
parents[m_rootPackage.name] = null;
foreach (p; getTopologicalPackageList())
foreach (d; p.getAllDependencies())
parents[d.name.toString()] ~= p.name;
// cached package information to avoid continuous re-computation
// during the resolution process
const(Package)[] package_list;
Rebindable!(const(Package))[string] package_map;
string[const(Package)] package_names;
atilaneves marked this conversation as resolved.
Show resolved Hide resolved
PackageDependency[][const(Package)] package_dependencies;

package_list.reserve(m_dependencies.length);
foreach (p; getTopologicalPackageList()) {
auto pname = p.name;
auto pdeps = p.getAllDependencies();
package_list ~= p;
package_map[pname] = p;
package_names[p] = pname;
package_dependencies[p] = pdeps;
foreach (d; pdeps)
parents[d.name.toString()] ~= pname;
}

size_t createConfig(string pack, string config) {
foreach (i, v; configs)
Expand Down Expand Up @@ -724,16 +741,18 @@ class Project {

void determineDependencyConfigs(in Package p, string c)
{
auto pname = package_names[p];
atilaneves marked this conversation as resolved.
Show resolved Hide resolved
auto pdeps = package_dependencies[p];

// below we call createConfig for the main package if
// config.length is not zero. Carry on for that case,
// otherwise we've handle the pair (p, c) already
if(haveConfig(p.name, c) && !(config.length && p.name == m_rootPackage.name && config == c))
if(haveConfig(pname, c) && !(config.length && pname == m_rootPackage.name && config == c))
return;

string[][string] depconfigs;
foreach (d; p.getAllDependencies()) {
auto dp = getDependency(d.name.toString(), true);
foreach (d; pdeps) {
auto dp = package_map.get(d.name.toString(), rebindable(const(Package).init));
if (!dp) continue;

string[] cfgs;
Expand All @@ -748,36 +767,39 @@ class Project {
// if no valid configuration was found for a dependency, don't include the
// current configuration
if (!cfgs.length) {
logDebug("Skip %s %s (missing configuration for %s)", p.name, c, dp.name);
logDebug("Skip %s %s (missing configuration for %s)", pname, c, dp.name);
return;
}
depconfigs[d.name.toString()] = cfgs;
}

// add this configuration to the graph
size_t cidx = createConfig(p.name, c);
foreach (d; p.getAllDependencies())
size_t cidx = createConfig(pname, c);
foreach (d; pdeps)
foreach (sc; depconfigs.get(d.name.toString(), null))
createEdge(cidx, createConfig(d.name.toString(), sc));
}

// create a graph of all possible package configurations (package, config) -> (sub-package, sub-config)
void determineAllConfigs(in Package p)
{
auto idx = allconfigs_path.countUntil(p.name);
enforce(idx < 0, format("Detected dependency cycle: %s", (allconfigs_path[idx .. $] ~ p.name).join("->")));
allconfigs_path ~= p.name;
auto pname = package_names[p];
auto pdeps = package_dependencies[p];

auto idx = allconfigs_path.countUntil(pname);
enforce(idx < 0, format("Detected dependency cycle: %s", (allconfigs_path[idx .. $] ~ pname).join("->")));
allconfigs_path ~= pname;
scope (exit) allconfigs_path.length--;

// first, add all dependency configurations
foreach (d; p.getAllDependencies) {
auto dp = getDependency(d.name.toString(), true);
foreach (d; pdeps) {
auto dp = package_map.get(d.name.toString(), rebindable(const(Package).init));
if (!dp) continue;
determineAllConfigs(dp);
}

// for each configuration, determine the configurations usable for the dependencies
if (auto pc = p.name in m_overriddenConfigs)
if (auto pc = pname in m_overriddenConfigs)
determineDependencyConfigs(p, *pc);
else
foreach (c; p.getPlatformConfigurations(platform, p is m_rootPackage && allow_non_library))
Expand All @@ -802,10 +824,10 @@ class Project {

// when all edges are cleaned up, pick one package and remove all but one config
if (!changed) {
foreach (p; getTopologicalPackageList()) {
foreach (p; package_list) {
size_t cnt = 0;
foreach (i, ref c; configs)
if (c.pack == p.name && ++cnt > 1) {
if (c.pack == package_names[p] && ++cnt > 1) {
logDebug("NON-PRIMARY: %s %s", c.pack, c.config);
removeConfig(i);
}
Expand Down