From 17d76f74a86c39fcab746472aba770c594514ab8 Mon Sep 17 00:00:00 2001
From: Oliver Hamlet <oliver@ortham.io>
Date: Thu, 12 Jan 2023 18:17:19 +0000
Subject: [PATCH] Don't add edges from fully processed group plugins twice

This optimisation reduces the time taken to sort my 1619 plugin load
order with 9 extra groups (the userlist is 77 KB!) from 1440s to 209s.
---
 src/api/sorting/plugin_graph.cpp | 39 ++++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/src/api/sorting/plugin_graph.cpp b/src/api/sorting/plugin_graph.cpp
index 2452f28c..e852d21c 100644
--- a/src/api/sorting/plugin_graph.cpp
+++ b/src/api/sorting/plugin_graph.cpp
@@ -121,19 +121,23 @@ class GroupsVisitor : public boost::dfs_visitor<> {
 
   explicit GroupsVisitor(
       PluginGraph& pluginGraph,
+      std::unordered_set<GroupGraphVertex>& finishedVertices,
       const std::unordered_map<std::string, std::vector<vertex_t>>&
           groupsPlugins) :
       pluginGraph_(&pluginGraph),
       groupsPlugins_(&groupsPlugins),
+      finishedVertices_(&finishedVertices),
       logger_(getLogger()) {}
 
   explicit GroupsVisitor(
       PluginGraph& pluginGraph,
+      std::unordered_set<GroupGraphVertex>& finishedVertices,
       const std::unordered_map<std::string, std::vector<vertex_t>>&
           groupsPlugins,
       const GroupGraphVertex vertexToIgnoreAsSource) :
       pluginGraph_(&pluginGraph),
       groupsPlugins_(&groupsPlugins),
+      finishedVertices_(&finishedVertices),
       vertexToIgnoreAsSource_(vertexToIgnoreAsSource),
       logger_(getLogger()) {}
 
@@ -175,7 +179,7 @@ class GroupsVisitor : public boost::dfs_visitor<> {
     // For each source plugin, add an edge to each target plugin, unless the
     // source group should be ignored (i.e. because the visitor has been
     // configured to ignore the default group's plugins as sources).
-    if (source != vertexToIgnoreAsSource_) {
+    if (!ShouldIgnoreSourceVertex(source)) {
       newBuffer = FindPluginsInGroup(source, graph);
       // Current edge is the last one in the stack.
       addEdges(newBuffer, edgeStack_.size() - 1);
@@ -191,9 +195,25 @@ class GroupsVisitor : public boost::dfs_visitor<> {
     // A forward or cross edge doesn't visit its target vertex, so pop it and
     // its buffer back off the stacks.
     PopStacks();
+
+    // Mark the source vertex as unfinishable, because none of the plugins in
+    // in the path so far can have edges added to plugins past the target
+    // vertex.
+    unfinishableVertices_.insert(boost::source(edge, graph));
   }
 
-  void finish_vertex(GroupGraphVertex, const GroupGraph&) { PopStacks(); }
+  void finish_vertex(GroupGraphVertex vertex, const GroupGraph&) {
+    // Now that the source vertex's plugins have had edges added and been
+    // added to the buffer, mark the source vertex as finished so that it
+    // won't have edges added from its plugins again in a different DFS
+    // that uses the same finished vertices set.
+    if (vertex != vertexToIgnoreAsSource_ &&
+        unfinishableVertices_.count(vertex) == 0) {
+      finishedVertices_->insert(vertex);
+    }
+
+    PopStacks();
+  }
 
 private:
   bool PathToGroupInvolvesUserMetadata(const size_t sourceGroupEdgeStackIndex,
@@ -267,6 +287,11 @@ class GroupsVisitor : public boost::dfs_visitor<> {
     return !anyEdgeSkipped;
   }
 
+  bool ShouldIgnoreSourceVertex(const GroupGraphVertex& vertex) {
+    return vertex == vertexToIgnoreAsSource_ ||
+           finishedVertices_->count(vertex) == 1;
+  }
+
   void PopStacks() {
     if (!edgeStack_.empty()) {
       edgeStack_.pop_back();
@@ -278,6 +303,7 @@ class GroupsVisitor : public boost::dfs_visitor<> {
   }
 
   PluginGraph* pluginGraph_{nullptr};
+  std::unordered_set<GroupGraphVertex>* finishedVertices_;
   const std::unordered_map<std::string, std::vector<vertex_t>>* groupsPlugins_{
       nullptr};
   std::optional<GroupGraphVertex> vertexToIgnoreAsSource_;
@@ -289,6 +315,8 @@ class GroupsVisitor : public boost::dfs_visitor<> {
   // This represents the plugins carried forward from each vertex in the path
   // to the current target vertex in the group path.
   std::vector<std::vector<vertex_t>> pluginsBuffers_;
+
+  std::unordered_set<GroupGraphVertex> unfinishableVertices_;
 };
 
 void DepthFirstVisit(
@@ -892,6 +920,8 @@ void PluginGraph::AddGroupEdges(const GroupGraph& groupGraph) {
       });
 
   // Now loop over the vertices in the groups graph.
+  //
+  std::unordered_set<GroupGraphVertex> finishedVertices;
 
   for (const auto& groupVertex : groupVertices) {
     // Run a DFS from each vertex in the group graph, adding edges except from
@@ -900,14 +930,15 @@ void PluginGraph::AddGroupEdges(const GroupGraph& groupGraph) {
     // and merge inside a given root's DAG would result in plugins from one of
     // the branches not being carried forwards past the point at which the
     // branches merge.
-    GroupsVisitor visitor(*this, groupsPlugins, defaultVertex);
+    GroupsVisitor visitor(
+        *this, finishedVertices, groupsPlugins, defaultVertex);
 
     DepthFirstVisit(groupGraph, groupVertex, visitor);
   }
 
   // Now do one last DFS starting from the default group and not ignoring its
   // plugins.
-  GroupsVisitor visitor(*this, groupsPlugins);
+  GroupsVisitor visitor(*this, finishedVertices, groupsPlugins);
 
   DepthFirstVisit(groupGraph, defaultVertex, visitor);
 }