Do better in distance estimator

src/common/paired_info/distance_estimation.cpp

@@ -7,10 +7,10 @@
 //***************************************************************************
 
 #include "distance_estimation.hpp"
+#include "pair_info_bounds.hpp"
 #include "assembly_graph/paths/path_processor.hpp"
 
-namespace omnigraph {
-namespace de {
+namespace omnigraph::de {
 
 using namespace debruijn_graph;
 
@@ -75,32 +75,26 @@ AbstractDistanceEstimator::OutHistogram AbstractDistanceEstimator::ClusterResult
     return result;
 }
 
-void AbstractDistanceEstimator::AddToResult(const OutHistogram &clustered, EdgePair ep,
-                                            PairedInfoBuffer<Graph> &result) const  {
-    result.AddMany(ep.first, ep.second, clustered);
-}
-
 void DistanceEstimator::Estimate(PairedInfoIndexT<Graph> &result, size_t nthreads) const  {
     this->Init();
     const auto &index = this->index();
+    ConcurrentUnorderedClusteredPairedInfoBuffer<Graph> buffer(graph());
 
-    DEBUG("Collecting edge infos");
-    std::vector<EdgeId> edges;
-    for (EdgeId e : this->graph().edges())
-        edges.push_back(e);
+    omnigraph::IterationHelper<Graph, EdgeId> edges(graph());
+    auto ranges = edges.Ranges(nthreads * 16);
 
     DEBUG("Processing");
-    PairedInfoBuffersT<Graph> buffer(this->graph(), nthreads);
-#   pragma omp parallel for num_threads(nthreads) schedule(guided, 10)
-    for (size_t i = 0; i < edges.size(); ++i) {
-        EdgeId edge = edges[i];
-        ProcessEdge(edge, index, buffer[omp_get_thread_num()]);
-    }
+#   pragma omp parallel for schedule(guided) num_threads(nthreads)
+    for (size_t i = 0; i < ranges.size(); ++i) {
+        TRACE("Processing chunk #" << i);
 
-    for (size_t i = 0; i < nthreads; ++i) {
-        result.Merge(buffer[i]);
-        buffer[i].clear();
+        for (EdgeId e : ranges[i]) {
+            TRACE("Estimating for edge " << e);
+            ProcessEdge(e, index, buffer);
+        }
     }
+
+    result.Merge(buffer);
 }
 
 DistanceEstimator::EstimHist DistanceEstimator::EstimateEdgePairDistances(EdgePair ep, const InHistogram &histogram,
@@ -158,7 +152,7 @@ DistanceEstimator::EstimHist DistanceEstimator::EstimateEdgePairDistances(EdgePa
     return result;
 }
 
-void DistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi, PairedInfoBuffer<Graph> &result) const {
+void DistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi, Buffer &result) const {
     typename base::LengthMap second_edges;
     auto inner_map = pi.GetHalf(e1);
     for (auto i : inner_map)
@@ -181,5 +175,6 @@ void DistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi, PairedIn
         this->AddToResult(res, ep, result);
     }
 }
-}
-}
+
+} // namespace omnigraph::de
+

src/common/paired_info/distance_estimation.hpp

@@ -10,7 +10,7 @@
 #define DISTANCE_ESTIMATION_HPP_
 
 #include "paired_info.hpp"
-#include "pair_info_bounds.hpp"
+#include "concurrent_pair_info_buffer.hpp"
 
 #include "assembly_graph/core/graph.hpp"
 #include "utils/parallel/openmp_wrapper.h"
@@ -78,7 +78,10 @@ class AbstractDistanceEstimator {
 
     OutHistogram ClusterResult(EdgePair /*ep*/, const EstimHist &estimated) const;
 
-    void AddToResult(const OutHistogram &clustered, EdgePair ep, PairedInfoBuffer<debruijn_graph::Graph> &result) const;
+    template<class Buffer>
+    void AddToResult(const OutHistogram &clustered, EdgePair ep, Buffer &result) const {
+        result.AddMany(ep.first, ep.second, clustered);
+    }
 
 private:
     const debruijn_graph::Graph &graph_;
@@ -102,6 +105,7 @@ class DistanceEstimator : public AbstractDistanceEstimator {
     typedef typename base::OutPairedIndex OutPairedIndex;
     typedef typename base::InHistogram InHistogram;
     typedef typename base::OutHistogram OutHistogram;
+    typedef ConcurrentUnorderedClusteredPairedInfoBuffer<debruijn_graph::Graph> Buffer;
 
 public:
     DistanceEstimator(const debruijn_graph::Graph &graph,
@@ -128,7 +132,7 @@ class DistanceEstimator : public AbstractDistanceEstimator {
 private:
     virtual void ProcessEdge(debruijn_graph::EdgeId e1,
                              const InPairedIndex &pi,
-                             PairedInfoBuffer<debruijn_graph::Graph> &result) const;
+                             Buffer &result) const;
 
     virtual const std::string Name() const {
         static const std::string my_name = "SIMPLE";

src/common/paired_info/distance_estimation_utils.cpp

@@ -21,13 +21,13 @@ using namespace omnigraph::de;
 void EstimateWithEstimator(PairedInfoIndexT<Graph> &clustered_index,
                            const AbstractDistanceEstimator &estimator,
                            AbstractPairInfoChecker<Graph> &checker) {
-    DEBUG("Estimating distances");
+    INFO("Estimating distances");
 
     estimator.Estimate(clustered_index, omp_get_max_threads());
 
     INFO("Filtering info");
     PairInfoFilter<Graph>(checker).Filter(clustered_index);
-    DEBUG("Info Filtered");
+    INFO("Info Filtered");
 }
 
 // Postprocessing, checking that clusters do not intersect
@@ -130,8 +130,6 @@ void EstimatePairedDistances(PairedInfoIndexT<Graph> &clustered_index,
 
     PairInfoWeightChecker<Graph> checker(graph, de_config.clustered_filter_threshold);
 
-    INFO("Weight Filter Done");
-
     DistanceEstimator estimator(graph, paired_index, dist_finder, linkage_distance, max_distance);
 
     EstimateWithEstimator(clustered_index, estimator, checker);

src/common/paired_info/paired_info.hpp

@@ -596,7 +596,7 @@ class NoLockingConstAdapter : public T {
 };
 
 
-//Aliases for common graphs
+// Aliases for common graphs
 template<typename K, typename V>
 using const_btree_map = NoLockingConstAdapter<phmap::btree_map<K, V>>; //Two-parameters wrapper
 
@@ -615,7 +615,6 @@ using btree_map = NoLockingAdapter<phmap::btree_map<K, V>>; //Two-parameters wra
 template<typename Graph>
 using UnclusteredPairedInfoIndexT = PairedIndex<Graph, RawPointTraits, btree_map>;
 
-
 template<typename G, typename Traits, template<typename, typename> class Container>
 class PairedIndexHandler : public omnigraph::GraphActionHandler<G> {
 
@@ -758,7 +757,6 @@ class PairedIndices {
 template<class Graph>
 using PairedInfoIndicesT = PairedIndices<PairedInfoIndexT<Graph>>;
 
-
 template<typename Graph>
 using PairedInfoIndexHandlerT = PairedIndexHandler<Graph, PointTraits, const_btree_map>;
 
@@ -768,14 +766,6 @@ using PairedInfoIndicesHandlerT = std::vector<PairedInfoIndexHandlerT<Graph>>;
 template<class Graph>
 using UnclusteredPairedInfoIndicesT = PairedIndices<UnclusteredPairedInfoIndexT<Graph>>;
 
-template<typename K, typename V>
-using unordered_map = NoLockingAdapter<std::unordered_map<K, V>>; //Two-parameters wrapper
-template<class Graph>
-using PairedInfoBuffer = PairedBuffer<Graph, RawPointTraits, unordered_map>;
-
-template<class Graph>
-using PairedInfoBuffersT = PairedIndices<PairedInfoBuffer<Graph>>;
-
 }
 
 }

src/common/paired_info/smoothing_distance_estimation.cpp

@@ -83,7 +83,7 @@ SmoothingDistanceEstimator::EstimHist SmoothingDistanceEstimator::FindEdgePairDi
 }
 
 void SmoothingDistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi,
-                                             PairedInfoBuffer<Graph> &result) const {
+                                             Buffer &result) const {
     typename base::LengthMap second_edges;
     auto inner_map = pi.GetHalf(e1);
     for (auto I : inner_map)

src/common/paired_info/smoothing_distance_estimation.hpp

@@ -83,7 +83,7 @@ class SmoothingDistanceEstimator : public WeightedDistanceEstimator {
 
     void ProcessEdge(debruijn_graph::EdgeId e1,
                      const InPairedIndex &pi,
-                     PairedInfoBuffer<debruijn_graph::Graph> &result) const override;
+                     Buffer &result) const override;
 
     bool IsTipTip(debruijn_graph::EdgeId e1, debruijn_graph::EdgeId e2) const;