diff --git a/src/common/paired_info/distance_estimation.cpp b/src/common/paired_info/distance_estimation.cpp index 6304f9d2a..b51936796 100644 --- a/src/common/paired_info/distance_estimation.cpp +++ b/src/common/paired_info/distance_estimation.cpp @@ -7,10 +7,10 @@ //*************************************************************************** #include "distance_estimation.hpp" +#include "pair_info_bounds.hpp" #include "assembly_graph/paths/path_processor.hpp" -namespace omnigraph { -namespace de { +namespace omnigraph::de { using namespace debruijn_graph; @@ -75,32 +75,26 @@ AbstractDistanceEstimator::OutHistogram AbstractDistanceEstimator::ClusterResult return result; } -void AbstractDistanceEstimator::AddToResult(const OutHistogram &clustered, EdgePair ep, - PairedInfoBuffer &result) const { - result.AddMany(ep.first, ep.second, clustered); -} - void DistanceEstimator::Estimate(PairedInfoIndexT &result, size_t nthreads) const { this->Init(); const auto &index = this->index(); + ConcurrentUnorderedClusteredPairedInfoBuffer buffer(graph()); - DEBUG("Collecting edge infos"); - std::vector edges; - for (EdgeId e : this->graph().edges()) - edges.push_back(e); + omnigraph::IterationHelper edges(graph()); + auto ranges = edges.Ranges(nthreads * 16); DEBUG("Processing"); - PairedInfoBuffersT buffer(this->graph(), nthreads); -# pragma omp parallel for num_threads(nthreads) schedule(guided, 10) - for (size_t i = 0; i < edges.size(); ++i) { - EdgeId edge = edges[i]; - ProcessEdge(edge, index, buffer[omp_get_thread_num()]); - } +# pragma omp parallel for schedule(guided) num_threads(nthreads) + for (size_t i = 0; i < ranges.size(); ++i) { + TRACE("Processing chunk #" << i); - for (size_t i = 0; i < nthreads; ++i) { - result.Merge(buffer[i]); - buffer[i].clear(); + for (EdgeId e : ranges[i]) { + TRACE("Estimating for edge " << e); + ProcessEdge(e, index, buffer); + } } + + result.Merge(buffer); } DistanceEstimator::EstimHist DistanceEstimator::EstimateEdgePairDistances(EdgePair ep, const InHistogram &histogram, @@ -158,7 +152,7 @@ DistanceEstimator::EstimHist DistanceEstimator::EstimateEdgePairDistances(EdgePa return result; } -void DistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi, PairedInfoBuffer &result) const { +void DistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi, Buffer &result) const { typename base::LengthMap second_edges; auto inner_map = pi.GetHalf(e1); for (auto i : inner_map) @@ -181,5 +175,6 @@ void DistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi, PairedIn this->AddToResult(res, ep, result); } } -} -} + +} // namespace omnigraph::de + diff --git a/src/common/paired_info/distance_estimation.hpp b/src/common/paired_info/distance_estimation.hpp index 9755d25d5..159c3609f 100644 --- a/src/common/paired_info/distance_estimation.hpp +++ b/src/common/paired_info/distance_estimation.hpp @@ -10,7 +10,7 @@ #define DISTANCE_ESTIMATION_HPP_ #include "paired_info.hpp" -#include "pair_info_bounds.hpp" +#include "concurrent_pair_info_buffer.hpp" #include "assembly_graph/core/graph.hpp" #include "utils/parallel/openmp_wrapper.h" @@ -78,7 +78,10 @@ class AbstractDistanceEstimator { OutHistogram ClusterResult(EdgePair /*ep*/, const EstimHist &estimated) const; - void AddToResult(const OutHistogram &clustered, EdgePair ep, PairedInfoBuffer &result) const; + template + void AddToResult(const OutHistogram &clustered, EdgePair ep, Buffer &result) const { + result.AddMany(ep.first, ep.second, clustered); + } private: const debruijn_graph::Graph &graph_; @@ -102,6 +105,7 @@ class DistanceEstimator : public AbstractDistanceEstimator { typedef typename base::OutPairedIndex OutPairedIndex; typedef typename base::InHistogram InHistogram; typedef typename base::OutHistogram OutHistogram; + typedef ConcurrentUnorderedClusteredPairedInfoBuffer Buffer; public: DistanceEstimator(const debruijn_graph::Graph &graph, @@ -128,7 +132,7 @@ class DistanceEstimator : public AbstractDistanceEstimator { private: virtual void ProcessEdge(debruijn_graph::EdgeId e1, const InPairedIndex &pi, - PairedInfoBuffer &result) const; + Buffer &result) const; virtual const std::string Name() const { static const std::string my_name = "SIMPLE"; diff --git a/src/common/paired_info/distance_estimation_utils.cpp b/src/common/paired_info/distance_estimation_utils.cpp index c67c8b62e..4601103a9 100644 --- a/src/common/paired_info/distance_estimation_utils.cpp +++ b/src/common/paired_info/distance_estimation_utils.cpp @@ -21,13 +21,13 @@ using namespace omnigraph::de; void EstimateWithEstimator(PairedInfoIndexT &clustered_index, const AbstractDistanceEstimator &estimator, AbstractPairInfoChecker &checker) { - DEBUG("Estimating distances"); + INFO("Estimating distances"); estimator.Estimate(clustered_index, omp_get_max_threads()); INFO("Filtering info"); PairInfoFilter(checker).Filter(clustered_index); - DEBUG("Info Filtered"); + INFO("Info Filtered"); } // Postprocessing, checking that clusters do not intersect @@ -130,8 +130,6 @@ void EstimatePairedDistances(PairedInfoIndexT &clustered_index, PairInfoWeightChecker checker(graph, de_config.clustered_filter_threshold); - INFO("Weight Filter Done"); - DistanceEstimator estimator(graph, paired_index, dist_finder, linkage_distance, max_distance); EstimateWithEstimator(clustered_index, estimator, checker); diff --git a/src/common/paired_info/paired_info.hpp b/src/common/paired_info/paired_info.hpp index 7a2ac6af6..b07cc326b 100644 --- a/src/common/paired_info/paired_info.hpp +++ b/src/common/paired_info/paired_info.hpp @@ -596,7 +596,7 @@ class NoLockingConstAdapter : public T { }; -//Aliases for common graphs +// Aliases for common graphs template using const_btree_map = NoLockingConstAdapter>; //Two-parameters wrapper @@ -615,7 +615,6 @@ using btree_map = NoLockingAdapter>; //Two-parameters wra template using UnclusteredPairedInfoIndexT = PairedIndex; - template class Container> class PairedIndexHandler : public omnigraph::GraphActionHandler { @@ -758,7 +757,6 @@ class PairedIndices { template using PairedInfoIndicesT = PairedIndices>; - template using PairedInfoIndexHandlerT = PairedIndexHandler; @@ -768,14 +766,6 @@ using PairedInfoIndicesHandlerT = std::vector>; template using UnclusteredPairedInfoIndicesT = PairedIndices>; -template -using unordered_map = NoLockingAdapter>; //Two-parameters wrapper -template -using PairedInfoBuffer = PairedBuffer; - -template -using PairedInfoBuffersT = PairedIndices>; - } } diff --git a/src/common/paired_info/smoothing_distance_estimation.cpp b/src/common/paired_info/smoothing_distance_estimation.cpp index f427ed0a3..640a4fada 100644 --- a/src/common/paired_info/smoothing_distance_estimation.cpp +++ b/src/common/paired_info/smoothing_distance_estimation.cpp @@ -83,7 +83,7 @@ SmoothingDistanceEstimator::EstimHist SmoothingDistanceEstimator::FindEdgePairDi } void SmoothingDistanceEstimator::ProcessEdge(EdgeId e1, const InPairedIndex &pi, - PairedInfoBuffer &result) const { + Buffer &result) const { typename base::LengthMap second_edges; auto inner_map = pi.GetHalf(e1); for (auto I : inner_map) diff --git a/src/common/paired_info/smoothing_distance_estimation.hpp b/src/common/paired_info/smoothing_distance_estimation.hpp index ea80e1d9a..5b88b7051 100644 --- a/src/common/paired_info/smoothing_distance_estimation.hpp +++ b/src/common/paired_info/smoothing_distance_estimation.hpp @@ -83,7 +83,7 @@ class SmoothingDistanceEstimator : public WeightedDistanceEstimator { void ProcessEdge(debruijn_graph::EdgeId e1, const InPairedIndex &pi, - PairedInfoBuffer &result) const override; + Buffer &result) const override; bool IsTipTip(debruijn_graph::EdgeId e1, debruijn_graph::EdgeId e2) const;