From 6bb2562ceba5dfb3275d3f3de90bd38aef570746 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 9 Feb 2023 15:54:02 +0200 Subject: [PATCH 01/28] Make hyperedge weights make sense --- dccrg.hpp | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index c51a9f3..5442103 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11476,19 +11476,7 @@ template < hyperedges[i] = item.first; if (number_of_weights_per_hyperedge > 0) { - int number_of_hyperedges = 0; - - for (const auto& neighbor_i: dccrg_instance->neighbors_of.at(item.first)) { - const auto neighbor = neighbor_i.first; - if (neighbor != 0 - /* Zoltan 3.501 crashes in hierarchial - if a cell is a neighbor to itself (periodic grid) */ - && neighbor != item.first) { - number_of_hyperedges++; - } - } - - hyperedge_weights[i] = float(1.0 * number_of_hyperedges); + hyperedge_weights[i] = get_cell_weight(i); } i++; From 1a661fbdff48f79d946a17e96f6e2d67a9b9a659 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 9 Feb 2023 16:04:02 +0200 Subject: [PATCH 02/28] Placeholder for higher dimensional weights --- dccrg.hpp | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 5442103..e74075f 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11470,16 +11470,11 @@ template < return; } - int i = 0; - for (const auto& item: dccrg_instance->cell_data) { - - hyperedges[i] = item.first; - - if (number_of_weights_per_hyperedge > 0) { - hyperedge_weights[i] = get_cell_weight(i); + for (int i = 0; i < dccrg_instance->cell_data.size(); ++i) { + hyperedges[i] = dccrg_instance->cell_data[i]; + for (int j = 0; j < number_of_weights_per_hyperedge; ++j) { + hyperedge_weights[i+j] = dccrg_instance->get_cell_weight(i); } - - i++; } } From 7405f7fe7046f8d69d3c74dc05b515347faabad4 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 9 Feb 2023 16:14:17 +0200 Subject: [PATCH 03/28] Don't write to hyperedges --- dccrg.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dccrg.hpp b/dccrg.hpp index e74075f..90b8341 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11471,7 +11471,6 @@ template < } for (int i = 0; i < dccrg_instance->cell_data.size(); ++i) { - hyperedges[i] = dccrg_instance->cell_data[i]; for (int j = 0; j < number_of_weights_per_hyperedge; ++j) { hyperedge_weights[i+j] = dccrg_instance->get_cell_weight(i); } From fe2dcb863f517889c8f309eee6d851f9d026ae0e Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 15 Feb 2023 11:23:41 +0200 Subject: [PATCH 04/28] Support separate weights for hyperedges --- dccrg.hpp | 97 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 24 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 90b8341..de0d6bb 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -365,6 +365,7 @@ template < no_load_balancing(other.get_no_load_balancing()), reserved_options(other.get_reserved_options()), cell_weights(other.get_cell_weights()), + communication_weights(other.get_communication_weights()), neighbor_processes(other.get_neighbor_processes()), balancing_load(other.get_balancing_load()) { @@ -803,6 +804,9 @@ template < return NULL; } + void set_partitioning_neighborhood_id (int id) { + partitioning_neighborhood_id = id; + } /*! Returns a pointer to the cells that consider given cell as a neighbor. @@ -3422,14 +3426,20 @@ template < } // children of refined cells inherit their weight - if (this->rank == process_of_refined - && this->cell_weights.count(refined) > 0) { + if (this->rank == process_of_refined && this->cell_weights.count(refined) > 0) { for (const uint64_t child: children) { this->cell_weights[child] = this->cell_weights.at(refined); } this->cell_weights.erase(refined); } + if (this->rank == process_of_refined && this->communication_weights.count(refined) > 0) { + for (const uint64_t child: children) { + this->communication_weights[child] = this->communication_weights.at(refined); + } + this->communication_weights.erase(refined); + } + // use local neighbor lists to find cells whose neighbor lists have to updated if (this->rank == process_of_refined) { // update the neighbor lists of created local cells @@ -3589,6 +3599,7 @@ template < this->pin_requests.erase(unrefined); this->new_pin_requests.erase(unrefined); this->cell_weights.erase(unrefined); + this->communication_weights.erase(unrefined); // don't send unrefined cells' user data to self if (this->rank == process_of_unrefined @@ -4266,6 +4277,7 @@ template < this->cells_not_to_refine.clear(); this->cells_not_to_unrefine.clear(); this->cell_weights.clear(); + this->communication_weights.clear(); #ifdef DEBUG // check that there are no duplicate adds / removes @@ -6456,6 +6468,25 @@ template < return true; } + bool set_communication_weight(const uint64_t cell, const double weight) + { + if (this->cell_process.count(cell) == 0) { + return false; + } + + if (this->cell_process.at(cell) != this->rank) { + return false; + } + + if (cell != this->get_child(cell)) { + return false; + } + + this->communication_weights[cell] = weight; + + return true; + } + /*! Returns the weight of given local existing cell without children. @@ -6476,13 +6507,31 @@ template < return std::numeric_limits::quiet_NaN(); } - if (this->cell_weights.count(cell) == 0) { - return 1; - } else { - return this->cell_weights.at(cell); - } + return this->cell_weights.count(cell) ? this->cell_weights.at(cell) : 1.0; } + /*! + Returns the weight of given local existing cell without children. + + Returns a quiet nan if above conditions are not met. + Unset cell weights are assumed to be 1. + */ + double get_communication_weight(const uint64_t cell) const + { + if (this->cell_process.count(cell) == 0) { + return std::numeric_limits::quiet_NaN(); + } + + if (this->cell_process.at(cell) != this->rank) { + return std::numeric_limits::quiet_NaN(); + } + + if (cell != this->get_child(cell)) { + return std::numeric_limits::quiet_NaN(); + } + + return this->communication_weights.count(cell) ? this->communication_weights.at(cell) : 1.0; + } /*! Returns the cells that will be added to this process by load balancing. @@ -6617,6 +6666,10 @@ template < return this->cell_weights; } + const std::unordered_map& get_communication_weights() const + { + return this->communication_weights; + } /*! Adds a new neighborhood for updating Cell_Data between neighbors on different processes. @@ -7345,6 +7398,8 @@ template < std::vector::neighborhood_item_t> > user_hood_of, user_hood_to; + int partitioning_neighborhood_id{default_neighborhood_id}; + /*! Cell on this process and those cells that aren't neighbors of this cell but whose neighbor this cell is. @@ -7458,9 +7513,12 @@ template < // reserved options that the user cannot change std::unordered_set reserved_options; - // optional user-given weights of cells on this process + // optional user-given processing weights of cells on this process std::unordered_map cell_weights; + // optional user-given communication weights of cells on this process + std::unordered_map communication_weights; + // processes which have cells close enough from cells of this process std::unordered_set neighbor_processes; @@ -11147,11 +11205,7 @@ template < global_ids[i] = item.first; if (number_of_weights_per_object > 0) { - if (dccrg_instance->cell_weights.count(item.first) > 0) { - object_weights[i] = float(dccrg_instance->cell_weights.at(item.first)); - } else { - object_weights[i] = 1; - } + object_weights[i] = dccrg_instance->get_cell_weight(item.first); } i++; @@ -11314,12 +11368,11 @@ template < (*number_of_connections)++; - for (const auto& neighbor_i: dccrg_instance->neighbors_of.at(item.first)) { + for (const auto& neighbor_i: *dccrg_instance->get_neighbors_of(item.first, dccrg_instance->partitioning_neighborhood_id)) { const auto& neighbor = neighbor_i.first; - if (neighbor != 0 /* Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself */ - && neighbor != item.first) { + if (neighbor != 0 && neighbor != item.first) { (*number_of_connections)++; } } @@ -11381,12 +11434,11 @@ template < // add a connection to the cell itself from its hyperedge connections[connection_number++] = item.first; - for (const auto& neighbor_i: dccrg_instance->neighbors_of.at(item.first)) { + for (const auto& neighbor_i: *dccrg_instance->get_neighbors_of(item.first, dccrg_instance->partitioning_neighborhood_id)) { const auto& neighbor = neighbor_i.first; - if (neighbor == 0 /* Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself */ - || neighbor == item.first) { + if (neighbor == 0 || neighbor == item.first) { continue; } @@ -11471,8 +11523,8 @@ template < } for (int i = 0; i < dccrg_instance->cell_data.size(); ++i) { - for (int j = 0; j < number_of_weights_per_hyperedge; ++j) { - hyperedge_weights[i+j] = dccrg_instance->get_cell_weight(i); + if (number_of_weights_per_hyperedge) { + hyperedge_weights[i] = dccrg_instance->get_communication_weight(i); } } } @@ -11572,7 +11624,6 @@ template < std::tuple >* >(data); - if (level < 0 || level >= int(dccrg_instance->processes_per_part.size())) { std::cerr << "Zoltan wanted partitioning options for an invalid hierarchy " @@ -11595,8 +11646,6 @@ template < } } - - #ifdef DEBUG /*! Returns false if the same cells don't exist on the same process for all processes. From 595168967e7ceabbeb1fde54aa9f4277b38782d4 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 15 Feb 2023 13:54:49 +0200 Subject: [PATCH 05/28] Use mutual neighbors for accuracy --- dccrg.hpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index de0d6bb..ddcc79a 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -804,6 +804,21 @@ template < return NULL; } + const std::vector>> get_mutual_neighbors( + const uint64_t cell, + const int neighborhood_id = default_neighborhood_id + ) const { + std::vector>> neighbors; + for (auto& i : *get_neighbors_of(cell, neighborhood_id)) { + for (auto& j : *get_neighbors_to(cell, neighborhood_id)) { + if (i.first == j.first) { + neighbors.push_back(i); + } + } + } + return neighbors; + } + void set_partitioning_neighborhood_id (int id) { partitioning_neighborhood_id = id; } @@ -11368,7 +11383,7 @@ template < (*number_of_connections)++; - for (const auto& neighbor_i: *dccrg_instance->get_neighbors_of(item.first, dccrg_instance->partitioning_neighborhood_id)) { + for (const auto& neighbor_i: dccrg_instance->get_mutual_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { const auto& neighbor = neighbor_i.first; /* Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself */ @@ -11434,7 +11449,7 @@ template < // add a connection to the cell itself from its hyperedge connections[connection_number++] = item.first; - for (const auto& neighbor_i: *dccrg_instance->get_neighbors_of(item.first, dccrg_instance->partitioning_neighborhood_id)) { + for (const auto& neighbor_i: dccrg_instance->get_mutual_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { const auto& neighbor = neighbor_i.first; /* Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself */ From 013e5a74b818b9211ad4b1c243d8b28d6fc791ef Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 22 Feb 2023 17:00:28 +0200 Subject: [PATCH 06/28] Attempt to improve hyperedges --- dccrg.hpp | 161 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 129 insertions(+), 32 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index ddcc79a..274d804 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -2669,39 +2669,40 @@ template < // get location of face neighbors' offsets in neighborhood_of std::array neighborhood_of_indices = {{0, 0, 0, 0, 0, 0}}; - for (int direction = -1; direction <= 1; direction += 2) - for (size_t dimension = 0; dimension < 3; dimension++) { + for (int direction : {-1, 1}) { + for (size_t dimension = 0; dimension < 3; dimension++) { - // neigh_of_indices[n] == negative direction in dimension n, - // n + 1 positive direction - const size_t neigh_of_indices_index - = 2 * dimension + ((direction > 0) ? 1 : 0); + // neigh_of_indices[n] == negative direction in dimension n, + // n + 1 positive direction + const size_t neigh_of_indices_index + = 2 * dimension + ((direction > 0) ? 1 : 0); - for (size_t i = 0; i <= this->neighborhood_of.size(); i++) { - if (i == this->neighborhood_of.size()) { - std::cerr << __FILE__ << ":" << __LINE__ - << " Neighborhood_of offsets not found for face neighbors in dimension: " - << dimension << ", direction: " << direction - << std::endl; - abort(); - } + for (size_t i = 0; i <= this->neighborhood_of.size(); i++) { + if (i == this->neighborhood_of.size()) { + std::cerr << __FILE__ << ":" << __LINE__ + << " Neighborhood_of offsets not found for face neighbors in dimension: " + << dimension << ", direction: " << direction + << std::endl; + abort(); + } - bool found = true; - for (size_t other_dims = 0; other_dims < 3; other_dims++) { - if (other_dims != dimension - && this->neighborhood_of[i][other_dims] != 0) { - found = false; - break; + bool found = true; + for (size_t other_dims = 0; other_dims < 3; other_dims++) { + if (other_dims != dimension + && this->neighborhood_of[i][other_dims] != 0) { + found = false; + break; + } } - } - if (this->neighborhood_of[i][dimension] != direction) { - found = false; - } + if (this->neighborhood_of[i][dimension] != direction) { + found = false; + } - if (found) { - neighborhood_of_indices[neigh_of_indices_index] = i; - break; + if (found) { + neighborhood_of_indices[neigh_of_indices_index] = i; + break; + } } } } @@ -2814,6 +2815,50 @@ template < return ret_val; } + // Recursively find Vlasov solver neighbors + // Janky, and most likely incredibly inefficient + std::set get_vlasov_neighbors( + const uint64_t cell + ) const { + int stencil_width {0}; + switch (neighborhood_length) { + case 1: + break; + case 3: + stencil_width = 2; + break; + case 5: + stencil_width = 3; + break; + default: + // Placeholder error + std::cerr << "Weird stencil width" << std::endl; + break; + } + + std::set ret; + auto last_neighbors {get_face_neighbors_of(cell)}; + for (auto p : last_neighbors) { + ret.insert(p.first); + } + + for (int i = 1; i < stencil_width; ++i) { + std::vector> new_neighbors; + for (auto p : last_neighbors) { + for (auto pp : get_face_neighbors_of(p.first)) { + if (p.second == pp.second) { + new_neighbors.push_back(pp); + ret.insert(pp.first); + } + } + } + + last_neighbors = new_neighbors; + } + + return ret; + } + /*! Returns true if given cell's neighbor types match given criterion, false otherwise. @@ -7848,7 +7893,7 @@ template < Set reserved options */ // 0 because Zoltan crashes in hierarchial with larger values - Zoltan_Set_Param(this->zoltan, "EDGE_WEIGHT_DIM", "0"); + Zoltan_Set_Param(this->zoltan, "EDGE_WEIGHT_DIM", "1"); Zoltan_Set_Param(this->zoltan, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(this->zoltan, "NUM_LID_ENTRIES", "0"); Zoltan_Set_Param(this->zoltan, "OBJ_WEIGHT_DIM", "1"); @@ -7928,6 +7973,17 @@ template < this ); + Zoltan_Set_Obj_Size_Multi_Fn( + this->zoltan, + &Dccrg< + Cell_Data, + Geometry, + std::tuple, + std::tuple + >::fill_with_communication_weights, + this + ); + Zoltan_Set_HG_Size_CS_Fn( this->zoltan, &Dccrg< @@ -11351,6 +11407,49 @@ template < } + /*! + Fills sizes with communication weights of cells in global_ids + */ + static void fill_with_communication_weights( + void *data, + int /*global_id_size*/, + int /*local_id_size*/, + int number_of_cells, + ZOLTAN_ID_PTR global_ids, + ZOLTAN_ID_PTR /*local_ids*/, + int *sizes, + int *error + ) { + Dccrg< + Cell_Data, + Geometry, + std::tuple, + std::tuple + >* dccrg_instance = reinterpret_cast< + Dccrg< + Cell_Data, + Geometry, + std::tuple, + std::tuple + >* + >(data); + *error = ZOLTAN_OK; + + for (int i = 0; i < number_of_cells; i++) { + uint64_t cell = uint64_t(global_ids[i]); + if (dccrg_instance->cell_data.count(cell) == 0) { + *error = ZOLTAN_FATAL; + std::cerr << "Process " << dccrg_instance->rank + << ": Zoltan wanted the weight of a non-existing cell " << cell + << std::endl; + return; + } + + sizes[i] = dccrg_instance->get_communication_weight(i); + } + } + + /*! Writes the number of hyperedges (self + one per neighbor cell) in the grid for all cells on this process. */ @@ -11383,8 +11482,7 @@ template < (*number_of_connections)++; - for (const auto& neighbor_i: dccrg_instance->get_mutual_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { - const auto& neighbor = neighbor_i.first; + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first)) { /* Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself */ if (neighbor != 0 && neighbor != item.first) { @@ -11449,8 +11547,7 @@ template < // add a connection to the cell itself from its hyperedge connections[connection_number++] = item.first; - for (const auto& neighbor_i: dccrg_instance->get_mutual_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { - const auto& neighbor = neighbor_i.first; + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first)) { /* Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself */ if (neighbor == 0 || neighbor == item.first) { From 33478798b6d9f1c4369481dc7c49ac7d5113dde2 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 22 Feb 2023 17:07:41 +0200 Subject: [PATCH 07/28] Revert "Don't write to hyperedges" This reverts commit 509221b750d5f23e53651f3297e167217f46e3f3. --- dccrg.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 274d804..336779e 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11445,7 +11445,7 @@ template < return; } - sizes[i] = dccrg_instance->get_communication_weight(i); + sizes[i] = dccrg_instance->get_communication_weight(cell); } } @@ -11634,10 +11634,13 @@ template < return; } - for (int i = 0; i < dccrg_instance->cell_data.size(); ++i) { + int i = 0; + for (const auto& item : dccrg_instance->cell_data) { + hyperedges[i] = item.first; if (number_of_weights_per_hyperedge) { - hyperedge_weights[i] = dccrg_instance->get_communication_weight(i); + hyperedge_weights[i] = dccrg_instance->get_communication_weight(item.first); } + ++i; } } From 048d8c16cc6cd6feeaf8dd6c31566660c1aab3ba Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 27 Feb 2023 10:22:45 +0200 Subject: [PATCH 08/28] Hardcpded hierarchical partitioning --- dccrg.hpp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 336779e..5cf53c3 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -2838,14 +2838,14 @@ template < std::set ret; auto last_neighbors {get_face_neighbors_of(cell)}; - for (auto p : last_neighbors) { - ret.insert(p.first); + for (const auto& [id, dir] : last_neighbors) { + ret.insert(id); } for (int i = 1; i < stencil_width; ++i) { std::vector> new_neighbors; - for (auto p : last_neighbors) { - for (auto pp : get_face_neighbors_of(p.first)) { + for (const auto& p : last_neighbors) { + for (const auto pp : get_face_neighbors_of(p.first)) { if (p.second == pp.second) { new_neighbors.push_back(pp); ret.insert(pp.first); @@ -7879,6 +7879,15 @@ template < this->no_load_balancing = false; } + // Hardcoded for now + if (this->load_balancing_method == "HIER") { + add_partitioning_level(4); // Level 0 - Nodes + add_partitioning_option(0, "LB_METHOD", "HYPERGRAPH"); + + add_partitioning_level(1); // Level 1 - Processes + add_partitioning_option(1, "LB_METHOD", "RCB"); + } + // reserved options that the user cannot change this->reserved_options.insert("EDGE_WEIGHT_DIM"); this->reserved_options.insert("NUM_GID_ENTRIES"); @@ -11323,8 +11332,7 @@ template < } number_of_neighbors[i] = 0; - for (const auto& neighbor_i: dccrg_instance->neighbors_of.at(cell)) { - const auto& neighbor = neighbor_i.first; + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell)) { if (neighbor != 0 /* Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself */ @@ -11380,8 +11388,7 @@ template < number_of_neighbors[i] = 0; - for (const auto& neighbor_i: dccrg_instance->neighbors_of.at(cell)) { - const auto& neighbor = neighbor_i.first; + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell)) { if (neighbor == 0 /* Zoltan 3.501 crashes in hierarchial @@ -11398,7 +11405,7 @@ template < // weight of edge from cell to *neighbor if (number_of_weights_per_edge > 0) { - edge_weights[current_neighbor_number] = 1.0; + edge_weights[current_neighbor_number] = dccrg_instance->get_communication_weight(neighbor); } current_neighbor_number++; From 9635facf515b00bcd0d6cba27190e7e56d5d728c Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 28 Feb 2023 16:54:15 +0200 Subject: [PATCH 09/28] Better vlasov neighbor finding --- dccrg.hpp | 77 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 5cf53c3..f998527 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -2815,10 +2815,27 @@ template < return ret_val; } - // Recursively find Vlasov solver neighbors - // Janky, and most likely incredibly inefficient - std::set get_vlasov_neighbors( - const uint64_t cell + // Get maximum displacement between cells a and b + double get_maximum_displacement( + const uint64_t a, + const uint64_t b + ) const { + auto x_a = get_center(a); + auto x_b = get_center(b); + double ret {0}; + for (int i = 0; i < 3; ++i) { + double r = std::abs(x_a[i] - x_b[i]); + ret = r > ret ? r : ret; + } + return ret; + } + + // Get Vlasov stencil neighbors by calculating distance + // Assumes neighborhood given is a + -shaped stencil + // Probably inefficient + std::vector get_vlasov_neighbors( + const uint64_t cell, + const int neighborhood_id = default_neighborhood_id ) const { int stencil_width {0}; switch (neighborhood_length) { @@ -2835,25 +2852,18 @@ template < std::cerr << "Weird stencil width" << std::endl; break; } + + std::vector ret; + int my_ref {mapping.get_refinement_level(cell)}; - std::set ret; - auto last_neighbors {get_face_neighbors_of(cell)}; - for (const auto& [id, dir] : last_neighbors) { - ret.insert(id); - } + for (auto& [neigh, dir] : *get_neighbors_of(cell, neighborhood_id)) { + double r {get_maximum_displacement(cell, neigh)}; + int other_ref {mapping.get_refinement_level(neigh)}; - for (int i = 1; i < stencil_width; ++i) { - std::vector> new_neighbors; - for (const auto& p : last_neighbors) { - for (const auto pp : get_face_neighbors_of(p.first)) { - if (p.second == pp.second) { - new_neighbors.push_back(pp); - ret.insert(pp.first); - } - } + // 0.1 purely for floating point errors, assume cubical cells + if (r < (stencil_width + 0.1) * geometry.get_length((my_ref < other_ref) ? neigh : cell)[0]) { + ret.push_back(neigh); } - - last_neighbors = new_neighbors; } return ret; @@ -11332,11 +11342,9 @@ template < } number_of_neighbors[i] = 0; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell)) { - if (neighbor != 0 - /* Zoltan 3.501 crashes in hierarchial - if a cell is a neighbor to itself */ - && neighbor != cell) { + for (const auto& neighbor : dccrg_instance->get_vlasov_neighbors(cell, dccrg_instance->partitioning_neighborhood_id)) { + // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself + if (neighbor != 0 && neighbor != cell) { number_of_neighbors[i]++; } } @@ -11388,12 +11396,9 @@ template < number_of_neighbors[i] = 0; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell)) { - - if (neighbor == 0 - /* Zoltan 3.501 crashes in hierarchial - if a cell is a neighbor to itself */ - || neighbor == cell) { + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell, dccrg_instance->partitioning_neighborhood_id)) { + // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself + if (neighbor == 0 || neighbor == cell) { continue; } @@ -11489,9 +11494,8 @@ template < (*number_of_connections)++; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first)) { - /* Zoltan 3.501 crashes in hierarchial - if a cell is a neighbor to itself */ + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { + // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself if (neighbor != 0 && neighbor != item.first) { (*number_of_connections)++; } @@ -11554,9 +11558,8 @@ template < // add a connection to the cell itself from its hyperedge connections[connection_number++] = item.first; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first)) { - /* Zoltan 3.501 crashes in hierarchial - if a cell is a neighbor to itself */ + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { + // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself if (neighbor == 0 || neighbor == item.first) { continue; } From 6cb6f03d735c9b5416ef3bf423793434812d877d Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 3 Mar 2023 15:37:28 +0200 Subject: [PATCH 10/28] Attempt to make Vlasov neighbors more accurate --- dccrg.hpp | 101 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 26 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index f998527..665404a 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -2815,27 +2815,82 @@ template < return ret_val; } - // Get maximum displacement between cells a and b - double get_maximum_displacement( - const uint64_t a, - const uint64_t b - ) const { - auto x_a = get_center(a); - auto x_b = get_center(b); - double ret {0}; + // Consider giving index as argument to reduce calculations + static std::array distance_to_double(const std::array& in) + { + std::array ret; for (int i = 0; i < 3; ++i) { - double r = std::abs(x_a[i] - x_b[i]); - ret = r > ret ? r : ret; + ret[i] = static_cast(in[i]) / (in[3] > 0 ? in[3] : 1.0); } return ret; } - // Get Vlasov stencil neighbors by calculating distance - // Assumes neighborhood given is a + -shaped stencil - // Probably inefficient - std::vector get_vlasov_neighbors( + std::map> get_vlasov_neighbors( const uint64_t cell, - const int neighborhood_id = default_neighborhood_id + const int neighborhood_id, + const int dimension, + const int stencil_width + ) const { + std::map> ret; + int my_ref {mapping.get_refinement_level(cell)}; + + std::vector>> neighs; + const auto* p = get_neighbors_of(cell, neighborhood_id); + if (!p) { + return ret; + } + + neighs = *p; + // We still need to sort since more refined cells are in xyz order + std::stable_sort(neighs.begin(), neighs.end(), + [this, dimension] (const std::pair>& a, const std::pair>& b) { + return distance_to_double(a.second)[dimension] < distance_to_double(b.second)[dimension]; + } + ); + + auto it = neighs.begin(); + while (it < neighs.end() && it->second[dimension] < 0) { + ++it; + } + + int found {0}; + for (auto neg = it - 1; neg >= neighs.begin(); --neg) { + double offset = distance_to_double(neg->second)[dimension]; + if (ret.count(offset)) { + ret[offset].push_back(neg->first); + } else if (found < stencil_width) { + ++found; + ret[offset] = std::vector {neg->first}; + } else { + break; + } + } + + while (it < neighs.end() && it->second[dimension] <= 0) { + ++it; + } + + found = 0; + for (auto pos = it; pos < neighs.end(); ++pos) { + double offset = distance_to_double(pos->second)[dimension]; + if (ret.count(offset)) { + ret[offset].push_back(pos->first); + } else if (found < stencil_width) { + ++found; + ret[offset] = std::vector {pos->first}; + } else { + break; + } + } + + return ret; + } + + // Get actual Vlasov stencil neighbors + // Assumes linear 3 linear stencils from neighborhood_id to neighborhood_id + 2 + std::set get_vlasov_neighbors( + const uint64_t cell, + const int neighborhood_id ) const { int stencil_width {0}; switch (neighborhood_length) { @@ -2852,20 +2907,13 @@ template < std::cerr << "Weird stencil width" << std::endl; break; } - - std::vector ret; - int my_ref {mapping.get_refinement_level(cell)}; - for (auto& [neigh, dir] : *get_neighbors_of(cell, neighborhood_id)) { - double r {get_maximum_displacement(cell, neigh)}; - int other_ref {mapping.get_refinement_level(neigh)}; - - // 0.1 purely for floating point errors, assume cubical cells - if (r < (stencil_width + 0.1) * geometry.get_length((my_ref < other_ref) ? neigh : cell)[0]) { - ret.push_back(neigh); + std::set ret; + for (int dim = 0; dim < 3; ++dim) { + for (auto& [dist, cells] : get_vlasov_neighbors(cell, neighborhood_id + dim, dim, stencil_width)) { + ret.insert(cells.begin(), cells.end()); } } - return ret; } @@ -7896,6 +7944,7 @@ template < add_partitioning_level(1); // Level 1 - Processes add_partitioning_option(1, "LB_METHOD", "RCB"); + //add_partitioning_option(1, "LB_METHOD", "RIB"); } // reserved options that the user cannot change From 0486bffe410c64be13897d8993e8e1e661c66c5d Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 9 Mar 2023 11:02:25 +0200 Subject: [PATCH 11/28] Don't use double as map key --- dccrg.hpp | 83 +++++++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 46 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 665404a..a31dbf6 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -819,7 +819,7 @@ template < return neighbors; } - void set_partitioning_neighborhood_id (int id) { + void set_partitioning_neighborhood (int id) { partitioning_neighborhood_id = id; } @@ -2820,23 +2820,23 @@ template < { std::array ret; for (int i = 0; i < 3; ++i) { - ret[i] = static_cast(in[i]) / (in[3] > 0 ? in[3] : 1.0); + ret[i] = static_cast(in[i]) / static_cast(in[3] > 0 ? in[3] : 1); } return ret; } - std::map> get_vlasov_neighbors( + std::map> get_vlasov_neighbors( const uint64_t cell, const int neighborhood_id, const int dimension, const int stencil_width ) const { - std::map> ret; + std::map> ret; int my_ref {mapping.get_refinement_level(cell)}; - std::vector>> neighs; const auto* p = get_neighbors_of(cell, neighborhood_id); if (!p) { + std::cerr << "Cell " << cell << ", neighborhood " << neighborhood_id << ", dimension " << dimension << " not found" << std::endl; return ret; } @@ -2848,40 +2848,46 @@ template < } ); - auto it = neighs.begin(); - while (it < neighs.end() && it->second[dimension] < 0) { + auto it = neighs.cbegin(); + while (it < neighs.cend() && it->second[dimension] < 0) { ++it; } int found {0}; - for (auto neg = it - 1; neg >= neighs.begin(); --neg) { + std::uint64_t previous_cell {error_cell}; + double previous_offset {0.0}; + for (auto neg = std::make_reverse_iterator(it); neg != neighs.crend(); ++neg) { double offset = distance_to_double(neg->second)[dimension]; - if (ret.count(offset)) { - ret[offset].push_back(neg->first); - } else if (found < stencil_width) { - ++found; - ret[offset] = std::vector {neg->first}; - } else { - break; + if (neg->first == previous_cell || neg->first == error_cell) { + continue; + } else if (offset < previous_offset) { + if (++found > stencil_width) + break; } + + ret[-found].push_back(neg->first); + previous_cell = neg->first; } - while (it < neighs.end() && it->second[dimension] <= 0) { + while (it < neighs.cend() && it->second[dimension] <= 0) { ++it; } found = 0; - for (auto pos = it; pos < neighs.end(); ++pos) { + previous_cell = error_cell; + previous_offset = 0.0; + for (auto pos = it; pos != neighs.cend(); ++pos) { double offset = distance_to_double(pos->second)[dimension]; - if (ret.count(offset)) { - ret[offset].push_back(pos->first); - } else if (found < stencil_width) { - ++found; - ret[offset] = std::vector {pos->first}; - } else { - break; + if (pos->first == previous_cell || pos->first == error_cell) { + continue; + } else if (offset > previous_offset) { + if (++found > stencil_width) + break; } - } + + ret[found].push_back(pos->first); + previous_cell = pos->first; + } return ret; } @@ -2889,8 +2895,7 @@ template < // Get actual Vlasov stencil neighbors // Assumes linear 3 linear stencils from neighborhood_id to neighborhood_id + 2 std::set get_vlasov_neighbors( - const uint64_t cell, - const int neighborhood_id + const uint64_t cell ) const { int stencil_width {0}; switch (neighborhood_length) { @@ -2910,7 +2915,7 @@ template < std::set ret; for (int dim = 0; dim < 3; ++dim) { - for (auto& [dist, cells] : get_vlasov_neighbors(cell, neighborhood_id + dim, dim, stencil_width)) { + for (auto& [dist, cells] : get_vlasov_neighbors(cell, partitioning_neighborhood_id + dim, dim, stencil_width)) { ret.insert(cells.begin(), cells.end()); } } @@ -11390,13 +11395,7 @@ template < return; } - number_of_neighbors[i] = 0; - for (const auto& neighbor : dccrg_instance->get_vlasov_neighbors(cell, dccrg_instance->partitioning_neighborhood_id)) { - // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself - if (neighbor != 0 && neighbor != cell) { - number_of_neighbors[i]++; - } - } + number_of_neighbors[i] = dccrg_instance->get_vlasov_neighbors(cell).size(); } } @@ -11445,7 +11444,7 @@ template < number_of_neighbors[i] = 0; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell, dccrg_instance->partitioning_neighborhood_id)) { + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself if (neighbor == 0 || neighbor == cell) { continue; @@ -11540,15 +11539,7 @@ template < *number_of_connections = 0; for (const auto& item: dccrg_instance->cell_data) { - - (*number_of_connections)++; - - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { - // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself - if (neighbor != 0 && neighbor != item.first) { - (*number_of_connections)++; - } - } + (*number_of_connections) += 1 + dccrg_instance->get_vlasov_neighbors(item.first).size(); } } @@ -11607,7 +11598,7 @@ template < // add a connection to the cell itself from its hyperedge connections[connection_number++] = item.first; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first, dccrg_instance->partitioning_neighborhood_id)) { + for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself if (neighbor == 0 || neighbor == item.first) { continue; From f0485f4ba77e423e38573fd9d69d126a9f7dd70d Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 10 Mar 2023 11:34:37 +0200 Subject: [PATCH 12/28] Copypaste vlasov neighbors from vlasiator --- dccrg.hpp | 109 +++++++++++++++++++++++++++++------------------------- 1 file changed, 58 insertions(+), 51 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index a31dbf6..912f15b 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -2825,69 +2825,77 @@ template < return ret; } - std::map> get_vlasov_neighbors( + std::set get_vlasov_neighbors( const uint64_t cell, const int neighborhood_id, const int dimension, const int stencil_width ) const { - std::map> ret; - int my_ref {mapping.get_refinement_level(cell)}; - std::vector>> neighs; + std::set ret; const auto* p = get_neighbors_of(cell, neighborhood_id); if (!p) { std::cerr << "Cell " << cell << ", neighborhood " << neighborhood_id << ", dimension " << dimension << " not found" << std::endl; return ret; } - neighs = *p; - // We still need to sort since more refined cells are in xyz order - std::stable_sort(neighs.begin(), neighs.end(), - [this, dimension] (const std::pair>& a, const std::pair>& b) { - return distance_to_double(a.second)[dimension] < distance_to_double(b.second)[dimension]; + // Create list of unique distances + std::set distances_plus; + std::set distances_minus; + std::set found_neighbors_plus; + std::set found_neighbors_minus; + /** Using sets of cells as well, we should only get one distance per + (potentially less refined) cell. This should result in safe behaviour + as long as the neighborhood of a cell does not contain cells with a + refinement level more than 1 level apart from the cell itself. + */ + for (const auto& [neighbor, coords] : *p) { + if(coords[dimension] > 0) { + if (!found_neighbors_plus.count(neighbor)) { + distances_plus.insert(coords[dimension]); + found_neighbors_plus.insert(neighbor); + } } - ); - - auto it = neighs.cbegin(); - while (it < neighs.cend() && it->second[dimension] < 0) { - ++it; - } - - int found {0}; - std::uint64_t previous_cell {error_cell}; - double previous_offset {0.0}; - for (auto neg = std::make_reverse_iterator(it); neg != neighs.crend(); ++neg) { - double offset = distance_to_double(neg->second)[dimension]; - if (neg->first == previous_cell || neg->first == error_cell) { - continue; - } else if (offset < previous_offset) { - if (++found > stencil_width) - break; + if(coords[dimension] < 0) { + if (!found_neighbors_minus.count(neighbor)) { + distances_minus.insert(-coords[dimension]); + found_neighbors_minus.insert(neighbor); + } } - - ret[-found].push_back(neg->first); - previous_cell = neg->first; } - while (it < neighs.cend() && it->second[dimension] <= 0) { - ++it; - } - - found = 0; - previous_cell = error_cell; - previous_offset = 0.0; - for (auto pos = it; pos != neighs.cend(); ++pos) { - double offset = distance_to_double(pos->second)[dimension]; - if (pos->first == previous_cell || pos->first == error_cell) { - continue; - } else if (offset > previous_offset) { - if (++found > stencil_width) - break; - } - - ret[found].push_back(pos->first); - previous_cell = pos->first; - } + int iSrc = stencil_width - 1; + // Iterate through positive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + for (const auto& distance : distances_plus) { + if (iSrc < 0) + break; // found enough elements + for (const auto& [neighbor, coords] : *p) { + if (neighbor == error_cell) + continue; + if (coords[dimension] == distance) { + if (ret.count(neighbor)) + continue; + ret.insert(neighbor); + } + } // end loop over neighbors + iSrc--; + } // end loop over positive distances + + iSrc = stencil_width - 1; + // Iterate through negtive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. + for (const auto& distance : distances_minus) { + if (iSrc < 0) + break; // found enough elements + for (const auto& [neighbor, coords] : *p) { + if (neighbor == error_cell) + continue; + if (coords[dimension] == distance) { + if (ret.count(neighbor)) + continue; + ret.insert(neighbor); + } + } // end loop over neighbors + iSrc--; + } // end loop over negative distances return ret; } @@ -2915,9 +2923,8 @@ template < std::set ret; for (int dim = 0; dim < 3; ++dim) { - for (auto& [dist, cells] : get_vlasov_neighbors(cell, partitioning_neighborhood_id + dim, dim, stencil_width)) { - ret.insert(cells.begin(), cells.end()); - } + auto neighs_dim = get_vlasov_neighbors(cell, partitioning_neighborhood_id + dim, dim, stencil_width); + ret.insert(neighs_dim.begin(), neighs_dim.end()); } return ret; } From 47e26204e6b49626f08356798986591e5923dc1d Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 1 Jun 2023 14:22:03 +0300 Subject: [PATCH 13/28] Some customization for HIER --- dccrg.hpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 912f15b..bf02485 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -7480,6 +7480,7 @@ template < unsigned int neighborhood_length = 1; std::string load_balancing_method{"RCB"}; + int hierarchical_level_zero {16}; // maximum value an MPI tag can have unsigned int max_tag; @@ -7951,12 +7952,12 @@ template < // Hardcoded for now if (this->load_balancing_method == "HIER") { - add_partitioning_level(4); // Level 0 - Nodes + add_partitioning_level(this->hierarchical_level_zero); // Level 0 - Nodes add_partitioning_option(0, "LB_METHOD", "HYPERGRAPH"); add_partitioning_level(1); // Level 1 - Processes - add_partitioning_option(1, "LB_METHOD", "RCB"); - //add_partitioning_option(1, "LB_METHOD", "RIB"); + add_partitioning_option(1, "LB_METHOD", "RIB"); + //add_partitioning_option(1, "LB_METHOD", "RCB"); } // reserved options that the user cannot change @@ -8486,6 +8487,23 @@ template < const std::string& get_load_balancing_method() const { return this->load_balancing_method; } + + /*! + Sets amount of tasks for node-level of HIER partition for Zoltan + + Must be called before initialize(). + + \see balance_load() + */ + Dccrg< + Cell_Data, + Geometry, + std::tuple, + std::tuple + >& set_load_balancing_hier_level_zero(const int n) { + this->hierarchical_level_zero = n; + return *this; + } private: From 582cc05ae79beabb3d839302e06eef76f5aa74c8 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 13 Jun 2023 13:38:52 +0300 Subject: [PATCH 14/28] Automagical hierarchical partitioning --- dccrg.hpp | 67 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index bf02485..b6578fe 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -7480,7 +7480,6 @@ template < unsigned int neighborhood_length = 1; std::string load_balancing_method{"RCB"}; - int hierarchical_level_zero {16}; // maximum value an MPI tag can have unsigned int max_tag; @@ -7636,6 +7635,7 @@ template < Zoltan_Struct* zoltan; // number of processes per part in a hierarchy level (numbering starts from 0) std::vector processes_per_part; + std::vector partition_number; // options for each level of hierarchial load balancing (numbering start from 0) std::vector> partitioning_options; // record whether Zoltan_LB_Partition is expected to fail @@ -7952,10 +7952,39 @@ template < // Hardcoded for now if (this->load_balancing_method == "HIER") { - add_partitioning_level(this->hierarchical_level_zero); // Level 0 - Nodes + // Automagical hierarchical partition + std::hash hasher; + + //get name of this node + char nodename[MPI_MAX_PROCESSOR_NAME]; + int namelength, nodehash; + MPI_Get_processor_name(nodename,&namelength); + nodehash = static_cast(hasher(std::string(nodename, namelength)) % std::numeric_limits::max()); + + //intra-node communicator + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm nodeComm; + MPI_Comm_split(MPI_COMM_WORLD, nodehash, rank, &nodeComm); + + //inter-node communicator + int nodeRank; + MPI_Comm_rank(nodeComm,&nodeRank); + MPI_Comm interComm; + MPI_Comm_split(MPI_COMM_WORLD, nodeRank, rank, &interComm); + int interRank; + MPI_Comm_rank(interComm, &interRank); + + // Make sure everyone in node agrees on the number of the node + int nodeNumber {interRank}; + MPI_Bcast(&nodeNumber, 1, MPI_INT, 0, nodeComm); + + add_partitioning_level(1); // Level 0 - Nodes + partition_number.push_back(nodeNumber); add_partitioning_option(0, "LB_METHOD", "HYPERGRAPH"); add_partitioning_level(1); // Level 1 - Processes + partition_number.push_back(nodeRank); add_partitioning_option(1, "LB_METHOD", "RIB"); //add_partitioning_option(1, "LB_METHOD", "RCB"); } @@ -8488,24 +8517,6 @@ template < return this->load_balancing_method; } - /*! - Sets amount of tasks for node-level of HIER partition for Zoltan - - Must be called before initialize(). - - \see balance_load() - */ - Dccrg< - Cell_Data, - Geometry, - std::tuple, - std::tuple - >& set_load_balancing_hier_level_zero(const int n) { - this->hierarchical_level_zero = n; - return *this; - } - - private: /*! Initializes local cells' neighbor lists and related data structures. @@ -11774,15 +11785,17 @@ template < *error = ZOLTAN_OK; } - int process = int(dccrg_instance->rank); - int part; + return dccrg_instance->partition_number[level]; - for (int i = 0; i <= level; i++) { - part = process / dccrg_instance->processes_per_part[i]; - process %= dccrg_instance->processes_per_part[i]; - } + // int process = int(dccrg_instance->rank); + // int part; + + // for (int i = 0; i <= level; i++) { + // part = process / dccrg_instance->processes_per_part[i]; + // process %= dccrg_instance->processes_per_part[i]; + // } - return part; + // return part; } From f0b1294f2c6550369fd7166cd2e605c241131dbf Mon Sep 17 00:00:00 2001 From: lkotipal Date: Thu, 8 Feb 2024 13:28:05 +0200 Subject: [PATCH 15/28] Use partitioning neighborhood directly for hypergraph --- dccrg.hpp | 120 +++++------------------------------------------------- 1 file changed, 10 insertions(+), 110 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index b6578fe..13ae334 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -2825,110 +2825,6 @@ template < return ret; } - std::set get_vlasov_neighbors( - const uint64_t cell, - const int neighborhood_id, - const int dimension, - const int stencil_width - ) const { - std::set ret; - const auto* p = get_neighbors_of(cell, neighborhood_id); - if (!p) { - std::cerr << "Cell " << cell << ", neighborhood " << neighborhood_id << ", dimension " << dimension << " not found" << std::endl; - return ret; - } - - // Create list of unique distances - std::set distances_plus; - std::set distances_minus; - std::set found_neighbors_plus; - std::set found_neighbors_minus; - /** Using sets of cells as well, we should only get one distance per - (potentially less refined) cell. This should result in safe behaviour - as long as the neighborhood of a cell does not contain cells with a - refinement level more than 1 level apart from the cell itself. - */ - for (const auto& [neighbor, coords] : *p) { - if(coords[dimension] > 0) { - if (!found_neighbors_plus.count(neighbor)) { - distances_plus.insert(coords[dimension]); - found_neighbors_plus.insert(neighbor); - } - } - if(coords[dimension] < 0) { - if (!found_neighbors_minus.count(neighbor)) { - distances_minus.insert(-coords[dimension]); - found_neighbors_minus.insert(neighbor); - } - } - } - - int iSrc = stencil_width - 1; - // Iterate through positive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - for (const auto& distance : distances_plus) { - if (iSrc < 0) - break; // found enough elements - for (const auto& [neighbor, coords] : *p) { - if (neighbor == error_cell) - continue; - if (coords[dimension] == distance) { - if (ret.count(neighbor)) - continue; - ret.insert(neighbor); - } - } // end loop over neighbors - iSrc--; - } // end loop over positive distances - - iSrc = stencil_width - 1; - // Iterate through negtive distances for VLASOV_STENCIL_WIDTH elements starting from the smallest distance. - for (const auto& distance : distances_minus) { - if (iSrc < 0) - break; // found enough elements - for (const auto& [neighbor, coords] : *p) { - if (neighbor == error_cell) - continue; - if (coords[dimension] == distance) { - if (ret.count(neighbor)) - continue; - ret.insert(neighbor); - } - } // end loop over neighbors - iSrc--; - } // end loop over negative distances - - return ret; - } - - // Get actual Vlasov stencil neighbors - // Assumes linear 3 linear stencils from neighborhood_id to neighborhood_id + 2 - std::set get_vlasov_neighbors( - const uint64_t cell - ) const { - int stencil_width {0}; - switch (neighborhood_length) { - case 1: - break; - case 3: - stencil_width = 2; - break; - case 5: - stencil_width = 3; - break; - default: - // Placeholder error - std::cerr << "Weird stencil width" << std::endl; - break; - } - - std::set ret; - for (int dim = 0; dim < 3; ++dim) { - auto neighs_dim = get_vlasov_neighbors(cell, partitioning_neighborhood_id + dim, dim, stencil_width); - ret.insert(neighs_dim.begin(), neighs_dim.end()); - } - return ret; - } - /*! Returns true if given cell's neighbor types match given criterion, false otherwise. @@ -11394,6 +11290,7 @@ template < /*! + Graph partitioning Writes the number of neighbors into number_of_neighbors for all cells given in global_ids. */ static void fill_number_of_neighbors_for_cells( @@ -11431,12 +11328,13 @@ template < return; } - number_of_neighbors[i] = dccrg_instance->get_vlasov_neighbors(cell).size(); + number_of_neighbors[i] = dccrg_instance->get_neighbors_to(cell, dccrg_instance->partitioning_neighborhood_id)->size(); } } /*! + Graph partitioning Writes neighbor lists of given cells into neighbors, etc. */ static void fill_neighbor_lists( @@ -11480,7 +11378,9 @@ template < number_of_neighbors[i] = 0; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(cell)) { + // We consider the communication weight from this cell to others + auto weight {dccrg_instance->get_communication_weight(cell)}; + for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, dccrg_instance->partitioning_neighborhood_id)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself if (neighbor == 0 || neighbor == cell) { continue; @@ -11494,7 +11394,7 @@ template < // weight of edge from cell to *neighbor if (number_of_weights_per_edge > 0) { - edge_weights[current_neighbor_number] = dccrg_instance->get_communication_weight(neighbor); + edge_weights[current_neighbor_number] = weight; } current_neighbor_number++; @@ -11547,7 +11447,7 @@ template < /*! - Writes the number of hyperedges (self + one per neighbor cell) in the grid for all cells on this process. + Writes the number of connections (self + one per neighbor cell) in the grid for all cells on this process. */ static void fill_number_of_hyperedges( void* data, @@ -11575,7 +11475,7 @@ template < *number_of_connections = 0; for (const auto& item: dccrg_instance->cell_data) { - (*number_of_connections) += 1 + dccrg_instance->get_vlasov_neighbors(item.first).size(); + (*number_of_connections) += 1 + dccrg_instance->get_neighbors_to(item.first, dccrg_instance->partitioning_neighborhood_id)->size(); } } @@ -11634,7 +11534,7 @@ template < // add a connection to the cell itself from its hyperedge connections[connection_number++] = item.first; - for (const auto& neighbor: dccrg_instance->get_vlasov_neighbors(item.first)) { + for (const auto& [neighbor, dir]: *dccrg_instance->get_neighbors_to(item.first, dccrg_instance->partitioning_neighborhood_id)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself if (neighbor == 0 || neighbor == item.first) { continue; From fbd74e3dd7e32764b79528963bee7d75fe22ee5a Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 12 Feb 2024 11:30:29 +0200 Subject: [PATCH 16/28] Set NUM_LID_ENTRIES so hypergraph works on current versions of Zoltan --- dccrg.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dccrg.hpp b/dccrg.hpp index 13ae334..a92817e 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -7901,7 +7901,7 @@ template < // 0 because Zoltan crashes in hierarchial with larger values Zoltan_Set_Param(this->zoltan, "EDGE_WEIGHT_DIM", "1"); Zoltan_Set_Param(this->zoltan, "NUM_GID_ENTRIES", "1"); - Zoltan_Set_Param(this->zoltan, "NUM_LID_ENTRIES", "0"); + Zoltan_Set_Param(this->zoltan, "NUM_LID_ENTRIES", "1"); Zoltan_Set_Param(this->zoltan, "OBJ_WEIGHT_DIM", "1"); Zoltan_Set_Param(this->zoltan, "RETURN_LISTS", "ALL"); From 1c8276718538f014f2f2b0f4c85ce557a1ca6e10 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 12 Feb 2024 12:50:08 +0200 Subject: [PATCH 17/28] Multi-neighborhood partitioning --- dccrg.hpp | 80 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 35 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index a92817e..7debeb5 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -819,8 +819,8 @@ template < return neighbors; } - void set_partitioning_neighborhood (int id) { - partitioning_neighborhood_id = id; + void add_partitioning_neighborhood (int id) { + partitioning_neighborhoods.push_back(id); } /*! @@ -7424,7 +7424,7 @@ template < std::vector::neighborhood_item_t> > user_hood_of, user_hood_to; - int partitioning_neighborhood_id{default_neighborhood_id}; + std::vector partitioning_neighborhoods {}; /*! Cell on this process and those cells that aren't neighbors of @@ -11328,7 +11328,9 @@ template < return; } - number_of_neighbors[i] = dccrg_instance->get_neighbors_to(cell, dccrg_instance->partitioning_neighborhood_id)->size(); + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { + number_of_neighbors[i] += dccrg_instance->get_neighbors_to(cell, neighborhood)->size(); + } } } @@ -11380,24 +11382,26 @@ template < // We consider the communication weight from this cell to others auto weight {dccrg_instance->get_communication_weight(cell)}; - for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, dccrg_instance->partitioning_neighborhood_id)) { - // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself - if (neighbor == 0 || neighbor == cell) { - continue; - } + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { + for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { + // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself + if (neighbor == 0 || neighbor == cell) { + continue; + } - number_of_neighbors[i]++; + number_of_neighbors[i]++; - neighbors[current_neighbor_number] = neighbor; - processes_of_neighbors[current_neighbor_number] - = int(dccrg_instance->cell_process.at(neighbor)); + neighbors[current_neighbor_number] = neighbor; + processes_of_neighbors[current_neighbor_number] + = int(dccrg_instance->cell_process.at(neighbor)); - // weight of edge from cell to *neighbor - if (number_of_weights_per_edge > 0) { - edge_weights[current_neighbor_number] = weight; - } + // weight of edge from cell to *neighbor + if (number_of_weights_per_edge > 0) { + edge_weights[current_neighbor_number] = weight; + } - current_neighbor_number++; + current_neighbor_number++; + } } } } @@ -11470,12 +11474,14 @@ template < >(data); *error = ZOLTAN_OK; - *number_of_hyperedges = int(dccrg_instance->cell_data.size()); + *number_of_hyperedges = int(dccrg_instance->partitioning_neighborhoods.size() * dccrg_instance->cell_data.size()); *format = ZOLTAN_COMPRESSED_EDGE; *number_of_connections = 0; for (const auto& item: dccrg_instance->cell_data) { - (*number_of_connections) += 1 + dccrg_instance->get_neighbors_to(item.first, dccrg_instance->partitioning_neighborhood_id)->size(); + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { + *number_of_connections += 1 + dccrg_instance->get_neighbors_to(item.first, neighborhood)->size(); + } } } @@ -11516,7 +11522,7 @@ template < return; } - if ((unsigned int) number_of_hyperedges != dccrg_instance->cell_data.size()) { + if ((unsigned int) number_of_hyperedges != dccrg_instance->partitioning_neighborhoods.size() * dccrg_instance->cell_data.size()) { std::cerr << "Zoltan is expecting wrong number of hyperedges: " << number_of_hyperedges << " instead of " << dccrg_instance->cell_data.size() << std::endl; @@ -11524,26 +11530,30 @@ template < return; } - int i = 0; int connection_number = 0; - for (const auto& item: dccrg_instance->cell_data) { - - hyperedges[i] = item.first; - hyperedge_connection_offsets[i] = connection_number; - - // add a connection to the cell itself from its hyperedge - connections[connection_number++] = item.first; + int j = 0; + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { + int i = 0; + for (const auto& item: dccrg_instance->cell_data) { + hyperedges[i + j * dccrg_instance->cell_data.size()] = item.first; + hyperedge_connection_offsets[i + j * dccrg_instance->cell_data.size()] = connection_number; + + // add a connection to the cell itself from its hyperedge + connections[connection_number++] = item.first; + + for (const auto& [neighbor, dir]: *dccrg_instance->get_neighbors_to(item.first, neighborhood)) { + // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself + if (neighbor == 0 || neighbor == item.first) { + continue; + } - for (const auto& [neighbor, dir]: *dccrg_instance->get_neighbors_to(item.first, dccrg_instance->partitioning_neighborhood_id)) { - // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself - if (neighbor == 0 || neighbor == item.first) { - continue; + connections[connection_number++] = neighbor; } - connections[connection_number++] = neighbor; + ++i; } - i++; + ++j; } if (connection_number != number_of_connections) { From 6ac81458ed0ac2e8f23cc37a26228e420773ec9a Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 12 Feb 2024 19:09:51 +0200 Subject: [PATCH 18/28] Attempt to fix test crashes --- dccrg.hpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 7debeb5..77de364 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11329,7 +11329,9 @@ template < } for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { - number_of_neighbors[i] += dccrg_instance->get_neighbors_to(cell, neighborhood)->size(); + for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { + number_of_neighbors[i] += neighbor > 0 && neighbor != cell; // We apparently have to do this stupid filtering here + } } } } @@ -11480,7 +11482,10 @@ template < *number_of_connections = 0; for (const auto& item: dccrg_instance->cell_data) { for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { - *number_of_connections += 1 + dccrg_instance->get_neighbors_to(item.first, neighborhood)->size(); + ++*number_of_connections; + for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(item.first, neighborhood)) { + *number_of_connections += neighbor > 0 && neighbor != item.first; // We apparently have to do this stupid filtering here + } } } } From b2647b7d0fe33c47c39e7daf6fcda97ee53eb8a5 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 13 Feb 2024 00:33:26 +0200 Subject: [PATCH 19/28] Fix uninitialized memory --- dccrg.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 77de364..1c00894 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11328,6 +11328,7 @@ template < return; } + number_of_neighbors[i] = 0; for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { number_of_neighbors[i] += neighbor > 0 && neighbor != cell; // We apparently have to do this stupid filtering here @@ -11371,6 +11372,7 @@ template < int current_neighbor_number = 0; for (int i = 0; i < number_of_cells; i++) { + int n {0}; uint64_t cell = uint64_t(global_ids[i]); if (dccrg_instance->cell_data.count(cell) == 0) { *error = ZOLTAN_FATAL; @@ -11380,7 +11382,7 @@ template < return; } - number_of_neighbors[i] = 0; + // number_of_neighbors[i] = 0; // We consider the communication weight from this cell to others auto weight {dccrg_instance->get_communication_weight(cell)}; @@ -11391,7 +11393,7 @@ template < continue; } - number_of_neighbors[i]++; + // number_of_neighbors[i]++; neighbors[current_neighbor_number] = neighbor; processes_of_neighbors[current_neighbor_number] @@ -11402,7 +11404,8 @@ template < edge_weights[current_neighbor_number] = weight; } - current_neighbor_number++; + ++current_neighbor_number; + ++n; } } } From 71bd2b864a3131d1bf6e816e0d0c7ca95b4b5378 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 26 Feb 2024 13:12:37 +0200 Subject: [PATCH 20/28] Per-cell partitioning neighborhoods --- dccrg.hpp | 74 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 1c00894..b8ed4e5 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -366,6 +366,7 @@ template < reserved_options(other.get_reserved_options()), cell_weights(other.get_cell_weights()), communication_weights(other.get_communication_weights()), + partitioning_neighborhoods(other.partitioning_neighborhoods), neighbor_processes(other.get_neighbor_processes()), balancing_load(other.get_balancing_load()) { @@ -819,8 +820,12 @@ template < return neighbors; } - void add_partitioning_neighborhood (int id) { - partitioning_neighborhoods.push_back(id); + void clear_partitioning_neighborhoods (uint64_t cell) { + partitioning_neighborhoods[cell].clear(); + } + + void add_partitioning_neighborhood (uint64_t cell, int neighborhood) { + partitioning_neighborhoods[cell].insert(neighborhood); } /*! @@ -3466,6 +3471,13 @@ template < this->communication_weights.erase(refined); } + if (this->rank == process_of_refined && this->partitioning_neighborhoods.count(refined) > 0) { + for (const uint64_t child: children) { + this->partitioning_neighborhoods[child] = this->partitioning_neighborhoods.at(refined); + } + this->partitioning_neighborhoods.erase(refined); + } + // use local neighbor lists to find cells whose neighbor lists have to updated if (this->rank == process_of_refined) { // update the neighbor lists of created local cells @@ -3626,6 +3638,7 @@ template < this->new_pin_requests.erase(unrefined); this->cell_weights.erase(unrefined); this->communication_weights.erase(unrefined); + this->partitioning_neighborhoods.erase(unrefined); // don't send unrefined cells' user data to self if (this->rank == process_of_unrefined @@ -7424,8 +7437,6 @@ template < std::vector::neighborhood_item_t> > user_hood_of, user_hood_to; - std::vector partitioning_neighborhoods {}; - /*! Cell on this process and those cells that aren't neighbors of this cell but whose neighbor this cell is. @@ -7546,6 +7557,9 @@ template < // optional user-given communication weights of cells on this process std::unordered_map communication_weights; + // Set of neighborhoods each cell communicates in (hyper)graph partitioning + std::unordered_map> partitioning_neighborhoods; + // processes which have cells close enough from cells of this process std::unordered_set neighbor_processes; @@ -11329,7 +11343,7 @@ template < } number_of_neighbors[i] = 0; - for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods[cell]) { for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { number_of_neighbors[i] += neighbor > 0 && neighbor != cell; // We apparently have to do this stupid filtering here } @@ -11386,7 +11400,7 @@ template < // We consider the communication weight from this cell to others auto weight {dccrg_instance->get_communication_weight(cell)}; - for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods[cell]) { for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself if (neighbor == 0 || neighbor == cell) { @@ -11479,18 +11493,21 @@ template < >(data); *error = ZOLTAN_OK; - *number_of_hyperedges = int(dccrg_instance->partitioning_neighborhoods.size() * dccrg_instance->cell_data.size()); *format = ZOLTAN_COMPRESSED_EDGE; + *number_of_hyperedges = 0; *number_of_connections = 0; - for (const auto& item: dccrg_instance->cell_data) { - for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { + for (const auto& [cell, data]: dccrg_instance->cell_data) { + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods[cell]) { + ++*number_of_hyperedges; ++*number_of_connections; - for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(item.first, neighborhood)) { - *number_of_connections += neighbor > 0 && neighbor != item.first; // We apparently have to do this stupid filtering here + for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { + *number_of_connections += neighbor > 0 && neighbor != cell; // We apparently have to do this stupid filtering here } } } + + // std::cerr << "I have " + std::to_string(dccrg_instance->cell_data.size()) + "cells, " + std::to_string(*number_of_hyperedges) + " hedges and " + std::to_string(*number_of_connections) + " connections!\n"; } @@ -11530,38 +11547,35 @@ template < return; } - if ((unsigned int) number_of_hyperedges != dccrg_instance->partitioning_neighborhoods.size() * dccrg_instance->cell_data.size()) { - std::cerr << "Zoltan is expecting wrong number of hyperedges: " << number_of_hyperedges - << " instead of " << dccrg_instance->cell_data.size() - << std::endl; - *error = ZOLTAN_FATAL; - return; - } - int connection_number = 0; - int j = 0; - for (auto neighborhood : dccrg_instance->partitioning_neighborhoods) { - int i = 0; - for (const auto& item: dccrg_instance->cell_data) { - hyperedges[i + j * dccrg_instance->cell_data.size()] = item.first; - hyperedge_connection_offsets[i + j * dccrg_instance->cell_data.size()] = connection_number; + int hedge_number = 0; + for (const auto& [cell, data]: dccrg_instance->cell_data) { + for (auto neighborhood : dccrg_instance->partitioning_neighborhoods[cell]) { + hyperedges[hedge_number] = cell; + hyperedge_connection_offsets[hedge_number] = connection_number; // add a connection to the cell itself from its hyperedge - connections[connection_number++] = item.first; + connections[connection_number++] = cell; - for (const auto& [neighbor, dir]: *dccrg_instance->get_neighbors_to(item.first, neighborhood)) { + for (const auto& [neighbor, dir]: *dccrg_instance->get_neighbors_to(cell, neighborhood)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself - if (neighbor == 0 || neighbor == item.first) { + if (neighbor == 0 || neighbor == cell) { continue; } connections[connection_number++] = neighbor; } - ++i; + ++hedge_number; } + } - ++j; + if (number_of_hyperedges != hedge_number) { + std::cerr << "Zoltan is expecting wrong number of hyperedges: " << number_of_hyperedges + << " instead of " << dccrg_instance->cell_data.size() + << std::endl; + *error = ZOLTAN_FATAL; + return; } if (connection_number != number_of_connections) { From 379d4546b824ae7c45b26eca9fb7d073bf732183 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Wed, 3 Apr 2024 16:15:01 +0300 Subject: [PATCH 21/28] Use cell weights for (hyper)graph --- dccrg.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index b8ed4e5..f51de37 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11399,7 +11399,7 @@ template < // number_of_neighbors[i] = 0; // We consider the communication weight from this cell to others - auto weight {dccrg_instance->get_communication_weight(cell)}; + auto weight {dccrg_instance->get_cell_weight(cell)}; for (auto neighborhood : dccrg_instance->partitioning_neighborhoods[cell]) { for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself @@ -11464,7 +11464,7 @@ template < return; } - sizes[i] = dccrg_instance->get_communication_weight(cell); + sizes[i] = (int) dccrg_instance->get_cell_weight(cell); } } @@ -11656,7 +11656,7 @@ template < for (const auto& item : dccrg_instance->cell_data) { hyperedges[i] = item.first; if (number_of_weights_per_hyperedge) { - hyperedge_weights[i] = dccrg_instance->get_communication_weight(item.first); + hyperedge_weights[i] = dccrg_instance->get_cell_weight(item.first); } ++i; } From 00e1010d9c6640f99ef7ab189f953f0edfbe587f Mon Sep 17 00:00:00 2001 From: lkotipal Date: Fri, 5 Apr 2024 16:12:05 +0300 Subject: [PATCH 22/28] Multi-weight partitioning, reduction via norms (WIP) --- dccrg.hpp | 111 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 85 insertions(+), 26 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index f51de37..d287856 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -41,6 +41,7 @@ dccrg::Dccrg for a starting point in the API. #include "iterator" #include "limits" #include "map" +#include #include "set" #include "stdexcept" #include "tuple" @@ -368,7 +369,8 @@ template < communication_weights(other.get_communication_weights()), partitioning_neighborhoods(other.partitioning_neighborhoods), neighbor_processes(other.get_neighbor_processes()), - balancing_load(other.get_balancing_load()) + balancing_load(other.get_balancing_load()), + load_balance_norm(other.load_balance_norm) { if (other.get_balancing_load()) { std::cerr << __FILE__ << ":" << __LINE__ @@ -5952,7 +5954,6 @@ template < - RETURN_LISTS - EDGE_WEIGHT_DIM - NUM_GID_ENTRIES - - OBJ_WEIGHT_DIM Call this with name = LB_METHOD and value = HIER to use hierarchial partitioning. @@ -5975,6 +5976,10 @@ template < ); } + if (name == "OBJ_WEIGHT_DIM") { + this->cell_weight_dim = std::stoi(value); + } + Zoltan_Set_Param(this->zoltan, name.c_str(), value.c_str()); return *this; } @@ -6488,7 +6493,7 @@ template < Returns true on success, does nothing and returns false otherwise. */ - bool set_cell_weight(const uint64_t cell, const double weight) + bool set_cell_weight(const uint64_t cell, const std::vector& weight) { if (this->cell_process.count(cell) == 0) { return false; @@ -6502,11 +6507,42 @@ template < return false; } + if (cell_weight_dim == 1 && weight.size() > 1) { + // TODO consider p-norms besides Manhattan and Euclidean + switch (load_balance_norm) { + case 0: + // Infinity norm i.e. maximum + this->cell_weights[cell] = std::vector{*std::max_element(weight.begin(), weight.end())}; + break; + case 1: + // 1-norm i.e. Manhattan (sum in this case) + this->cell_weights[cell] = std::vector{std::reduce(weight.begin(), weight.end())}; + break; + case 2: + // 2-norm i.e. Euclidean + this->cell_weights[cell] = std::vector{std::sqrt(std::inner_product(weight.begin(), weight.end(), weight.begin(), 0.0))}; + break; + default: + throw std::invalid_argument("Invalid load balance norm!"); + } + + } else if (cell_weight_dim != weight.size()) { + throw std::invalid_argument("Invalid cell weight dimension!"); + } + this->cell_weights[cell] = weight; return true; } + bool set_cell_weight(const uint64_t cell, const double weight) { + return set_cell_weight(cell, std::vector({weight})); + } + + void set_load_balance_norm(int norm) { + this->load_balance_norm = norm; + } + bool set_communication_weight(const uint64_t cell, const double weight) { if (this->cell_process.count(cell) == 0) { @@ -6532,21 +6568,21 @@ template < Returns a quiet nan if above conditions are not met. Unset cell weights are assumed to be 1. */ - double get_cell_weight(const uint64_t cell) const + std::vector get_cell_weight(const uint64_t cell) const { if (this->cell_process.count(cell) == 0) { - return std::numeric_limits::quiet_NaN(); + return std::vector(cell_weight_dim, std::numeric_limits::quiet_NaN()); } if (this->cell_process.at(cell) != this->rank) { - return std::numeric_limits::quiet_NaN(); + return std::vector(cell_weight_dim, std::numeric_limits::quiet_NaN()); } if (cell != this->get_child(cell)) { - return std::numeric_limits::quiet_NaN(); + return std::vector(cell_weight_dim, std::numeric_limits::quiet_NaN()); } - return this->cell_weights.count(cell) ? this->cell_weights.at(cell) : 1.0; + return this->cell_weights.count(cell) ? this->cell_weights.at(cell) : std::vector(cell_weight_dim, 1.0); } /*! @@ -6700,7 +6736,7 @@ template < /*! Returns cell weights that have been set. */ - const std::unordered_map& get_cell_weights() const + const std::unordered_map>& get_cell_weights() const { return this->cell_weights; } @@ -7552,7 +7588,9 @@ template < std::unordered_set reserved_options; // optional user-given processing weights of cells on this process - std::unordered_map cell_weights; + int cell_weight_dim {1}; + int load_balance_norm {1}; // Sum + std::unordered_map> cell_weights; // optional user-given communication weights of cells on this process std::unordered_map communication_weights; @@ -7903,7 +7941,6 @@ template < this->reserved_options.insert("EDGE_WEIGHT_DIM"); this->reserved_options.insert("NUM_GID_ENTRIES"); this->reserved_options.insert("NUM_LID_ENTRIES"); - this->reserved_options.insert("OBJ_WEIGHT_DIM"); this->reserved_options.insert("RETURN_LISTS"); this->reserved_options.insert("NUM_GLOBAL_PARTS"); this->reserved_options.insert("NUM_LOCAL_PARTS"); @@ -7916,7 +7953,6 @@ template < Zoltan_Set_Param(this->zoltan, "EDGE_WEIGHT_DIM", "1"); Zoltan_Set_Param(this->zoltan, "NUM_GID_ENTRIES", "1"); Zoltan_Set_Param(this->zoltan, "NUM_LID_ENTRIES", "1"); - Zoltan_Set_Param(this->zoltan, "OBJ_WEIGHT_DIM", "1"); Zoltan_Set_Param(this->zoltan, "RETURN_LISTS", "ALL"); // set other options @@ -11283,6 +11319,7 @@ template < *error = ZOLTAN_OK; int i = 0; + int j = 0; for (const auto& item: dccrg_instance->cell_data) { #ifdef DEBUG @@ -11294,8 +11331,12 @@ template < global_ids[i] = item.first; - if (number_of_weights_per_object > 0) { - object_weights[i] = dccrg_instance->get_cell_weight(item.first); + if (number_of_weights_per_object != dccrg_instance->get_cell_weight(item.first).size()) { + std::cerr << "ERROR expected " << number_of_weights_per_object << " weights, got " << dccrg_instance->get_cell_weight(item.first).size() << std::endl; + abort(); + } + for (auto w : dccrg_instance->get_cell_weight(item.first)) { + object_weights[j++] = w; } i++; @@ -11384,9 +11425,19 @@ template < >(data); *error = ZOLTAN_OK; + // TODO: this could be a different parameter, possibly later + if ((unsigned int) number_of_weights_per_edge != dccrg_instance->cell_weight_dim) { + std::cerr + << "Zoltan is expecting wrong number of weights per edge: " << number_of_weights_per_edge + << " instead of " << dccrg_instance->cell_weight_dim + << std::endl; + *error = ZOLTAN_FATAL; + return; + } + int current_neighbor_number = 0; + int current_wgt_number = 0; for (int i = 0; i < number_of_cells; i++) { - int n {0}; uint64_t cell = uint64_t(global_ids[i]); if (dccrg_instance->cell_data.count(cell) == 0) { *error = ZOLTAN_FATAL; @@ -11399,7 +11450,7 @@ template < // number_of_neighbors[i] = 0; // We consider the communication weight from this cell to others - auto weight {dccrg_instance->get_cell_weight(cell)}; + auto weights {dccrg_instance->get_cell_weight(cell)}; for (auto neighborhood : dccrg_instance->partitioning_neighborhoods[cell]) { for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself @@ -11410,16 +11461,14 @@ template < // number_of_neighbors[i]++; neighbors[current_neighbor_number] = neighbor; - processes_of_neighbors[current_neighbor_number] - = int(dccrg_instance->cell_process.at(neighbor)); + processes_of_neighbors[current_neighbor_number] = int(dccrg_instance->cell_process.at(neighbor)); // weight of edge from cell to *neighbor - if (number_of_weights_per_edge > 0) { - edge_weights[current_neighbor_number] = weight; + for (auto w : weights) { + edge_weights[current_wgt_number++] = w; } ++current_neighbor_number; - ++n; } } } @@ -11464,7 +11513,7 @@ template < return; } - sizes[i] = (int) dccrg_instance->get_cell_weight(cell); + sizes[i] = (int) dccrg_instance->get_communication_weight(cell); } } @@ -11652,13 +11701,23 @@ template < return; } + // TODO: this coudl be a different parameter, possibly later + if ((unsigned int) number_of_weights_per_hyperedge != dccrg_instance->cell_weight_dim) { + std::cerr + << "Zoltan is expecting wrong number of weights per hyperedge: " << number_of_weights_per_hyperedge + << " instead of " << dccrg_instance->cell_weight_dim + << std::endl; + *error = ZOLTAN_FATAL; + return; + } + int i = 0; + int j = 0; for (const auto& item : dccrg_instance->cell_data) { - hyperedges[i] = item.first; - if (number_of_weights_per_hyperedge) { - hyperedge_weights[i] = dccrg_instance->get_cell_weight(item.first); + hyperedges[i++] = item.first; + for (auto w : dccrg_instance->get_cell_weight(item.first)) { + hyperedge_weights[j++] = w; } - ++i; } } From 99369dc70b57c66260b7815d6a06fb4b46e3c7f4 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Mon, 8 Apr 2024 15:24:11 +0300 Subject: [PATCH 23/28] Arbitrary LB norm --- dccrg.hpp | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index d287856..9be7e92 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -6508,24 +6508,19 @@ template < } if (cell_weight_dim == 1 && weight.size() > 1) { - // TODO consider p-norms besides Manhattan and Euclidean - switch (load_balance_norm) { - case 0: + if (load_balance_norm == 0) { // Infinity norm i.e. maximum this->cell_weights[cell] = std::vector{*std::max_element(weight.begin(), weight.end())}; - break; - case 1: - // 1-norm i.e. Manhattan (sum in this case) - this->cell_weights[cell] = std::vector{std::reduce(weight.begin(), weight.end())}; - break; - case 2: - // 2-norm i.e. Euclidean - this->cell_weights[cell] = std::vector{std::sqrt(std::inner_product(weight.begin(), weight.end(), weight.begin(), 0.0))}; - break; - default: + } else if (load_balance_norm > 0) { + // p-norm, including Manhattan (sum!) and Euclidian + double sum {0.0}; + for (auto w : weight) { + sum += std::pow(w, load_balance_norm); + } + this->cell_weights[cell] = std::vector{pow(sum, 1.0 / load_balance_norm)}; + } else { throw std::invalid_argument("Invalid load balance norm!"); } - } else if (cell_weight_dim != weight.size()) { throw std::invalid_argument("Invalid cell weight dimension!"); } From e5f9f2f272e32e9a1d714ab79a0a561cff066eba Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 9 Apr 2024 11:15:43 +0300 Subject: [PATCH 24/28] Default to graph in hierarchical partitioning --- dccrg.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index f51de37..62e3765 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -7891,12 +7891,11 @@ template < add_partitioning_level(1); // Level 0 - Nodes partition_number.push_back(nodeNumber); - add_partitioning_option(0, "LB_METHOD", "HYPERGRAPH"); + add_partitioning_option(0, "LB_METHOD", "GRAPH"); add_partitioning_level(1); // Level 1 - Processes partition_number.push_back(nodeRank); add_partitioning_option(1, "LB_METHOD", "RIB"); - //add_partitioning_option(1, "LB_METHOD", "RCB"); } // reserved options that the user cannot change From f9ca438853017a0e41aacd02f1ac51b3202348dd Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 9 Apr 2024 11:15:45 +0300 Subject: [PATCH 25/28] Revert "Use cell weights for (hyper)graph" This reverts commit 379d4546b824ae7c45b26eca9fb7d073bf732183. --- dccrg.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index 62e3765..fea3180 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -11398,7 +11398,7 @@ template < // number_of_neighbors[i] = 0; // We consider the communication weight from this cell to others - auto weight {dccrg_instance->get_cell_weight(cell)}; + auto weight {dccrg_instance->get_communication_weight(cell)}; for (auto neighborhood : dccrg_instance->partitioning_neighborhoods[cell]) { for (const auto& [neighbor, dir] : *dccrg_instance->get_neighbors_to(cell, neighborhood)) { // Zoltan 3.501 crashes in hierarchial if a cell is a neighbor to itself @@ -11463,7 +11463,7 @@ template < return; } - sizes[i] = (int) dccrg_instance->get_cell_weight(cell); + sizes[i] = dccrg_instance->get_communication_weight(cell); } } @@ -11655,7 +11655,7 @@ template < for (const auto& item : dccrg_instance->cell_data) { hyperedges[i] = item.first; if (number_of_weights_per_hyperedge) { - hyperedge_weights[i] = dccrg_instance->get_cell_weight(item.first); + hyperedge_weights[i] = dccrg_instance->get_communication_weight(item.first); } ++i; } From 4e9a3124109e02099cee2cb16945eff9bfe055d3 Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 16 Apr 2024 15:16:58 +0300 Subject: [PATCH 26/28] Fix control flow --- dccrg.hpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dccrg.hpp b/dccrg.hpp index aa04127..137b906 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -6523,10 +6523,10 @@ template < } } else if (cell_weight_dim != weight.size()) { throw std::invalid_argument("Invalid cell weight dimension!"); + } else { + this->cell_weights[cell] = weight; } - this->cell_weights[cell] = weight; - return true; } @@ -11312,6 +11312,11 @@ template < >(data); *error = ZOLTAN_OK; + if (number_of_weights_per_object != dccrg_instance->cell_weight_dim) { + std::cerr << "ERROR Zoltan expected " << number_of_weights_per_object << " weights, dccrg has " << dccrg_instance->cell_weight_dim << std::endl; + abort(); + } + int i = 0; int j = 0; for (const auto& item: dccrg_instance->cell_data) { @@ -11326,9 +11331,10 @@ template < global_ids[i] = item.first; if (number_of_weights_per_object != dccrg_instance->get_cell_weight(item.first).size()) { - std::cerr << "ERROR expected " << number_of_weights_per_object << " weights, got " << dccrg_instance->get_cell_weight(item.first).size() << std::endl; + std::cerr << "ERROR Zoltan expected " << number_of_weights_per_object << " weights, cell has " << dccrg_instance->get_cell_weight(item.first).size() << std::endl; abort(); } + for (auto w : dccrg_instance->get_cell_weight(item.first)) { object_weights[j++] = w; } From 8132de2836f25894d2c8956474f909bc8ade5bda Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 26 Nov 2024 14:53:12 +0200 Subject: [PATCH 27/28] Warning fix --- dccrg.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dccrg.hpp b/dccrg.hpp index b554ec4..b4018fb 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -10798,7 +10798,7 @@ template < global_ids[i] = item.first; - if (number_of_weights_per_object != dccrg_instance->get_cell_weight(item.first).size()) { + if (static_cast(number_of_weights_per_object) != dccrg_instance->get_cell_weight(item.first).size()) { std::cerr << "ERROR Zoltan expected " << number_of_weights_per_object << " weights, cell has " << dccrg_instance->get_cell_weight(item.first).size() << std::endl; abort(); } From 5462b3b5825022eb481ae8d88ddab81f6fc04a6b Mon Sep 17 00:00:00 2001 From: lkotipal Date: Tue, 26 Nov 2024 14:54:45 +0200 Subject: [PATCH 28/28] Another warning --- dccrg.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dccrg.hpp b/dccrg.hpp index b4018fb..f71c26a 100644 --- a/dccrg.hpp +++ b/dccrg.hpp @@ -6181,7 +6181,7 @@ template < } else { throw std::invalid_argument("Invalid load balance norm!"); } - } else if (cell_weight_dim != weight.size()) { + } else if (static_cast(cell_weight_dim) != weight.size()) { throw std::invalid_argument("Invalid cell weight dimension!"); } else { this->cell_weights[cell] = weight;