fix definition of init_mpi_data_structures (#1432)

trixi-framework · May 9, 2023 · a5f1a9c · a5f1a9c
1 parent 3b78369
commit a5f1a9c
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 41 deletions.
diff --git a/src/solvers/dgsem_p4est/dg_parallel.jl b/src/solvers/dgsem_p4est/dg_parallel.jl
@@ -444,28 +444,6 @@ function init_neighbor_rank_connectivity_iter_face_inner(info, user_data)
   return nothing
 end
 
-# Once the TreeMesh supports custom `uEltype`s, the following method could be used for both
-# mesh types.
-# See https://github.com/trixi-framework/Trixi.jl/pull/977#discussion_r793694635 for further
-# discussion.
-function init_mpi_data_structures(mpi_neighbor_interfaces, mpi_neighbor_mortars, n_dims, nvars, n_nodes, uEltype)
-  data_size = nvars * n_nodes^(n_dims - 1)
-  n_small_elements = 2^(n_dims-1)
-  mpi_send_buffers = Vector{Vector{uEltype}}(undef, length(mpi_neighbor_interfaces))
-  mpi_recv_buffers = Vector{Vector{uEltype}}(undef, length(mpi_neighbor_interfaces))
-  for index in 1:length(mpi_neighbor_interfaces)
-    mpi_send_buffers[index] = Vector{uEltype}(undef, length(mpi_neighbor_interfaces[index]) * data_size +
-                                                     length(mpi_neighbor_mortars[index]) * n_small_elements * 2 * data_size)
-    mpi_recv_buffers[index] = Vector{uEltype}(undef, length(mpi_neighbor_interfaces[index]) * data_size +
-                                                     length(mpi_neighbor_mortars[index]) * n_small_elements * 2 * data_size)
-  end
-
-  mpi_send_requests = Vector{MPI.Request}(undef, length(mpi_neighbor_interfaces))
-  mpi_recv_requests = Vector{MPI.Request}(undef, length(mpi_neighbor_interfaces))
-
-  return mpi_send_buffers, mpi_recv_buffers, mpi_send_requests, mpi_recv_requests
-end
-
 
 # Exchange normal directions of small elements of the MPI mortars. They are needed on all involved
 # MPI ranks to calculate the mortar fluxes.
@@ -550,4 +528,5 @@ end
 include("dg_2d_parallel.jl")
 include("dg_3d_parallel.jl")
 
+
 end # muladd
diff --git a/src/solvers/dgsem_tree/dg.jl b/src/solvers/dgsem_tree/dg.jl
@@ -54,6 +54,9 @@ include("indicators_3d.jl")
 # Container data structures
 include("containers.jl")
 
+# Dimension-agnostic parallel setup
+include("dg_parallel.jl")
+
 # 1D DG implementation
 include("dg_1d.jl")
 include("dg_1d_parabolic.jl")

diff --git a/src/solvers/dgsem_tree/dg_2d_parallel.jl b/src/solvers/dgsem_tree/dg_2d_parallel.jl
@@ -426,25 +426,6 @@ function init_mpi_neighbor_connectivity(elements, mpi_interfaces, mpi_mortars, m
 end
 
 
-# TODO: MPI dimension agnostic
-# Initialize MPI data structures
-function init_mpi_data_structures(mpi_neighbor_interfaces, mpi_neighbor_mortars, ndims, nvars, n_nodes, uEltype)
-  data_size = nvars * n_nodes^(ndims - 1)
-  mpi_send_buffers = Vector{Vector{uEltype}}(undef, length(mpi_neighbor_interfaces))
-  mpi_recv_buffers = Vector{Vector{uEltype}}(undef, length(mpi_neighbor_interfaces))
-  for index in 1:length(mpi_neighbor_interfaces)
-    mpi_send_buffers[index] = Vector{uEltype}(undef, length(mpi_neighbor_interfaces[index]) * data_size +
-                                                     length(mpi_neighbor_mortars[index]) * 4 * data_size)
-    mpi_recv_buffers[index] = Vector{uEltype}(undef, length(mpi_neighbor_interfaces[index]) * data_size +
-                                                     length(mpi_neighbor_mortars[index]) * 4 * data_size)
-  end
-
-  mpi_send_requests = Vector{MPI.Request}(undef, length(mpi_neighbor_interfaces))
-  mpi_recv_requests = Vector{MPI.Request}(undef, length(mpi_neighbor_interfaces))
-
-  return mpi_send_buffers, mpi_recv_buffers, mpi_send_requests, mpi_recv_requests
-end
-
 
 function rhs!(du, u, t,
               mesh::Union{ParallelTreeMesh{2}, ParallelP4estMesh{2}}, equations,

diff --git a/src/solvers/dgsem_tree/dg_parallel.jl b/src/solvers/dgsem_tree/dg_parallel.jl
@@ -0,0 +1,29 @@
+# By default, Julia/LLVM does not use fused multiply-add operations (FMAs).
+# Since these FMAs can increase the performance of many numerical algorithms,
+# we need to opt-in explicitly.
+# See https://ranocha.de/blog/Optimizing_EC_Trixi for further details.
+@muladd begin
+
+
+# Initialize MPI data structures. This works for both the
+# `TreeMesh` and the `P4estMesh` and is dimension-agnostic.
+function init_mpi_data_structures(mpi_neighbor_interfaces, mpi_neighbor_mortars, n_dims, nvars, n_nodes, uEltype)
+  data_size = nvars * n_nodes^(n_dims - 1)
+  n_small_elements = 2^(n_dims-1)
+  mpi_send_buffers = Vector{Vector{uEltype}}(undef, length(mpi_neighbor_interfaces))
+  mpi_recv_buffers = Vector{Vector{uEltype}}(undef, length(mpi_neighbor_interfaces))
+  for index in 1:length(mpi_neighbor_interfaces)
+    mpi_send_buffers[index] = Vector{uEltype}(undef, length(mpi_neighbor_interfaces[index]) * data_size +
+                                                     length(mpi_neighbor_mortars[index]) * n_small_elements * 2 * data_size)
+    mpi_recv_buffers[index] = Vector{uEltype}(undef, length(mpi_neighbor_interfaces[index]) * data_size +
+                                                     length(mpi_neighbor_mortars[index]) * n_small_elements * 2 * data_size)
+  end
+
+  mpi_send_requests = Vector{MPI.Request}(undef, length(mpi_neighbor_interfaces))
+  mpi_recv_requests = Vector{MPI.Request}(undef, length(mpi_neighbor_interfaces))
+
+  return mpi_send_buffers, mpi_recv_buffers, mpi_send_requests, mpi_recv_requests
+end
+
+
+end # muladd