From 338ddb1acf96fc5130ff08d705bf9bdb560f2c2e Mon Sep 17 00:00:00 2001
From: "ben.copley" <bcopley@vld305.cmpd1.metoffice.gov.uk>
Date: Fri, 9 Sep 2022 11:29:58 +0100
Subject: [PATCH] MPI support, one output file per rank (#52)

---
 CMakeLists.txt         |  6 +++++-
 src/c++/CMakeLists.txt |  3 ++-
 src/c++/profiler.cpp   | 20 ++++++++++++++------
 src/c++/profiler.h     |  1 +
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ea48fdf9..c7b3dc7f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -17,7 +17,11 @@ include(cmake/Doxygen.cmake)
 enable_doxygen()
 
 # OpenMP required
-find_package(OpenMP 2.5 REQUIRED) 
+find_package(OpenMP 2.5 REQUIRED)
+
+# MPI required
+find_package(MPI REQUIRED)
+include_directories(SYSTEM ${MPI_INCLUDE_PATH})
 
 # Defines some standard install directories such as $(prefix)/include.
 include(GNUInstallDirs)
diff --git a/src/c++/CMakeLists.txt b/src/c++/CMakeLists.txt
index 18e05ac1..4324398b 100644
--- a/src/c++/CMakeLists.txt
+++ b/src/c++/CMakeLists.txt
@@ -13,7 +13,8 @@ add_library(${CMAKE_PROJECT_NAME} SHARED
 
 # Link library to and external libs (also use project warnings and options).
 target_link_libraries(${CMAKE_PROJECT_NAME} PRIVATE
-        OpenMP::OpenMP_CXX)
+        OpenMP::OpenMP_CXX
+        MPI::MPI_CXX)
 
 set_project_warnings(${CMAKE_PROJECT_NAME})
 
diff --git a/src/c++/profiler.cpp b/src/c++/profiler.cpp
index 6129e677..23851429 100644
--- a/src/c++/profiler.cpp
+++ b/src/c++/profiler.cpp
@@ -121,17 +121,25 @@ void Profiler::stop(size_t const hash)
 
 void Profiler::write()
 {
+  // Find current MPI rank
+  int current_rank;
+  MPI_Comm comm = MPI_COMM_WORLD;
+  MPI_Comm_rank(comm, &current_rank);
+
   // Pickup environment variable filename if it exists, if not use the default
-  // name of "profiler-output.txt"
-  const char* filename = getenv("ProfOut");
-  if (filename != NULL)
+  // name of "profiler-output.txt". In either case, include the MPI rank in the
+  // name of the file.
+  const char* env_variable = std::getenv("PROFILER_OUTFILE");
+  if (env_variable != NULL)
   {
-    output_stream.open(filename);
+    const char* user_filename = (env_variable + ("-" + std::to_string(current_rank))).c_str();
+    output_stream.open(user_filename);
   }
   else
   {
-    output_stream.open("profiler-output.txt");
-    delete filename;
+    delete env_variable;
+    std::string default_filename = "profiler-output-" + std::to_string(current_rank);
+    output_stream.open(default_filename);
   }
 
   // Write to file
diff --git a/src/c++/profiler.h b/src/c++/profiler.h
index 813bf360..24b0d228 100644
--- a/src/c++/profiler.h
+++ b/src/c++/profiler.h
@@ -22,6 +22,7 @@
 #include <vector>
 
 #include "omp.h"
+#include "mpi.h"
 
 #include "hashtable.h"