CMake: Integrate librepa and kdpart

Co-authored-by: Jean-Noël Grad <[email protected]>

.gitlab-ci.yml

@@ -106,6 +106,7 @@ maxset:
      with_coverage: 'true'
      with_scafacos: 'true'
      with_stokesian_dynamics: 'true'
+     with_load_balancing: 'true'
      check_skip_long: 'true'
      cmake_params: '-DTEST_NP=8'
   script:

CMakeLists.txt

@@ -115,6 +115,7 @@ option(WARNINGS_ARE_ERRORS "Treat warnings as errors during compilation" OFF)
 option(WITH_CCACHE "Use ccache compiler invocation." OFF)
 option(WITH_PROFILER "Enable profiler annotations." OFF)
 option(INSIDE_DOCKER "Enable when running inside Docker." OFF)
+option(WITH_LOAD_BALANCING "Build with load balancing." OFF)
 set(TEST_TIMEOUT "300" CACHE STRING
                              "Timeout in seconds for each testsuite test")
 
@@ -138,6 +139,10 @@ set(MYCONFIG_NAME "myconfig.hpp" CACHE STRING
 # Check which config file to use
 include(MyConfig)
 
+if(WITH_LOAD_BALANCING)
+  set(CMAKE_CXX_STANDARD 17)
+endif()
+
 #
 # Pretty function
 #
@@ -207,6 +212,9 @@ set(ESPRESSO_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}")
 # folder for C++ and CUDA shared objects
 set(ESPRESSO_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 
+# folder for header files
+set(ESPRESSO_INSTALL_INCDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}")
+
 # python site-packages, can be overriden with CMake options
 if(WITH_PYTHON)
   if(NOT ESPRESSO_INSTALL_PYTHON)
@@ -302,6 +310,57 @@ if(WITH_STOKESIAN_DYNAMICS)
   endif()
 endif(WITH_STOKESIAN_DYNAMICS)
 
+# librepa and kdpart
+if(WITH_LOAD_BALANCING)
+  include(FetchContent)
+
+  # workaround for CMake FetchContent_Declare() PATCH_COMMAND bug, see details
+  # in https://gitlab.kitware.com/cmake/cmake/-/issues/21146
+  FetchContent_Declare(
+    kdpart
+    GIT_REPOSITORY https://github.com/hirschsn/kdpart.git
+    GIT_TAG afec3688008e0519410fdb58a3a88230e31ebd5e # v1.2.1
+    PATCH_COMMAND
+      patch -p1 -sN --ignore-whitespace -i
+      ${PROJECT_SOURCE_DIR}/cmake/kdpart.patch || echo "Patch not applied!")
+  FetchContent_GetProperties(kdpart)
+  if(NOT kdpart_POPULATED)
+    FetchContent_Populate(kdpart)
+
+    # build kdpart
+    execute_process(COMMAND ${CMAKE_COMMAND} -E env CXX=${CMAKE_CXX_COMPILER}
+                            make -C ${kdpart_SOURCE_DIR})
+    # install kdpart
+    execute_process(COMMAND make -C ${kdpart_SOURCE_DIR}
+                            PREFIX=${kdpart_BINARY_DIR} install)
+  endif()
+
+  # workaround for CMake FetchContent_Declare() PATCH_COMMAND bug, see details
+  # in https://gitlab.kitware.com/cmake/cmake/-/issues/21146
+  FetchContent_Declare(
+    librepa
+    GIT_REPOSITORY https://github.com/SC-SGS/repa.git
+    GIT_TAG ad4c30cd6451aa60b13060d80ddb55343716d8b5 # v1.1.3 with clang 10
+                                                     # warning fix
+    # patch to remove a boost version check which does not affect us (see #4429)
+    PATCH_COMMAND
+      patch -p1 -sN --ignore-whitespace -i
+      ${PROJECT_SOURCE_DIR}/cmake/librepa.patch || echo "Patch not applied!")
+  FetchContent_GetProperties(librepa)
+  if(NOT librepa_POPULATED)
+    FetchContent_Populate(librepa)
+    set(WITH_KDPART on CACHE BOOL "")
+    set(WITH_P4EST off CACHE BOOL "")
+    set(WITH_PARMETIS off CACHE BOOL "")
+    set(LIBREPA_WITH_TESTS off CACHE BOOL "")
+    set(LIBREPA_WITH_COVERAGE off CACHE BOOL "")
+    set(KDPART_DIR "${kdpart_BINARY_DIR}" CACHE PATH "")
+    add_subdirectory(${librepa_SOURCE_DIR} ${librepa_BINARY_DIR})
+  endif()
+
+  set(LOAD_BALANCING 1)
+endif(WITH_LOAD_BALANCING)
+
 if(WITH_VALGRIND_INSTRUMENTATION)
   find_package(PkgConfig REQUIRED)
   pkg_check_modules(VALGRIND valgrind REQUIRED)

cmake/cmake_config.cmakein

@@ -19,6 +19,8 @@
 
 #cmakedefine STOKESIAN_DYNAMICS
 
+#cmakedefine LOAD_BALANCING
+
 #cmakedefine VALGRIND_INSTRUMENTATION
 
 #define PACKAGE_NAME "${PROJECT_NAME}"

cmake/kdpart.patch

@@ -0,0 +1,21 @@
+--- a/kdpart.h
++++ b/kdpart.h
+@@ -140,7 +140,7 @@ struct NodeAccess {
+             std::abort();
+         }
+
+-        assert(false);
++        throw std::logic_error("[libkdpart] Fatal exception in local repartitioning.");
+     }
+
+     inline NodeAccess find_root_of_subtree(int depth) const
+--- a/Makefile
++++ b/Makefile
+@@ -5,5 +5,6 @@
+ CXX = mpic++
+ CXXFLAGS = -std=c++14 -O3 -march=native
++# flag -Wno-cast-function-type needed for GCC, see https://github.com/open-mpi/ompi/issues/5157
+-CXXFLAGS += -Wall -Wextra -pedantic
++CXXFLAGS += -Wall -Wextra -pedantic -Wno-cast-function-type
+
+ AR = ar

cmake/librepa.patch

@@ -0,0 +1,105 @@
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -30,8 +30,8 @@ option(WITH_P4EST "Build with SFC LB support" ON)
+ option(WITH_PARMETIS "Build with graph partitioning LB support" ON)
+ option(WITH_KDPART "Build with kd grid support" ON)
+
+-option(WITH_TESTS "Perform unit tests" ON)
+-option(WITH_COVERAGE "Coverage for tests" OFF)
++option(LIBREPA_WITH_TESTS "Perform unit tests" ON)
++option(LIBREPA_WITH_COVERAGE "Coverage for tests" OFF)
+
+ if(WITH_KDPART)
+   find_package(KDPart REQUIRED)
+@@ -49,15 +49,11 @@ find_package(MPI REQUIRED)
+ include_directories(SYSTEM ${MPI_CXX_INCLUDE_PATH})
+
+ list(APPEND REQ_BOOST_PACKAGES mpi serialization)
+-if(WITH_TESTS)
++if(LIBREPA_WITH_TESTS)
+   list(APPEND REQ_BOOST_PACKAGES unit_test_framework)
+-endif(WITH_TESTS)
++endif(LIBREPA_WITH_TESTS)
+ find_package(Boost "1.67.0" REQUIRED ${REQ_BOOST_PACKAGES})
+
+-if (Boost_VERSION VERSION_GREATER_EQUAL "1.69" AND Boost_VERSION VERSION_LESS_EQUAL "1.71")
+-  message(FATAL_ERROR "Boost 1.69 - 1.71 have a bug and are not supported.")
+-endif()
+-
+ include_directories(${Boost_INCLUDE_DIRS})
+
+ # Used for librepa itself as well as all tests
+@@ -77,7 +73,7 @@ set(REPA_DEFAULT_COMPILE_OPTIONS
+       "-Wno-conversion"
+       "-Wno-sign-conversion" >)
+
+-if(WITH_TESTS AND WITH_COVERAGE)
++if(LIBREPA_WITH_TESTS AND LIBREPA_WITH_COVERAGE)
+   if (NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
+     message(SEND_ERROR "COVERAGE is only possible with Gnu Compiler Collection.")
+   else()
+@@ -91,21 +87,21 @@ if(WITH_TESTS AND WITH_COVERAGE)
+       set(CMAKE_EXE_LINKER_FLAGS "--coverage -fprofile-arcs -ftest-coverage")
+
+       add_custom_target(coverage
+-                        COMMAND ${CMAKE_SOURCE_DIR}/util/coverage.sh
+-                        WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
++                        COMMAND ${PROJECT_SOURCE_DIR}/util/coverage.sh
++                        WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+       message(STATUS "Coverage enabled. Use `coverage' target after build to execute coverage tests.")
+     endif()
+   endif()
+-endif(WITH_TESTS AND WITH_COVERAGE)
++endif(LIBREPA_WITH_TESTS AND LIBREPA_WITH_COVERAGE)
+
+ add_subdirectory(repa)
+
+-if(WITH_TESTS)
++if(LIBREPA_WITH_TESTS)
+   set(TEST_MAX_NPROC 4
+       CACHE STRING "Maximum number of processes to run the tests with")
+   enable_testing()
+   add_subdirectory(tests)
+-endif(WITH_TESTS)
++endif(LIBREPA_WITH_TESTS)
+
+ add_subdirectory(examples)
+
+--- a/repa/CMakeLists.txt
++++ b/repa/CMakeLists.txt
+@@ -95,11 +95,5 @@ target_compile_options(repa
+                        PRIVATE ${REPA_DEFAULT_COMPILE_OPTIONS})
+
+ install(TARGETS repa
+-        LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
+-        ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+-# Keep folder structure, so no install for all repa_HDR
+-foreach(hdr ${repa_HDR})
+-    get_filename_component(dir ${hdr} DIRECTORY)
+-    install(FILES ${hdr}
+-            DESTINATION ${CMAKE_INSTALL_PREFIX}/include/repa/${dir})
+-endforeach()
++        LIBRARY DESTINATION ${ESPRESSO_INSTALL_LIBDIR}
++        ARCHIVE DESTINATION ${ESPRESSO_INSTALL_LIBDIR})
+--- a/cmake/define_example.cmake
++++ b/cmake/define_example.cmake
+@@ -38,7 +38,7 @@ function(define_example)
+                                   Boost::serialization)
+
+     target_include_directories(${EXAMPLE_NAME}
+-                               PRIVATE ${CMAKE_SOURCE_DIR})
++                               PRIVATE ${PROJECT_SOURCE_DIR})
+     target_compile_options(${EXAMPLE_NAME}
+                            PRIVATE ${REPA_DEFAULT_COMPILE_OPTIONS})
+ endfunction(define_example)
+--- a/cmake/define_test.cmake
++++ b/cmake/define_test.cmake
+@@ -40,7 +40,7 @@ function(define_test)
+                                   ${TEST_LIBRARIES})
+
+     target_include_directories(${TEST_NAME}
+-                               PRIVATE ${CMAKE_SOURCE_DIR})
++                               PRIVATE ${PROJECT_SOURCE_DIR})
+     target_compile_options(${TEST_NAME}
+                            PRIVATE ${REPA_DEFAULT_COMPILE_OPTIONS})
+

doc/sphinx/installation.rst

@@ -575,6 +575,9 @@ using a CMake flag (see :ref:`Options and Variables`).
 - ``STOKESIAN_DYNAMICS`` Enables the Stokesian Dynamics feature
   (see :ref:`Stokesian Dynamics`). Requires BLAS and LAPACK.
 
+- ``LOAD_BALANCING`` Enables the MD dynamic load balancing feature
+  (not yet implemented). Builds and install the kdpart and librepa libraries.
+
 
 
 .. _Configuring:

maintainer/CI/build_cmake.sh

@@ -109,6 +109,7 @@ set_default_value with_ccache false
 set_default_value with_hdf5 true
 set_default_value with_scafacos false
 set_default_value with_stokesian_dynamics false
+set_default_value with_load_balancing false
 set_default_value test_timeout 300
 set_default_value hide_gpu false
 
@@ -159,6 +160,12 @@ else
     cmake_params="${cmake_params} -DWITH_STOKESIAN_DYNAMICS=OFF"
 fi
 
+if [ "${with_load_balancing}" = true ]; then
+    cmake_params="${cmake_params} -DWITH_LOAD_BALANCING=ON"
+else
+    cmake_params="${cmake_params} -DWITH_LOAD_BALANCING=OFF"
+fi
+
 if [ "${with_coverage}" = true ]; then
     cmake_params="-DWITH_COVERAGE=ON ${cmake_params}"
 fi