Merge pull request #3 from E3SM-Project/configure-with-cmake

Configure with CMake.

CMakeLists.txt

@@ -0,0 +1,81 @@
+cmake_minimum_required (VERSION 3.5)
+
+project (compose CXX)
+set (CMAKE_CXX_STANDARD 11)
+
+function (prc var)
+  message ("${var}: ${${var}}")
+endfunction ()
+
+find_package (MPI REQUIRED)
+
+if (Kokkos_DIR)
+  include (${Kokkos_DIR}/kokkos.cmake)
+  set (Kokkos_INCLUDE ${Kokkos_DIR}/include)
+else ()
+  message (FATAL_ERROR "COMPOSE requires Kokkos_DIR")
+endif ()
+
+set (SOURCES
+  cedr/cedr_caas.cpp
+  cedr/cedr_local.cpp
+  cedr/cedr_mpi.cpp
+  cedr/cedr_qlt.cpp
+  cedr/cedr_test.cpp
+  cedr/cedr_test_1d_transport.cpp
+  cedr/cedr_test_randomized.cpp
+  cedr/cedr_util.cpp)
+
+set (HEADERS
+  cedr/cedr.hpp
+  cedr/cedr_caas.hpp
+  cedr/cedr_caas_inl.hpp
+  cedr/cedr_cdr.hpp
+  cedr/cedr_kokkos.hpp
+  cedr/cedr_local.hpp
+  cedr/cedr_local_inl.hpp
+  cedr/cedr_mpi.hpp
+  cedr/cedr_mpi_inl.hpp
+  cedr/cedr_qlt.hpp
+  cedr/cedr_qlt_inl.hpp
+  cedr/cedr_test.hpp
+  cedr/cedr_test_randomized.hpp
+  cedr/cedr_util.hpp
+  siqk/siqk.hpp
+  siqk/siqk_defs.hpp
+  siqk/siqk_geometry.hpp
+  siqk/siqk_intersect.hpp
+  siqk/siqk_quadrature.hpp
+  siqk/siqk_search.hpp
+  siqk/siqk_sqr.hpp)
+
+if (NOT COMPOSE_TEST_MPIRUN)
+  set (COMPOSE_TEST_MPIRUN mpirun)
+endif ()
+if (NOT COMPOSE_TEST_NRANK)
+  set (COMPOSE_TEST_NRANK 8)
+endif ()
+
+set (COMPOSE_COMPILE_FLAGS "${MPI_COMPILE_FLAGS} ${KOKKOS_CXXFLAGS} ${CMAKE_CXX_FLAGS}")
+set (COMPOSE_LINK_FLAGS "${MPI_LINK_FLAGS} ${KOKKOS_LDFLAGS}")
+set (COMPOSE_INCLUDES "${Kokkos_INCLUDE}")
+set (COMPOSE_LIBRARIES ${MPI_LIBRARIES} ${KOKKOS_LIBS})
+
+prc(MPI_COMPILE_FLAGS)
+prc(MPI_LINK_FLAGS)
+prc(MPI_LIBRARIES)
+add_library (${PROJECT_NAME} ${SOURCES})
+set_target_properties (${PROJECT_NAME} PROPERTIES
+  COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
+  LINK_FLAGS ${COMPOSE_LINK_FLAGS})
+target_include_directories (${PROJECT_NAME} PUBLIC cedr siqk)
+target_include_directories (${PROJECT_NAME} PRIVATE siqk cedr)
+target_include_directories (${PROJECT_NAME} PUBLIC ${COMPOSE_INCLUDES})
+target_link_libraries (${PROJECT_NAME} ${COMPOSE_LIBRARIES})
+
+install (TARGETS ${PROJECT_NAME} ARCHIVE DESTINATION lib)
+install (FILES ${HEADERS} DESTINATION include/compose)
+
+enable_testing ()
+add_subdirectory(siqk)
+add_subdirectory(cedr)

README.md

@@ -1,2 +1,22 @@
 # COMPOSE
-Compact Multi-moment Performance-Portable Semi-Lagrangian methods for non-hydrostatic dynamics
+Compact Multi-moment Performance-Portable Semi-Lagrangian methods
+
+COMPOSE provides libraries for semi-Lagrangian transport and, together or
+separately, property preservation.
+
+CEDR: Communication-Efficient Constrained Density Reconstructors.
+SIQK: Sphereical Polygon Intersection and Quadrature.
+
+First, install Kokkos:
+    https://github.com/kokkos/kokkos
+For example, in a typical environment using OpenMP, a simple build line is:
+    ./kokkos/generate_makefile.bash --with-serial --with-openmp --prefix=/path/to/my/libs --compiler=g++
+    make -j8 install
+
+Second, configure, build, and test COMPOSE:
+    cmake \
+        -D Kokkos_DIR=/path/to/my/kokkos/install \
+        -D CMAKE_INSTALL_PREFIX=/path/to/my/compose/install \
+        /path/to/compose/repo
+    make -j8
+    ctest

cedr/CMakeLists.txt

@@ -0,0 +1,15 @@
+add_executable (cedr_test cedr_test.cpp)
+set_target_properties (cedr_test PROPERTIES
+  COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
+  LINK_FLAGS ${COMPOSE_LINK_FLAGS})
+
+target_include_directories (cedr_test PRIVATE ${COMPOSE_INCLUDES})
+target_link_libraries (cedr_test ${PROJECT_NAME} ${COMPOSE_LIBRARIES})
+
+add_test (NAME cedr-test-unit
+  COMMAND $<TARGET_FILE:cedr_test> -t)
+add_test (NAME cedr-test-unit-mpi
+  COMMAND ${COMPOSE_TEST_MPIRUN} -np ${COMPOSE_TEST_NRANK}
+  $<TARGET_FILE:cedr_test> -t --proc-random -nc 111 -nt 11)
+add_test (NAME cedr-test-t1d
+  COMMAND $<TARGET_FILE:cedr_test> -t -t1d -nc 111)

cedr/cedr_qlt_inl.hpp

@@ -38,7 +38,8 @@ void QLT<ES>::set_Qm (const Int& lclcellidx, const Int& tracer_idx,
       cedr_kernel_throw_if(true, "set_Q: invalid problem_type.");
     }
     if (problem_type & ProblemType::conserve) {
-      cedr_kernel_throw_if(Qm_prev < 0, "Qm_prev was not provided to set_Q.");
+      cedr_kernel_throw_if(Qm_prev < -0.5,
+                           "Qm_prev was not provided to set_Q.");
       bd[3] = Qm_prev;
     }
   }

cedr/cedr_test.cpp

@@ -74,7 +74,7 @@ struct InputParser {
 } // namespace cedr
 
 int main (int argc, char** argv) {
-  int nerr = 0;
+  int nerr = 0, retval = 0;
   MPI_Init(&argc, &argv);
   auto p = cedr::mpi::make_parallel(MPI_COMM_WORLD);
   srand(p->rank());
@@ -93,15 +93,17 @@ int main (int argc, char** argv) {
     {
       int gnerr;
       cedr::mpi::reduce(*p, &nerr, &gnerr, 1, MPI_SUM, p->root());
+      retval = gnerr != 0 ? -1 : 0;
       if (p->amroot())
         std::cout << (gnerr != 0 ? "FAIL" : "PASS") << "\n";
     }
   } catch (const std::exception& e) {
     if (p->amroot())
       std::cerr << e.what();
+    retval = -1;
   }
   Kokkos::finalize_all();
   if (nerr) prc(nerr);
   MPI_Finalize();
-  return 0;
+  return retval;
 }

siqk/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_executable (siqk_test siqk_test.cpp)
+set_target_properties (siqk_test PROPERTIES
+  COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
+  LINK_FLAGS ${COMPOSE_LINK_FLAGS})
+target_include_directories (siqk_test PRIVATE ${COMPOSE_INCLUDES})
+target_link_libraries (siqk_test ${COMPOSE_LIBRARIES})
+
+configure_file (siqk_runtests.py siqk_runtests.py)
+
+add_test (NAME siqk-test-area
+  COMMAND python siqk_runtests.py $<TARGET_FILE:siqk_test> 0)
+add_test (NAME siqk-test-cube
+  COMMAND python siqk_runtests.py $<TARGET_FILE:siqk_test> 1)

siqk/siqk_defs.hpp

@@ -155,20 +155,20 @@ const_slice (const VT& v, Int i) { return ko::subview(v, i, ko::ALL()); }
 #else
 template <typename VT> KOKKOS_FORCEINLINE_FUNCTION
 typename VT::value_type*
-slice (const VT& v, Int i) { return v.ptr_on_device() + v.dimension_1()*i; }
+slice (const VT& v, Int i) { return v.data() + v.extent(1)*i; }
 
 template <typename VT> KOKKOS_FORCEINLINE_FUNCTION
 typename VT::const_value_type*
-const_slice (const VT& v, Int i) { return v.ptr_on_device() + v.dimension_1()*i; }
+const_slice (const VT& v, Int i) { return v.data() + v.extent(1)*i; }
 #endif
 
 // Number of slices in a 2D array, where each row is a slice.
 template <typename A2D> KOKKOS_FORCEINLINE_FUNCTION
-Int nslices (const A2D& a) { return static_cast<Int>(a.dimension_0()); }
+Int nslices (const A2D& a) { return static_cast<Int>(a.extent(0)); }
 
 // Number of entries in a 2D array's row.
 template <typename A2D> KOKKOS_FORCEINLINE_FUNCTION
-Int szslice (const A2D& a) { return static_cast<Int>(a.dimension_1()); }
+Int szslice (const A2D& a) { return static_cast<Int>(a.extent(1)); }
 
 template <typename V, typename CV>
 KOKKOS_INLINE_FUNCTION
@@ -184,8 +184,19 @@ void resize_and_copy (DV& d, const SV& s,
 }
 
 template <typename DV, typename SV>
-void resize_and_copy (DV& d, const SV& s,
-                      typename std::enable_if<DV::rank == 2>::type* = 0) {
+void resize_and_copy (
+  DV& d, const SV& s,
+  typename std::enable_if<DV::rank == 2 && DV::rank_dynamic == 1>::type* = 0)
+{
+  ko::resize(d, nslices(s));
+  ko::deep_copy(d, s);
+}
+
+template <typename DV, typename SV>
+void resize_and_copy (
+  DV& d, const SV& s,
+  typename std::enable_if<DV::rank == 2 && DV::rank_dynamic == 2>::type* = 0)
+{
   ko::resize(d, nslices(s), szslice(s));
   ko::deep_copy(d, s);
 }

siqk/siqk_intersect.hpp

@@ -195,7 +195,7 @@ void fill_normals (sh::Mesh<ko::HostSpace>& m) {
   // Fill.
   Idxs::HostMirror en("en", nslices(m.e), szslice(m.e));
   ko::deep_copy(en, -1);
-  Vec3s::HostMirror nml("nml", ne, 3);
+  Vec3s::HostMirror nml("nml", ne);
   Int ie = 0;
   for (Int ip = 0; ip < nslices(m.e); ++ip)
     for (Int iv = 0; iv < szslice(m.e); ++iv)
@@ -248,9 +248,9 @@ class AreaOTFunctor {
       // In and out vertex lists.
       Real buf[9*max_nvert];
       RawVec3s
-        vi(buf, max_nvert, 3),
-        vo(buf + 3*max_nvert, max_nvert, 3),
-        wrk(buf + 6*max_nvert, max_nvert, 3);
+        vi(buf, max_nvert),
+        vo(buf + 3*max_nvert, max_nvert),
+        wrk(buf + 6*max_nvert, max_nvert);
       Int ni;
       ni = 0;
       for (Int i = 0; i < szslice(e_); ++i) {

siqk/siqk_quadrature.hpp

@@ -566,40 +566,40 @@ class TriangleQuadrature {
                  RawConstArray& weight) const {
     switch (order) {
     case 4:
-      coord = RawConstVec3s(trisym_order4_coord_, 6, 3);
+      coord = RawConstVec3s(trisym_order4_coord_, 6);
       weight = RawConstArray(trisym_order4_weight_, 6);
       break;
     case 6:
-      coord = RawConstVec3s(tritay_order6_coord_, 11, 3);
+      coord = RawConstVec3s(tritay_order6_coord_, 11);
       weight = RawConstArray(tritay_order6_weight_, 11);
       break;
     case 8:
-      coord = RawConstVec3s(trisym_order8_coord_, 16, 3);
+      coord = RawConstVec3s(trisym_order8_coord_, 16);
       weight = RawConstArray(trisym_order8_weight_, 16);
       break;
     case 12:
 #ifdef SIQK_USE_TRITAY12
-      coord = RawConstVec3s(tritay_order12_coord_, 32, 3);
+      coord = RawConstVec3s(tritay_order12_coord_, 32);
       weight = RawConstArray(tritay_order12_weight_, 32);
 #else
-      coord = RawConstVec3s(trisym_order12_coord_, 33, 3);
+      coord = RawConstVec3s(trisym_order12_coord_, 33);
       weight = RawConstArray(trisym_order12_weight_, 33);
 #endif
       break;
     case 14:
-      coord = RawConstVec3s(trisym_order14_coord_, 46, 3);
+      coord = RawConstVec3s(trisym_order14_coord_, 46);
       weight = RawConstArray(trisym_order14_weight_, 46);
       break;
     case 16:
-      coord = RawConstVec3s(tritay_order16_coord_, 55, 3);
+      coord = RawConstVec3s(tritay_order16_coord_, 55);
       weight = RawConstArray(tritay_order16_weight_, 55);
       break;
     case 18:
-      coord = RawConstVec3s(tritay_order18_coord_, 66, 3);
+      coord = RawConstVec3s(tritay_order18_coord_, 66);
       weight = RawConstArray(tritay_order18_weight_, 66);
       break;
     case 20:
-      coord = RawConstVec3s(trisym_order20_coord_, 88, 3);
+      coord = RawConstVec3s(trisym_order20_coord_, 88);
       weight = RawConstArray(trisym_order20_weight_, 88);
       break;
     default: