Merge pull request #4 from E3SM-Project/develop

Merge develop into master for intial release.

CMakeLists.txt

@@ -0,0 +1,81 @@
+cmake_minimum_required (VERSION 3.5)
+
+project (compose CXX)
+set (CMAKE_CXX_STANDARD 11)
+
+function (prc var)
+  message ("${var}: ${${var}}")
+endfunction ()
+
+find_package (MPI REQUIRED)
+
+if (Kokkos_DIR)
+  include (${Kokkos_DIR}/kokkos.cmake)
+  set (Kokkos_INCLUDE ${Kokkos_DIR}/include)
+else ()
+  message (FATAL_ERROR "COMPOSE requires Kokkos_DIR")
+endif ()
+
+set (SOURCES
+  cedr/cedr_caas.cpp
+  cedr/cedr_local.cpp
+  cedr/cedr_mpi.cpp
+  cedr/cedr_qlt.cpp
+  cedr/cedr_test.cpp
+  cedr/cedr_test_1d_transport.cpp
+  cedr/cedr_test_randomized.cpp
+  cedr/cedr_util.cpp)
+
+set (HEADERS
+  cedr/cedr.hpp
+  cedr/cedr_caas.hpp
+  cedr/cedr_caas_inl.hpp
+  cedr/cedr_cdr.hpp
+  cedr/cedr_kokkos.hpp
+  cedr/cedr_local.hpp
+  cedr/cedr_local_inl.hpp
+  cedr/cedr_mpi.hpp
+  cedr/cedr_mpi_inl.hpp
+  cedr/cedr_qlt.hpp
+  cedr/cedr_qlt_inl.hpp
+  cedr/cedr_test.hpp
+  cedr/cedr_test_randomized.hpp
+  cedr/cedr_util.hpp
+  siqk/siqk.hpp
+  siqk/siqk_defs.hpp
+  siqk/siqk_geometry.hpp
+  siqk/siqk_intersect.hpp
+  siqk/siqk_quadrature.hpp
+  siqk/siqk_search.hpp
+  siqk/siqk_sqr.hpp)
+
+if (NOT COMPOSE_TEST_MPIRUN)
+  set (COMPOSE_TEST_MPIRUN mpirun)
+endif ()
+if (NOT COMPOSE_TEST_NRANK)
+  set (COMPOSE_TEST_NRANK 8)
+endif ()
+
+set (COMPOSE_COMPILE_FLAGS "${MPI_COMPILE_FLAGS} ${KOKKOS_CXXFLAGS} ${CMAKE_CXX_FLAGS}")
+set (COMPOSE_LINK_FLAGS "${MPI_LINK_FLAGS} ${KOKKOS_LDFLAGS}")
+set (COMPOSE_INCLUDES "${Kokkos_INCLUDE}")
+set (COMPOSE_LIBRARIES ${MPI_LIBRARIES} ${KOKKOS_LIBS})
+
+prc(MPI_COMPILE_FLAGS)
+prc(MPI_LINK_FLAGS)
+prc(MPI_LIBRARIES)
+add_library (${PROJECT_NAME} ${SOURCES})
+set_target_properties (${PROJECT_NAME} PROPERTIES
+  COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
+  LINK_FLAGS ${COMPOSE_LINK_FLAGS})
+target_include_directories (${PROJECT_NAME} PUBLIC cedr siqk)
+target_include_directories (${PROJECT_NAME} PRIVATE siqk cedr)
+target_include_directories (${PROJECT_NAME} PUBLIC ${COMPOSE_INCLUDES})
+target_link_libraries (${PROJECT_NAME} ${COMPOSE_LIBRARIES})
+
+install (TARGETS ${PROJECT_NAME} ARCHIVE DESTINATION lib)
+install (FILES ${HEADERS} DESTINATION include/compose)
+
+enable_testing ()
+add_subdirectory(siqk)
+add_subdirectory(cedr)

README.md

@@ -1,2 +1,22 @@
 # COMPOSE
-Compact Multi-moment Performance-Portable Semi-Lagrangian methods for non-hydrostatic dynamics
+Compact Multi-moment Performance-Portable Semi-Lagrangian methods
+
+COMPOSE provides libraries for semi-Lagrangian transport and, together or
+separately, property preservation.
+
+CEDR: Communication-Efficient Constrained Density Reconstructors.
+SIQK: Sphereical Polygon Intersection and Quadrature.
+
+First, install Kokkos:
+    https://github.com/kokkos/kokkos
+For example, in a typical environment using OpenMP, a simple build line is:
+    ./kokkos/generate_makefile.bash --with-serial --with-openmp --prefix=/path/to/my/libs --compiler=g++
+    make -j8 install
+
+Second, configure, build, and test COMPOSE:
+    cmake \
+        -D Kokkos_DIR=/path/to/my/kokkos/install \
+        -D CMAKE_INSTALL_PREFIX=/path/to/my/compose/install \
+        /path/to/compose/repo
+    make -j8
+    ctest

cedr/CMakeLists.txt

@@ -0,0 +1,15 @@
+add_executable (cedr_test cedr_test.cpp)
+set_target_properties (cedr_test PROPERTIES
+  COMPILE_FLAGS ${COMPOSE_COMPILE_FLAGS}
+  LINK_FLAGS ${COMPOSE_LINK_FLAGS})
+
+target_include_directories (cedr_test PRIVATE ${COMPOSE_INCLUDES})
+target_link_libraries (cedr_test ${PROJECT_NAME} ${COMPOSE_LIBRARIES})
+
+add_test (NAME cedr-test-unit
+  COMMAND $<TARGET_FILE:cedr_test> -t)
+add_test (NAME cedr-test-unit-mpi
+  COMMAND ${COMPOSE_TEST_MPIRUN} -np ${COMPOSE_TEST_NRANK}
+  $<TARGET_FILE:cedr_test> -t --proc-random -nc 111 -nt 11)
+add_test (NAME cedr-test-t1d
+  COMMAND $<TARGET_FILE:cedr_test> -t -t1d -nc 111)

cedr/cedr.hpp

@@ -0,0 +1,31 @@
+#ifndef INCLUDE_CEDR_HPP
+#define INCLUDE_CEDR_HPP
+
+#include "cedr_kokkos.hpp"
+
+// Communication-Efficient Constrained Density Reconstructors
+namespace cedr {
+typedef int Int;
+typedef long int Long;
+typedef std::size_t Size;
+typedef double Real;
+
+// CDRs in general implement
+// * tracer mass, Qm, conservation;
+// * mixing ratio, q, shape preservation, either local bound preservation or
+//   dynamic range preservation; and
+// * tracer consistency, which follows from dynamic range preservation or
+//   stronger (including local bound preservation) with rhom coming from the
+//   dynamics.
+//
+// One can solve a subset of these.
+//   If !conserve, then the CDR does not alter the tracer mass, but it does not
+// correct for any failure in mass conservation in the field given to it.
+//   If consistent but !shapepreserve, the the CDR solves the dynamic range
+// preservation problem rather than the local bound preservation problem.
+struct ProblemType {
+  enum : Int { conserve = 1, shapepreserve = 1 << 1, consistent = 1 << 2 };
+};
+}
+
+#endif

cedr/cedr_caas.cpp

@@ -0,0 +1,214 @@
+#include "cedr_caas.hpp"
+#include "cedr_util.hpp"
+#include "cedr_test_randomized.hpp"
+
+namespace cedr {
+namespace caas {
+
+template <typename ES>
+CAAS<ES>::CAAS (const mpi::Parallel::Ptr& p, const Int nlclcells)
+  : p_(p), nlclcells_(nlclcells), nrhomidxs_(0), need_conserve_(false)
+{
+  cedr_throw_if(nlclcells == 0, "CAAS does not support 0 cells on a rank.");
+  tracer_decls_ = std::make_shared<std::vector<Decl> >();  
+}
+
+template <typename ES>
+void CAAS<ES>::declare_tracer(int problem_type, const Int& rhomidx) {
+  cedr_throw_if( ! (problem_type & ProblemType::shapepreserve),
+                "CAAS is a WIP; ! shapepreserve is not supported yet.");
+  cedr_throw_if(rhomidx > 0, "rhomidx > 0 is not supported yet.");
+  tracer_decls_->push_back(Decl(problem_type, rhomidx));
+  if (problem_type & ProblemType::conserve)
+    need_conserve_ = true;
+  nrhomidxs_ = std::max(nrhomidxs_, rhomidx+1);
+}
+
+template <typename ES>
+void CAAS<ES>::end_tracer_declarations () {
+  cedr_throw_if(tracer_decls_->size() == 0, "#tracers is 0.");
+  cedr_throw_if(nrhomidxs_ == 0, "#rhomidxs is 0.");
+  probs_ = IntList("CAAS probs", static_cast<Int>(tracer_decls_->size()));
+  t2r_ = IntList("CAAS t2r", static_cast<Int>(tracer_decls_->size()));
+  for (Int i = 0; i < probs_.extent_int(0); ++i) {
+    probs_(i) = (*tracer_decls_)[i].probtype;
+    t2r_(i) = (*tracer_decls_)[i].rhomidx;
+  }
+  tracer_decls_ = nullptr;
+  // (rho, Qm, Qm_min, Qm_max, [Qm_prev])
+  const Int e = need_conserve_ ? 1 : 0;
+  d_ = RealList("CAAS data", nlclcells_ * ((3+e)*probs_.size() + 1));
+  const auto nslots = 4*probs_.size();
+  // (e'Qm_clip, e'Qm, e'Qm_min, e'Qm_max, [e'Qm_prev])
+  send_ = RealList("CAAS send", nslots);
+  recv_ = RealList("CAAS recv", nslots);
+}
+
+template <typename ES>
+int CAAS<ES>::get_problem_type (const Int& tracer_idx) const {
+  cedr_assert(tracer_idx >= 0 && tracer_idx < probs_.extent_int(0));
+  return probs_[tracer_idx];
+}
+
+template <typename ES>
+Int CAAS<ES>::get_num_tracers () const {
+  return probs_.extent_int(0);
+}
+
+template <typename ES>
+void CAAS<ES>::reduce_locally () {
+  const Int nt = probs_.size();
+  Int k = 0;
+  Int os = nlclcells_;
+  // Qm_clip
+  for ( ; k < nt; ++k) {
+    Real Qm_sum = 0, Qm_clip_sum = 0;
+    for (Int i = 0; i < nlclcells_; ++i) {
+      const Real Qm = d_(os+i);
+      Qm_sum += (probs_(k) & ProblemType::conserve ?
+                 d_(os + nlclcells_*3*nt + i) /* Qm_prev */ :
+                 Qm);
+      const Real Qm_min = d_(os + nlclcells_*  nt + i);
+      const Real Qm_max = d_(os + nlclcells_*2*nt + i);
+      const Real Qm_clip = cedr::impl::min(Qm_max, cedr::impl::max(Qm_min, Qm));
+      Qm_clip_sum += Qm_clip;
+      d_(os+i) = Qm_clip;
+    }
+    send_(     k) = Qm_clip_sum;
+    send_(nt + k) = Qm_sum;
+    os += nlclcells_;
+  }
+  k += nt;
+  // Qm_min, Qm_max
+  for ( ; k < 4*nt; ++k) {
+    Real accum = 0;
+    for (Int i = 0; i < nlclcells_; ++i)
+      accum += d_(os+i);
+    send_(k) = accum;
+    os += nlclcells_;
+  }
+}
+
+template <typename ES>
+void CAAS<ES>::reduce_globally () {
+  int err = mpi::all_reduce(*p_, send_.data(), recv_.data(), send_.size(), MPI_SUM);
+  cedr_throw_if(err != MPI_SUCCESS,
+                "CAAS::reduce_globally MPI_Allreduce returned " << err);
+}
+
+template <typename ES>
+void CAAS<ES>::finish_locally () {
+  const Int nt = probs_.size();
+  Int os = nlclcells_;
+  for (Int k = 0; k < nt; ++k) {
+    const Real Qm_clip_sum = recv_(     k);
+    const Real Qm_sum      = recv_(nt + k);
+    const Real m = Qm_sum - Qm_clip_sum;
+    if (m < 0) {
+      const Real Qm_min_sum = recv_(2*nt + k);
+      Real fac = Qm_clip_sum - Qm_min_sum;
+      if (fac > 0) {
+        fac = m/fac;
+        for (Int i = 0; i < nlclcells_; ++i) {
+          const Real Qm_min = d_(os + nlclcells_*  nt + i);
+          Real& Qm = d_(os+i);
+          Qm += fac*(Qm - Qm_min);
+        }
+      }
+    } else if (m > 0) {
+      const Real Qm_max_sum = recv_(3*nt + k);
+      Real fac = Qm_max_sum - Qm_clip_sum;
+      if (fac > 0) {
+        fac = m/fac;
+        for (Int i = 0; i < nlclcells_; ++i) {
+          const Real Qm_max = d_(os + nlclcells_*2*nt + i);
+          Real& Qm = d_(os+i);
+          Qm += fac*(Qm_max - Qm);
+        }
+      }
+    }
+    os += nlclcells_;
+  }
+}
+
+template <typename ES>
+void CAAS<ES>::run () {
+  reduce_locally();
+  reduce_globally();
+  finish_locally();
+}
+
+namespace test {
+struct TestCAAS : public cedr::test::TestRandomized {
+  typedef CAAS<Kokkos::DefaultExecutionSpace> CAAST;
+
+  TestCAAS (const mpi::Parallel::Ptr& p, const Int& ncells, const bool verbose)
+    : TestRandomized("CAAS", p, ncells, verbose),
+      p_(p)
+  {
+    const auto np = p->size(), rank = p->rank();
+    nlclcells_ = ncells / np;
+    const Int todo = ncells - nlclcells_ * np;
+    if (rank < todo) ++nlclcells_;
+    caas_ = std::make_shared<CAAST>(p, nlclcells_);
+    init();
+  }
+
+  CDR& get_cdr () override { return *caas_; }
+
+  void init_numbering () override {
+    const auto np = p_->size(), rank = p_->rank();
+    Int start = 0;
+    for (Int lrank = 0; lrank < rank; ++lrank)
+      start += get_nllclcells(ncells_, np, lrank);
+    gcis_.resize(nlclcells_);
+    for (Int i = 0; i < nlclcells_; ++i)
+      gcis_[i] = start + i;
+  }
+
+  void init_tracers () override {
+    // CAAS doesn't yet support everything, so remove a bunch of the tracers.
+    std::vector<TestRandomized::Tracer> tracers;
+    Int idx = 0;
+    for (auto& t : tracers_) {
+      if ( ! (t.problem_type & ProblemType::shapepreserve) ||
+           ! t.local_should_hold)
+        continue;
+      t.idx = idx++;
+      tracers.push_back(t);
+      caas_->declare_tracer(t.problem_type, 0);
+    }
+    tracers_ = tracers;
+    caas_->end_tracer_declarations();
+  }
+
+  void run_impl (const Int trial) override {
+    caas_->run();
+  }
+
+private:
+  mpi::Parallel::Ptr p_;
+  Int nlclcells_;
+  CAAST::Ptr caas_;
+
+  static Int get_nllclcells (const Int& ncells, const Int& np, const Int& rank) {
+    Int nlclcells = ncells / np;
+    const Int todo = ncells - nlclcells * np;
+    if (rank < todo) ++nlclcells;
+    return nlclcells;
+  }
+};
+
+Int unittest (const mpi::Parallel::Ptr& p) {
+  const auto np = p->size();
+  Int nerr = 0;
+  for (Int nlclcells : {1, 2, 4, 11}) {
+    Long ncells = np*nlclcells;
+    if (ncells > np) ncells -= np/2;
+    nerr += TestCAAS(p, ncells, false).run(1, false);
+  }
+  return nerr;
+}
+} // namespace test
+} // namespace caas
+} // namespace cedr