diff --git a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake index 8cc85b92cc6a..afcca8f479e5 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_frontier-scream-gpu.cmake @@ -34,5 +34,5 @@ if (COMP_NAME STREQUAL gptl) endif() set(PIO_FILESYSTEM_HINTS "lustre") -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DCMAKE_CXX_FLAGS='-std=gnu++14'") +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_VEGA90A=On -DCMAKE_CXX_FLAGS='-std=gnu++14' -DKokkos_ENABLE_OPENMP=OFF") set(USE_HIP "TRUE") diff --git a/components/eamxx/src/share/util/scream_utils.hpp b/components/eamxx/src/share/util/scream_utils.hpp index 6e4f08ffd22e..b1771f3a1aed 100644 --- a/components/eamxx/src/share/util/scream_utils.hpp +++ b/components/eamxx/src/share/util/scream_utils.hpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace scream { diff --git a/components/homme/CMakeLists.txt b/components/homme/CMakeLists.txt index c016182e3bf0..fdb996fa17af 100644 --- a/components/homme/CMakeLists.txt +++ b/components/homme/CMakeLists.txt @@ -447,18 +447,22 @@ if(HOMME_BUILD_EXECS AND NOT BUILD_HOMME_WITHOUT_PIOLIBRARY) ENDIF () ENDIF () -# If we don't need kokkos we don't need EKAT, and if -# Homme is built in EAMxx EKAT is already built -IF (HOMME_USE_KOKKOS AND HOMME_STANDALONE) - # Add ekat's cmake/pkg_build folder to cmake path +IF (HOMME_USE_KOKKOS) + # Add ekat's cmake scripts folders to cmake path set (EKAT_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat) set (EKAT_CMAKE_PATH ${EKAT_SOURCE_DIR}/cmake) list(APPEND CMAKE_MODULE_PATH - ${EKAT_CMAKE_PATH} - ${EKAT_CMAKE_PATH}/pkg_build - ${EKAT_CMAKE_PATH}/tpls + ${EKAT_CMAKE_PATH} + ${EKAT_CMAKE_PATH}/tpls ) - include (EkatBuildKokkos) + + # We first try to use find_package. If that doesn't work, build from EKAT's submodule + include (EkatFindKokkos) + if (NOT Kokkos_FOUND) + # The following script checks if Kokkos is already available as a target, and if so does nothing. + # For instance, if HOMME is built inside EAMxx, Kokkos will already be available + include (EkatBuildKokkos) + endif() ENDIF () # This folder contains the CMake macro used to build cxx unit tests diff --git a/components/homme/cmake/SetCompilerFlags.cmake b/components/homme/cmake/SetCompilerFlags.cmake index 49e79a463c41..9a37c6a36bd9 100644 --- a/components/homme/cmake/SetCompilerFlags.cmake +++ b/components/homme/cmake/SetCompilerFlags.cmake @@ -68,12 +68,6 @@ IF (${HOMME_USE_CXX}) SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") INCLUDE(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG("-std=c++14" CXX14_SUPPORTED) - IF (CXX14_SUPPORTED) - SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") - ELSEIF (${HOMME_USE_KOKKOS}) - MESSAGE (FATAL_ERROR "Kokkos needs C++14, but the C++ compiler does not support it.") - ENDIF () CHECK_CXX_COMPILER_FLAG("-cxxlib" CXXLIB_SUPPORTED) IF (CXXLIB_SUPPORTED) SET(CXXLIB_SUPPORTED_CACHE TRUE CACHE BOOL "") diff --git a/components/homme/src/preqx_kokkos/CMakeLists.txt b/components/homme/src/preqx_kokkos/CMakeLists.txt index 425e7c34da76..dff42bb97c66 100644 --- a/components/homme/src/preqx_kokkos/CMakeLists.txt +++ b/components/homme/src/preqx_kokkos/CMakeLists.txt @@ -168,6 +168,8 @@ MACRO(PREQX_KOKKOS_SETUP) ${SRC_SHARE_DIR}/cxx/EulerStepFunctor.cpp ${SRC_SHARE_DIR}/cxx/ExecSpaceDefs.cpp ${SRC_SHARE_DIR}/cxx/FunctorsBuffersManager.cpp + ${SRC_SHARE_DIR}/cxx/GllFvRemap.cpp + ${SRC_SHARE_DIR}/cxx/GllFvRemapImpl.cpp ${SRC_SHARE_DIR}/cxx/Hommexx_Session.cpp ${SRC_SHARE_DIR}/cxx/HybridVCoord.cpp ${SRC_SHARE_DIR}/cxx/HyperviscosityFunctor.cpp diff --git a/components/homme/src/preqx_kokkos/config.h.cmake.in b/components/homme/src/preqx_kokkos/config.h.cmake.in index c0bc4c311c39..119844f3def0 100644 --- a/components/homme/src/preqx_kokkos/config.h.cmake.in +++ b/components/homme/src/preqx_kokkos/config.h.cmake.in @@ -58,9 +58,8 @@ /* ZOLTAN2 SUBPACKAGE OF TRILINOS library */ #cmakedefine01 TRILINOS_HAVE_ZOLTAN2 -/* When doing BFB testing, we occasionally must use modified code. */ -/* Use this flag to protect such code. */ -#cmakedefine HOMMEXX_BFB_TESTING +/* Detect whether this is a kokkos target */ +#cmakedefine01 KOKKOS_TARGET /* Whether to use OpenMP4 */ #cmakedefine OMP4 diff --git a/components/homme/src/share/compose/cedr_util.hpp b/components/homme/src/share/compose/cedr_util.hpp index 52a1a4d77b5b..6d5ca7fecf57 100644 --- a/components/homme/src/share/compose/cedr_util.hpp +++ b/components/homme/src/share/compose/cedr_util.hpp @@ -4,11 +4,12 @@ #ifndef INCLUDE_CEDR_UTIL_HPP #define INCLUDE_CEDR_UTIL_HPP -#include - #include "cedr_kokkos.hpp" #include "cedr_mpi.hpp" +#include // For std::stringstream +#include // For std::cerr + namespace cedr { namespace util { diff --git a/components/homme/src/share/compose/compose_slmm.cpp b/components/homme/src/share/compose/compose_slmm.cpp index b8277b5423c6..27ab1e1a7d04 100644 --- a/components/homme/src/share/compose/compose_slmm.cpp +++ b/components/homme/src/share/compose/compose_slmm.cpp @@ -276,8 +276,6 @@ extern "C" { // Interface for Homme, through compose_mod.F90. void kokkos_init () { amb::dev_init_threads(); - Kokkos::InitArguments args; - args.disable_warnings = true; initialize_kokkos(); // Test these initialize correctly. Kokkos::View v("hi"); diff --git a/components/homme/src/share/compose/compose_slmm_siqk.cpp b/components/homme/src/share/compose/compose_slmm_siqk.cpp index 5f8c4474df17..628c023090cb 100644 --- a/components/homme/src/share/compose/compose_slmm_siqk.cpp +++ b/components/homme/src/share/compose/compose_slmm_siqk.cpp @@ -75,7 +75,7 @@ class TestSphereToRefKernel { } KOKKOS_INLINE_FUNCTION - void join (volatile value_type& dst, volatile value_type const& src) const { + void join (value_type& dst, value_type const& src) const { dst.max_nits = max(dst.max_nits, src.max_nits); dst.sum_nits += src.sum_nits; dst.nfails += src.nfails; diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.cpp b/components/homme/src/share/cxx/ExecSpaceDefs.cpp index 784d37b65d20..8d496bff5d16 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.cpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.cpp @@ -187,7 +187,7 @@ team_num_threads_vectors (const int num_parallel_iterations, #elif defined(KOKKOS_ENABLE_HIP) // Use 64 wavefronts per CU and 120 CUs. const int num_warps_device = 120*64; // no such thing Kokkos::Impl::hip_internal_maximum_warp_count(); - const int num_threads_warp = Kokkos::Experimental::Impl::HIPTraits::WarpSize; + const int num_threads_warp = Kokkos::Impl::HIPTraits::WarpSize; #else // I want thread-distribution rules to be unit-testable even when GPU spaces // are off. Thus, make up a GPU-like machine: diff --git a/components/homme/src/share/cxx/ExecSpaceDefs.hpp b/components/homme/src/share/cxx/ExecSpaceDefs.hpp index 8c18d8bcbb92..fbcd314cb2b9 100644 --- a/components/homme/src/share/cxx/ExecSpaceDefs.hpp +++ b/components/homme/src/share/cxx/ExecSpaceDefs.hpp @@ -143,11 +143,7 @@ struct DefaultThreadsDistribution { team_num_threads_vectors(const int num_parallel_iterations, const ThreadPreferences tp = ThreadPreferences()) { return Parallel::team_num_threads_vectors_from_pool( -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE - ExecSpaceType::thread_pool_size() -#else - ExecSpaceType::impl_thread_pool_size() -#endif + ExecSpaceType().impl_thread_pool_size() , num_parallel_iterations, tp); } }; diff --git a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp index 480c3782931c..771376e959fa 100644 --- a/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp +++ b/components/homme/test_execs/preqx_kokkos_ut/remap_preqx_ut.cpp @@ -10,6 +10,7 @@ #include "utilities/TestUtils.hpp" #include +#include TEST_CASE("remap_interface", "vertical remap") { diff --git a/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp b/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp index 12a7e9a1673e..ad15a1c10bdd 100644 --- a/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp +++ b/components/homme/test_execs/share_kokkos_ut/boundary_exchange_ut.cpp @@ -11,6 +11,7 @@ #include #include +#include using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp b/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp index b27b3f3281be..85368cd6ab0f 100644 --- a/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp +++ b/components/homme/test_execs/share_kokkos_ut/col_ops_ut.cpp @@ -1,15 +1,17 @@ #include -#include - #include "ColumnOps.hpp" -#include "Types.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SubviewUtils.hpp" #include "utilities/SyncUtils.hpp" #include "utilities/ViewUtils.hpp" +#include "Types.hpp" + +#include +#include + using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp b/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp index 5fe4a9c21633..401a029607fd 100644 --- a/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp +++ b/components/homme/test_execs/share_kokkos_ut/ppm_remap_ut.cpp @@ -8,6 +8,7 @@ #include "utilities/TestUtils.hpp" #include +#include using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp index e4cbb4ec4594..e261b60b390d 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_ml.cpp @@ -13,6 +13,7 @@ #include #include #include +#include using namespace Homme; diff --git a/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp b/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp index 26cec000976d..fd2f79fabae5 100644 --- a/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp +++ b/components/homme/test_execs/share_kokkos_ut/sphere_op_sl.cpp @@ -13,6 +13,7 @@ #include #include #include +#include using namespace Homme; diff --git a/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp b/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp index 8cccddb5118d..9627b0d6d61c 100644 --- a/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp +++ b/components/homme/test_execs/thetal_kokkos_ut/elem_ops_ut.cpp @@ -1,14 +1,15 @@ #include -#include - #include "ElementOps.hpp" -#include "Types.hpp" #include "utilities/TestUtils.hpp" #include "utilities/SubviewUtils.hpp" #include "utilities/SyncUtils.hpp" #include "utilities/ViewUtils.hpp" +#include "Types.hpp" + +#include +#include using namespace Homme; diff --git a/externals/ekat b/externals/ekat index 69a7b84e3963..44977d9eab51 160000 --- a/externals/ekat +++ b/externals/ekat @@ -1 +1 @@ -Subproject commit 69a7b84e39639aacb63f45861033d698b1cf085a +Subproject commit 44977d9eab51b812952b6bac7dfcb30aafdf7cb5