Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merge trilinos/develop into e3sm-project/develop #65

Merged
merged 9 commits into from
Dec 22, 2023
98 changes: 94 additions & 4 deletions packages/framework/ini-files/config-specs.ini
Original file line number Diff line number Diff line change
Expand Up @@ -1873,6 +1873,32 @@ opt-set-cmake-var CMAKE_CXX_FLAGS STRING
# Test failures as of 11-28-22
opt-set-cmake-var ROL_example_PDE-OPT_navier-stokes_example_01_MPI_4_DISABLE BOOL : ON

# Under loading and debug, these Tempus run too long and timeout (fail). 2023-12-21
opt-set-cmake-var Tempus_BackwardEuler_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_VanDerPol_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_ExplicitRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_HHTAlpha_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Combined_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_General_Partioned_IMEX_RK_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Tangent_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Newmark_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Test_NewmarkImplicitAForm_HarmonicOscillator_Damped_FirstOrder_MPI_1_DISABLE BOOL FORCE : ON

use GCC_PACKAGE_SPECIFIC_WARNING_FLAGS

use RHEL7_POST
Expand Down Expand Up @@ -2060,6 +2086,32 @@ opt-set-cmake-var Trilinos_ENABLE_Moertel BOOL CACHE FORCE : OFF
opt-set-cmake-var Trilinos_ENABLE_Komplex BOOL CACHE FORCE : OFF
opt-set-cmake-var Trilinos_ENABLE_Rythmos BOOL CACHE FORCE : OFF

# Under loading and debug, these Tempus run too long and timeout (fail). 2023-12-21
opt-set-cmake-var Tempus_BackwardEuler_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_VanDerPol_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_ExplicitRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_HHTAlpha_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Combined_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_General_Partioned_IMEX_RK_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Tangent_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Newmark_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Test_NewmarkImplicitAForm_HarmonicOscillator_Damped_FirstOrder_MPI_1_DISABLE BOOL FORCE : ON

[rhel7_sems-clang-11.0.1-openmpi-1.10.1-openmp_release-debug_shared_no-kokkos-arch_no-asan_no-complex_no-fpic_mpi_no-pt_no-rdc_no-uvm_deprecated-on_no-package-enables]
use rhel7_sems-clang-11.0.1-openmpi-1.10.1-serial_release-debug_shared_no-kokkos-arch_no-asan_no-complex_no-fpic_mpi_no-pt_no-rdc_no-uvm_deprecated-on_no-package-enables
opt-set-cmake-var Trilinos_ENABLE_OpenMP BOOL FORCE : ON
Expand Down Expand Up @@ -2179,19 +2231,31 @@ opt-set-cmake-var Zoltan_ch_simple_scotch_parallel_DISABLE BOOL FORCE : ON
opt-set-cmake-var Epetra_Directory_test_LL_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var SEACASAprepro_lib_aprepro_lib_array_test_DISABLE BOOL FORCE : ON

# These tests time out if other stuff is running on the machine at the same time.
# Not a LOT of other stuff, but they seem to be more resource-intensive than one would
# think looking at the MPI rank count.
# Under loading and debug, these Tempus run too long and timeout (fail). 2023-12-21
opt-set-cmake-var Tempus_BackwardEuler_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_VanDerPol_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_ExplicitRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_HHTAlpha_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_RSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Combined_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_General_Partioned_IMEX_RK_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Tangent_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Newmark_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Test_NewmarkImplicitAForm_HarmonicOscillator_Damped_FirstOrder_MPI_1_DISABLE BOOL FORCE : ON

# This is temporarily disabled because it seems to be particularly sensitive to the spack-built
# MPI issue (TRILFRAME-552)
Expand Down Expand Up @@ -2735,6 +2799,32 @@ opt-set-cmake-var TPL_ENABLE_ParMETIS BOOL FORCE : OFF
opt-set-cmake-var TPL_ENABLE_Pnetcdf BOOL FORCE : OFF
opt-set-cmake-var TPL_Netcdf_LIBRARIES STRING FORCE : -L${NETCDF_C_LIB|ENV};${NETCDF_C_LIB|ENV}/libnetcdf.a;${TPL_HDF5_LIBRARIES|CMAKE}

# Under loading and debug, these Tempus run too long and timeout (fail). 2023-12-21
opt-set-cmake-var Tempus_BackwardEuler_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_VanDerPol_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BackwardEuler_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_CDR_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_BDF2_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Combined_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_4th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_DIRK_Staggered_FSA_SDIRK_5_Stage_5th_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_ExplicitRK_ASA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_HHTAlpha_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Combined_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Combined_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_General_Partioned_IMEX_RK_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Partitioned_Staggered_FSA_Tangent_Partitioned_IMEX_RK_1st_Order_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_IMEX_RK_Staggered_FSA_Tangent_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Newmark_MPI_1_DISABLE BOOL FORCE : ON
opt-set-cmake-var Tempus_Test_NewmarkImplicitAForm_HarmonicOscillator_Damped_FirstOrder_MPI_1_DISABLE BOOL FORCE : ON

use RHEL7_POST

[rhel7_sems-v2-gnu-8.3.0-serial_release-debug_shared_no-kokkos-arch_no-asan_no-complex_no-fpic_no-mpi_no-pt_no-rdc_no-uvm_deprecated-on_all]
Expand Down
11 changes: 5 additions & 6 deletions packages/teuchos/comm/test/Comm/DefaultMpiComm_UnitTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,6 @@ TEUCHOS_UNIT_TEST_TEMPLATE_2_DECL( DefaultMpiComm, NonblockingSendReceiveSet, Or
using Teuchos::broadcast;
using Teuchos::SerialComm;
using Teuchos::rcp_dynamic_cast;
using std::cerr;
using std::endl;
typedef Teuchos::ScalarTraits<Packet> PT;
//typedef typename PT::magnitudeType PacketMag; // unused
Expand All @@ -645,7 +644,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_2_DECL( DefaultMpiComm, NonblockingSendReceiveSet, Or
const int numSendRecv = 4;
const int sendLen = 3;

cerr << "Creating data" << endl;
out << "Creating data" << endl;

const ArrayRCP<Packet> origInputData = arcp<Packet>(numSendRecv*sendLen);
const ArrayRCP<Packet> origOutputData = arcp<Packet>(numSendRecv*sendLen);
Expand All @@ -662,7 +661,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_2_DECL( DefaultMpiComm, NonblockingSendReceiveSet, Or
}
}
}
cerr << "Broadcasting data" << endl;
out << "Broadcasting data" << endl;
broadcast<Ordinal, Packet>( *comm, 0, origInputData() );

const ArrayRCP<Packet> inputData = arcpClone<Packet>(origInputData());
Expand All @@ -671,7 +670,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_2_DECL( DefaultMpiComm, NonblockingSendReceiveSet, Or
Array<RCP<Teuchos::CommRequest<Ordinal> > > recvRequests;
Array<RCP<Teuchos::CommRequest<Ordinal> > > sendRequests;

cerr << "Exchanging data" << endl;
out << "Exchanging data" << endl;

// Send from proc 0 to proc numProcs-1
if (procRank == 0) {
Expand Down Expand Up @@ -702,7 +701,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_2_DECL( DefaultMpiComm, NonblockingSendReceiveSet, Or
}
}

cerr << "Waiting on messages" << endl;
out << "Waiting on messages" << endl;

if (procRank == 0) {
waitAll( *comm, sendRequests() );
Expand All @@ -711,7 +710,7 @@ TEUCHOS_UNIT_TEST_TEMPLATE_2_DECL( DefaultMpiComm, NonblockingSendReceiveSet, Or
waitAll( *comm, recvRequests() );
}

cerr << "Testing received data" << endl;
out << "Testing received data" << endl;

if (!sendRequests.empty()) {
for (int i = 0; i < numSendRecv; ++i) {
Expand Down
18 changes: 18 additions & 0 deletions packages/tpetra/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,24 @@ ELSE () # NOT Tpetra_INST_SYCL
ENDIF ()
ENDIF () # Tpetra_INST_SYCL

############################################################
# Shared/Managed Memory Default Allocation
############################################################
# Normally we allocate Tpetra objects to the basic space by default,
# but some apps might want to allocate memory to the "shared" host/device
# space. This might be CudaUVMSpace, HIPManagedSpace or SYCLSharedUSMSpace.
# These options allow us to do that sort of thing, changing
# core/compat/Tpetra_KokkosCompat_ClassicNodeAPI_Wrapper.hpp
TRIBITS_ADD_OPTION_AND_DEFINE(
Tpetra_ALLOCATE_IN_SHARED_SPACE
HAVE_TPETRA_SHARED_ALLOCS
"Have Tpetra allocate memory in shared/managed space by default"
OFF
)
IF(Tpetra_ALLOCATE_IN_SHARED_SPACE)
MESSAGE(STATUS "- Tpetra: Allocating the shared/managed space by default.")
ENDIF()


############################################################
# Device-aware MPI Support
Expand Down
2 changes: 2 additions & 0 deletions packages/tpetra/core/cmake/TpetraCore_config.h.in
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,8 @@
#endif


#cmakedefine HAVE_TPETRA_SHARED_ALLOCS

#cmakedefine HAVE_TPETRA_EXPERIMENTAL

@TPETRA_DEPRECATED_DECLARATIONS@
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,16 +69,28 @@ class KokkosDeviceWrapperNode {
};

#ifdef KOKKOS_ENABLE_SYCL
#ifdef HAVE_TPETRA_SHARED_ALLOCS
typedef KokkosDeviceWrapperNode<::Kokkos::Experimental::SYCL, ::Kokkos::Experimental::SYCLSharedUSMSpace> KokkosSYCLWrapperNode;
#else
typedef KokkosDeviceWrapperNode<::Kokkos::Experimental::SYCL, ::Kokkos::Experimental::SYCLDeviceUSMSpace> KokkosSYCLWrapperNode;
#endif
#endif

#ifdef KOKKOS_ENABLE_HIP
#ifdef HAVE_TPETRA_SHARED_ALLOCS
typedef KokkosDeviceWrapperNode<::Kokkos::HIP, ::Kokkos::HIPManagedSpace> KokkosHIPWrapperNode;
#else
typedef KokkosDeviceWrapperNode<::Kokkos::HIP, ::Kokkos::HIPSpace> KokkosHIPWrapperNode;
#endif
#endif

#ifdef KOKKOS_ENABLE_CUDA
#ifdef HAVE_TPETRA_SHARED_ALLOCS
typedef KokkosDeviceWrapperNode<::Kokkos::Cuda,::Kokkos::CudaUVMSpace> KokkosCudaWrapperNode;
#else
typedef KokkosDeviceWrapperNode<::Kokkos::Cuda> KokkosCudaWrapperNode;
#endif
#endif

#ifdef KOKKOS_ENABLE_OPENMP
typedef KokkosDeviceWrapperNode<::Kokkos::OpenMP> KokkosOpenMPWrapperNode;
Expand Down
Loading