From ecb3576a8e4b3d87c3d746f7765a5829dfd41fa0 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 22 Sep 2023 14:01:19 -0700 Subject: [PATCH 1/3] CMake: Improve handling of Albany and related packages --- cime_config/machines/config_machines.xml | 2 + components/CMakeLists.txt | 6 +- components/cmake/common_setup.cmake | 74 +------------------ components/cmake/find_dep_packages.cmake | 47 ++++++++++++ .../mpas-albany-landice/src/landice.cmake | 2 + 5 files changed, 56 insertions(+), 75 deletions(-) create mode 100644 components/cmake/find_dep_packages.cmake diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 3c427d429080..129e25ee05e7 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -257,6 +257,8 @@ /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software MPI_Bcast + $SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi} + $SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi} /global/cfs/cdirs/e3sm/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.15/gcc-11.2.0 diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 3163a6418376..d7a433c93075 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -15,6 +15,7 @@ cmake_minimum_required(VERSION 3.9) cmake_policy(SET CMP0057 NEW) +cmake_policy(SET CMP0074 NEW) set(CMAKE_CXX_STANDARD 17) # Store caseroot in the cache, so that, if cmake re-runs, @@ -127,9 +128,8 @@ list(APPEND CMAKE_MODULE_PATH ${CIMEROOT}/CIME/non_py/src/CMake) set(CMAKE_VERBOSE_MAKEFILE TRUE) -if(USE_CUDA) - enable_language(CUDA) -endif() +# Find dependencies +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/find_dep_packages.cmake) # Scream manages its own flags build_eamxx() diff --git a/components/cmake/common_setup.cmake b/components/cmake/common_setup.cmake index b16fb5fd565b..89c0c35a15ce 100644 --- a/components/cmake/common_setup.cmake +++ b/components/cmake/common_setup.cmake @@ -205,65 +205,6 @@ if (USE_PETSC) set(PETSC_LIB ${PETSC_LIBRARIES}) endif() -if (USE_TRILINOS) - if (TRILINOS_PATH) - if (NOT INC_TRILINOS) - set(INC_TRILINOS ${TRILINOS_PATH}/include) - endif() - if (NOT LIB_TRILINOS) - set(LIB_TRILINOS ${TRILINOS_PATH}/lib) - endif() - else() - message(FATAL_ERROR "TRILINOS_PATH must be defined when USE_TRILINOS is TRUE") - endif() - - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${TRILINOS_PATH} PARENT_SCOPE) - find_package(Trilinos) -endif() - -if (USE_ALBANY) - if (ALBANY_PATH) - if (NOT INC_ALBANY) - set(INC_ALBANY ${ALBANY_PATH}/include) - endif() - if (NOT LIB_ALBANY) - set(LIB_ALBANY ${ALBANY_PATH}/lib) - endif() - else() - message(FATAL_ERROR "ALBANY_PATH must be defined when USE_ALBANY is TRUE") - endif() - - # get the "ALBANY_LINK_LIBS" list as an env var - file(READ ${ALBANY_PATH}/export_albany.in ALBANY_OUTPUT) - string(REPLACE "ALBANY_LINK_LIBS=" "" ALBANY_LINK_LIBS "${ALBANY_OUTPUT}") -endif() - -if (USE_KOKKOS) - # LB 01/23 - # CMake's find_package, when used with multiple PATHS and PATH_SUFFIXES, - # follows the following rule when looking in paths: - # 1. look in all the path suffixes of the first PATHS entry, in the order provided - # 2. look in the first PATH provided - # 3. repeat 1-2 with the following entry of PATHS - # 4. look in cmake/system default paths (unless told not to). - # So the following cmd will fist look in the KOKKOS_PATH folder and subfolders, - # if KOKKOS_PATH is non-empty. Then will proceed to look in the lib, lib/cmake, - # and lib64/cmake subfolders of the INSTALL_SHAREDPATH. If all of these fail, - # it will look in INSTALL_SHAREDPATH. - - if (KOKKOS_PATH) - set (PATHS ${KOKKOS_PATH} ${INSTALL_SHAREDPATH}) - elseif(DEFINED ENV{KOKKOS_PATH}) - set (PATHS $ENV{KOKKOS_PATH} ${INSTALL_SHAREDPATH}) - else() - set (PATHS ${INSTALL_SHAREDPATH}) - endif() - find_package(Kokkos REQUIRED - PATHS ${PATHS} - PATH_SUFFIXES lib lib/cmake lib64/cmake - NO_DEFAULT_PATH) -endif() - # JGF: No one seems to be using this # if (USE_MOAB) # if (MOAB_PATH) @@ -369,7 +310,7 @@ else() list(APPEND INCLDIR "${INC_NETCDF_C}" "${INC_NETCDF_FORTRAN}") endif() -foreach(ITEM MOD_NETCDF INC_MPI INC_PNETCDF INC_PETSC INC_TRILINOS INC_ALBANY) # INC_MOAB) +foreach(ITEM MOD_NETCDF INC_MPI INC_PNETCDF INC_PETSC) # INC_MOAB) if (${ITEM}) list(APPEND INCLDIR "${${ITEM}}") endif() @@ -417,7 +358,7 @@ if (NOT HAS_COSP EQUAL -1) set(USE_COSP TRUE) endif() -# System libraries (netcdf, mpi, pnetcdf, esmf, trilinos, etc.) +# System libraries (netcdf, mpi, pnetcdf, esmf, etc.) if (NOT SLIBS) if (NOT NETCDF_SEPARATE) set(SLIBS "-L${LIB_NETCDF} -lnetcdff -lnetcdf") @@ -443,17 +384,6 @@ if (USE_PETSC) set(SLIBS "${SLIBS} ${PETSC_LIB}") endif() -# Add trilinos libraries; too be safe, we include all libraries included in the trilinos build, -# as well as all necessary third-party libraries -if (USE_TRILINOS) - set(SLIBS "${SLIBS} -L${LIB_TRILINOS} ${Trilinos_LIBRARIES} ${Trilinos_TPL_LIBRARY_DIRS} ${Trilinos_TPL_LIBRARIES}") -endif() - -# Add Albany libraries. These are defined in the ALBANY_LINK_LIBS env var that was included above -if (USE_ALBANY) - set(SLIBS "${SLIBS} ${ALBANY_LINK_LIBS}") -endif() - # Add MOAB libraries. These are defined in the MOAB_LINK_LIBS env var that was included above # if (USE_MOAB) # set(SLIBS "${SLIBS} ${IMESH_LIBS}") diff --git a/components/cmake/find_dep_packages.cmake b/components/cmake/find_dep_packages.cmake new file mode 100644 index 000000000000..14e4a963e055 --- /dev/null +++ b/components/cmake/find_dep_packages.cmake @@ -0,0 +1,47 @@ +# This file is for finding pacakges needed by E3SM. It should be included +# from the main CMakeLists.txt file. +# +# Finding the correct packages will likely depend on the ${Package}_ROOT +# environment variable being set by config_machines.xml for your machine. Note +# that is environment var is case sensitive. + +# Machine env vars should follow the following pattern: +# $SHELL{if [ -z "$Package_ROOT" ]; then echo /default/install/location; else echo "$Package_ROOT"; fi} +# +# This will allow users to easily specify a different location for all their cases by +# simply setting ${Package}_ROOT in their shell. + +if (USE_KOKKOS) + # LB 01/23 + # CMake's find_package, when used with multiple PATHS and PATH_SUFFIXES, + # follows the following rule when looking in paths: + # 1. look in all the path suffixes of the first PATHS entry, in the order provided + # 2. look in the first PATH provided + # 3. repeat 1-2 with the following entry of PATHS + # 4. look in cmake/system default paths (unless told not to). + # So the following cmd will fist look in the KOKKOS_PATH folder and subfolders, + # if KOKKOS_PATH is non-empty. Then will proceed to look in the lib, lib/cmake, + # and lib64/cmake subfolders of the INSTALL_SHAREDPATH. If all of these fail, + # it will look in INSTALL_SHAREDPATH. + + if (KOKKOS_PATH) + set (PATHS ${KOKKOS_PATH} ${INSTALL_SHAREDPATH}) + elseif(DEFINED ENV{KOKKOS_PATH}) + set (PATHS $ENV{KOKKOS_PATH} ${INSTALL_SHAREDPATH}) + else() + set (PATHS ${INSTALL_SHAREDPATH}) + endif() + find_package(Kokkos REQUIRED + PATHS ${PATHS} + PATH_SUFFIXES lib lib/cmake lib64/cmake + NO_DEFAULT_PATH) +endif() + +if (USE_ALBANY) + find_package(Albany REQUIRED) +endif() + +# Albany depends on Trilinos +if (USE_ALBANY OR USE_TRILINOS) + find_package(Trilinos REQUIRED) +endif() diff --git a/components/mpas-albany-landice/src/landice.cmake b/components/mpas-albany-landice/src/landice.cmake index 3b550fd34067..83f542dbabf8 100644 --- a/components/mpas-albany-landice/src/landice.cmake +++ b/components/mpas-albany-landice/src/landice.cmake @@ -3,6 +3,8 @@ list(APPEND CPPDEFS "-DCORE_LANDICE") list(APPEND INCLUDES "${CMAKE_BINARY_DIR}/core_landice/shared" "${CMAKE_BINARY_DIR}/core_landice/analysis_members" "${CMAKE_BINARY_DIR}/core_landice/mode_forward") +list(APPEND LIBRARIES ${Albany_LIBRARIES}) + # # Check if building with LifeV, Albany, and/or PHG external libraries # From c4ecf6377dc7d919ec74d3e14e27f2eb83b96408 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 22 Sep 2023 15:12:01 -0600 Subject: [PATCH 2/3] Improve Kokkos handling --- components/cmake/find_dep_packages.cmake | 37 +++++++++--------------- share/build/buildlib.kokkos | 4 +-- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/components/cmake/find_dep_packages.cmake b/components/cmake/find_dep_packages.cmake index 14e4a963e055..88e488e84930 100644 --- a/components/cmake/find_dep_packages.cmake +++ b/components/cmake/find_dep_packages.cmake @@ -11,37 +11,26 @@ # This will allow users to easily specify a different location for all their cases by # simply setting ${Package}_ROOT in their shell. +# Kokkos' find_package needs to come before other find_packages +# that may define Kokkos targets so we can avoid duplicate target +# errors. if (USE_KOKKOS) - # LB 01/23 - # CMake's find_package, when used with multiple PATHS and PATH_SUFFIXES, - # follows the following rule when looking in paths: - # 1. look in all the path suffixes of the first PATHS entry, in the order provided - # 2. look in the first PATH provided - # 3. repeat 1-2 with the following entry of PATHS - # 4. look in cmake/system default paths (unless told not to). - # So the following cmd will fist look in the KOKKOS_PATH folder and subfolders, - # if KOKKOS_PATH is non-empty. Then will proceed to look in the lib, lib/cmake, - # and lib64/cmake subfolders of the INSTALL_SHAREDPATH. If all of these fail, - # it will look in INSTALL_SHAREDPATH. - if (KOKKOS_PATH) - set (PATHS ${KOKKOS_PATH} ${INSTALL_SHAREDPATH}) - elseif(DEFINED ENV{KOKKOS_PATH}) - set (PATHS $ENV{KOKKOS_PATH} ${INSTALL_SHAREDPATH}) - else() - set (PATHS ${INSTALL_SHAREDPATH}) + # Kokkos will be built in the sharedlibs if Kokkos_ROOT is + # unset. + if (NOT DEFINED ENV{Kokkos_ROOT}) + set(ENV{Kokkos_ROOT} ${INSTALL_SHAREDPATH}) endif() - find_package(Kokkos REQUIRED - PATHS ${PATHS} - PATH_SUFFIXES lib lib/cmake lib64/cmake - NO_DEFAULT_PATH) -endif() -if (USE_ALBANY) - find_package(Albany REQUIRED) + find_package(Kokkos REQUIRED) endif() # Albany depends on Trilinos if (USE_ALBANY OR USE_TRILINOS) find_package(Trilinos REQUIRED) endif() + +if (USE_ALBANY) + find_package(Albany REQUIRED) +endif() + diff --git a/share/build/buildlib.kokkos b/share/build/buildlib.kokkos index 9a5836d27636..177d296a581f 100755 --- a/share/build/buildlib.kokkos +++ b/share/build/buildlib.kokkos @@ -48,7 +48,7 @@ OR ############################################################################### def buildlib(bldroot, installpath, case): ############################################################################### - installed_kokkos_dir = os.environ.get("KOKKOS_PATH") + installed_kokkos_dir = os.environ.get("Kokkos_ROOT") if installed_kokkos_dir is not None: # We are trying to use a pre-installed kokkos. Look for the relevant folders/libs, # and if all looks good, return. Otherwise, crap out @@ -65,7 +65,7 @@ def buildlib(bldroot, installpath, case): # cmake build system will do that soon enough, so any error will be caught there. return else: - print ("no value foudn in env for KOKKOS_PATH. building from scratch") + print ("no value foudn in env for Kokkos_ROOT. building from scratch") srcroot = case.get_value("SRCROOT") ekat_dir = os.path.join(srcroot, "externals", "ekat") From c5ad9f975754c4fbf45c20f6e1cf548df0a7707f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 25 Sep 2023 14:11:41 -0600 Subject: [PATCH 3/3] Remove obsolete references to ALBANY_PATH in cmake macros --- cime_config/machines/cmake_macros/gnu9_mappy.cmake | 1 - cime_config/machines/cmake_macros/gnu_anlworkstation.cmake | 1 - cime_config/machines/cmake_macros/gnu_mappy.cmake | 1 - cime_config/machines/cmake_macros/gnu_melvin.cmake | 1 - cime_config/machines/cmake_macros/intel_bebop.cmake | 1 - cime_config/machines/cmake_macros/intel_ghost.cmake | 1 - cime_config/machines/cmake_macros/intel_mappy.cmake | 1 - cime_config/machines/cmake_macros/intel_melvin.cmake | 1 - cime_config/machines/cmake_macros/intel_sandiatoss3.cmake | 1 - 9 files changed, 9 deletions(-) diff --git a/cime_config/machines/cmake_macros/gnu9_mappy.cmake b/cime_config/machines/cmake_macros/gnu9_mappy.cmake index 02800982111d..8adaceed0895 100644 --- a/cime_config/machines/cmake_macros/gnu9_mappy.cmake +++ b/cime_config/machines/cmake_macros/gnu9_mappy.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() diff --git a/cime_config/machines/cmake_macros/gnu_anlworkstation.cmake b/cime_config/machines/cmake_macros/gnu_anlworkstation.cmake index eab6570b0cc5..a4b64f364e18 100644 --- a/cime_config/machines/cmake_macros/gnu_anlworkstation.cmake +++ b/cime_config/machines/cmake_macros/gnu_anlworkstation.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install") if (NOT DEBUG) string(APPEND CFLAGS " -O2") endif() diff --git a/cime_config/machines/cmake_macros/gnu_mappy.cmake b/cime_config/machines/cmake_macros/gnu_mappy.cmake index 02800982111d..8adaceed0895 100644 --- a/cime_config/machines/cmake_macros/gnu_mappy.cmake +++ b/cime_config/machines/cmake_macros/gnu_mappy.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() diff --git a/cime_config/machines/cmake_macros/gnu_melvin.cmake b/cime_config/machines/cmake_macros/gnu_melvin.cmake index c320ac339ace..8adaceed0895 100644 --- a/cime_config/machines/cmake_macros/gnu_melvin.cmake +++ b/cime_config/machines/cmake_macros/gnu_melvin.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() diff --git a/cime_config/machines/cmake_macros/intel_bebop.cmake b/cime_config/machines/cmake_macros/intel_bebop.cmake index 62221d2a5a8c..83b54a2cc4cf 100644 --- a/cime_config/machines/cmake_macros/intel_bebop.cmake +++ b/cime_config/machines/cmake_macros/intel_bebop.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/soft/climate/AlbanyTrilinos_06262017/Albany/buildintel/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() diff --git a/cime_config/machines/cmake_macros/intel_ghost.cmake b/cime_config/machines/cmake_macros/intel_ghost.cmake index 3bf190ad60c6..b91b4cecf701 100644 --- a/cime_config/machines/cmake_macros/intel_ghost.cmake +++ b/cime_config/machines/cmake_macros/intel_ghost.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install") set(ESMF_LIBDIR "/projects/ccsm/esmf-6.3.0rp1/lib/libO/Linux.intel.64.openmpi.default") if (MPILIB STREQUAL openmpi) set(MPI_PATH "/opt/openmpi-1.8-intel") diff --git a/cime_config/machines/cmake_macros/intel_mappy.cmake b/cime_config/machines/cmake_macros/intel_mappy.cmake index 64dd290052df..0d32fd47b91e 100644 --- a/cime_config/machines/cmake_macros/intel_mappy.cmake +++ b/cime_config/machines/cmake_macros/intel_mappy.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() diff --git a/cime_config/machines/cmake_macros/intel_melvin.cmake b/cime_config/machines/cmake_macros/intel_melvin.cmake index dd58c586d01c..0d32fd47b91e 100644 --- a/cime_config/machines/cmake_macros/intel_melvin.cmake +++ b/cime_config/machines/cmake_macros/intel_melvin.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif() diff --git a/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake b/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake index 60df91218402..836a8a6dbe17 100644 --- a/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake +++ b/cime_config/machines/cmake_macros/intel_sandiatoss3.cmake @@ -1,4 +1,3 @@ -set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_SLASHPROC") endif()