Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CMake: Improve handling of Albany and related packages #5950

Merged
merged 3 commits into from
Sep 26, 2023
Merged

CMake: Improve handling of Albany and related packages #5950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ecb3576
CMake: Improve handling of Albany and related packages
jgfouca Sep 22, 2023
c4ecf63
Improve Kokkos handling
jgfouca Sep 22, 2023
c5ad9f9
Remove obsolete references to ALBANY_PATH in cmake macros
jgfouca Sep 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/gnu9_mappy.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build/install")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/gnu_anlworkstation.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install")
if (NOT DEBUG)
string(APPEND CFLAGS " -O2")
endif()
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/gnu_mappy.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build/install")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/gnu_melvin.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/intel_bebop.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/soft/climate/AlbanyTrilinos_06262017/Albany/buildintel/install")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/intel_ghost.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install")
set(ESMF_LIBDIR "/projects/ccsm/esmf-6.3.0rp1/lib/libO/Linux.intel.64.openmpi.default")
if (MPILIB STREQUAL openmpi)
set(MPI_PATH "/opt/openmpi-1.8-intel")
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/intel_mappy.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/install/rhel7-x86_64/ACME/AlbanyTrilinos/Albany/build/install")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/intel_melvin.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
Expand Down
1 change: 0 additions & 1 deletion cime_config/machines/cmake_macros/intel_sandiatoss3.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
set(ALBANY_PATH "/projects/ccsm/AlbanyTrilinos_20190904/albany-build/install")
if (COMP_NAME STREQUAL gptl)
string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
endif()
Expand Down
2 changes: 2 additions & 0 deletions cime_config/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,8 @@
<env name="PERL5LIB">/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch</env>
<env name="FI_CXI_RX_MATCH_MODE">software</env>
<env name="MPICH_COLL_SYNC">MPI_Bcast</env>
<env name="Albany_ROOT">$SHELL{if [ -z "$Albany_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc; else echo "$Albany_ROOT"; fi}</env>
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The pattern for setting up ${Package}_ROOT is established here. It's a bit messy so I'm wondering if it would be better to move this login into the cmake macro file for the machine?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's a better bash syntax:

Albany_ROOT=${Albany_ROOT:=/global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc}

The ${VAR:=default} syntax expands to ${VAR} if VAR is defined, otherwise it expands to default. E.g.:

$ export FOO=foo
$ export BAR=${FOO:=bar} && echo $BAR
foo
$ export BAR={FOOBAR:=bar} && echo $BAR
bar

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bartgol , I tried that at first. The problem is that the CIME regex will incorrectly match the first } as terminating the SHELL command, so you can use that character in your syntax.

<env name="Trilinos_ROOT">$SHELL{if [ -z "$Trilinos_ROOT" ]; then echo /global/common/software/e3sm/mali_tpls/trilinos-e3sm-serial-release-gcc; else echo "$Trilinos_ROOT"; fi}</env>
</environment_variables>
<environment_variables compiler="gnu" mpilib="mpich">
<env name="ADIOS2_DIR">/global/cfs/cdirs/e3sm/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.15/gcc-11.2.0</env>
Expand Down
6 changes: 3 additions & 3 deletions components/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

cmake_minimum_required(VERSION 3.9)
cmake_policy(SET CMP0057 NEW)
cmake_policy(SET CMP0074 NEW)
set(CMAKE_CXX_STANDARD 17)

# Store caseroot in the cache, so that, if cmake re-runs,
Expand Down Expand Up @@ -127,9 +128,8 @@ list(APPEND CMAKE_MODULE_PATH ${CIMEROOT}/CIME/non_py/src/CMake)

set(CMAKE_VERBOSE_MAKEFILE TRUE)

if(USE_CUDA)
enable_language(CUDA)
endif()
# Find dependencies
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/find_dep_packages.cmake)

# Scream manages its own flags
build_eamxx()
Expand Down
74 changes: 2 additions & 72 deletions components/cmake/common_setup.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,65 +205,6 @@ if (USE_PETSC)
set(PETSC_LIB ${PETSC_LIBRARIES})
endif()

if (USE_TRILINOS)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So much better than what we had...

if (TRILINOS_PATH)
if (NOT INC_TRILINOS)
set(INC_TRILINOS ${TRILINOS_PATH}/include)
endif()
if (NOT LIB_TRILINOS)
set(LIB_TRILINOS ${TRILINOS_PATH}/lib)
endif()
else()
message(FATAL_ERROR "TRILINOS_PATH must be defined when USE_TRILINOS is TRUE")
endif()

set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${TRILINOS_PATH} PARENT_SCOPE)
find_package(Trilinos)
endif()

if (USE_ALBANY)
if (ALBANY_PATH)
if (NOT INC_ALBANY)
set(INC_ALBANY ${ALBANY_PATH}/include)
endif()
if (NOT LIB_ALBANY)
set(LIB_ALBANY ${ALBANY_PATH}/lib)
endif()
else()
message(FATAL_ERROR "ALBANY_PATH must be defined when USE_ALBANY is TRUE")
endif()

# get the "ALBANY_LINK_LIBS" list as an env var
file(READ ${ALBANY_PATH}/export_albany.in ALBANY_OUTPUT)
string(REPLACE "ALBANY_LINK_LIBS=" "" ALBANY_LINK_LIBS "${ALBANY_OUTPUT}")
endif()

if (USE_KOKKOS)
# LB 01/23
# CMake's find_package, when used with multiple PATHS and PATH_SUFFIXES,
# follows the following rule when looking in paths:
# 1. look in all the path suffixes of the first PATHS entry, in the order provided
# 2. look in the first PATH provided
# 3. repeat 1-2 with the following entry of PATHS
# 4. look in cmake/system default paths (unless told not to).
# So the following cmd will fist look in the KOKKOS_PATH folder and subfolders,
# if KOKKOS_PATH is non-empty. Then will proceed to look in the lib, lib/cmake,
# and lib64/cmake subfolders of the INSTALL_SHAREDPATH. If all of these fail,
# it will look in INSTALL_SHAREDPATH.

if (KOKKOS_PATH)
set (PATHS ${KOKKOS_PATH} ${INSTALL_SHAREDPATH})
elseif(DEFINED ENV{KOKKOS_PATH})
set (PATHS $ENV{KOKKOS_PATH} ${INSTALL_SHAREDPATH})
else()
set (PATHS ${INSTALL_SHAREDPATH})
endif()
find_package(Kokkos REQUIRED
PATHS ${PATHS}
PATH_SUFFIXES lib lib/cmake lib64/cmake
NO_DEFAULT_PATH)
endif()

# JGF: No one seems to be using this
# if (USE_MOAB)
# if (MOAB_PATH)
Expand Down Expand Up @@ -369,7 +310,7 @@ else()
list(APPEND INCLDIR "${INC_NETCDF_C}" "${INC_NETCDF_FORTRAN}")
endif()

foreach(ITEM MOD_NETCDF INC_MPI INC_PNETCDF INC_PETSC INC_TRILINOS INC_ALBANY) # INC_MOAB)
foreach(ITEM MOD_NETCDF INC_MPI INC_PNETCDF INC_PETSC) # INC_MOAB)
if (${ITEM})
list(APPEND INCLDIR "${${ITEM}}")
endif()
Expand Down Expand Up @@ -417,7 +358,7 @@ if (NOT HAS_COSP EQUAL -1)
set(USE_COSP TRUE)
endif()

# System libraries (netcdf, mpi, pnetcdf, esmf, trilinos, etc.)
# System libraries (netcdf, mpi, pnetcdf, esmf, etc.)
if (NOT SLIBS)
if (NOT NETCDF_SEPARATE)
set(SLIBS "-L${LIB_NETCDF} -lnetcdff -lnetcdf")
Expand All @@ -443,17 +384,6 @@ if (USE_PETSC)
set(SLIBS "${SLIBS} ${PETSC_LIB}")
endif()

# Add trilinos libraries; too be safe, we include all libraries included in the trilinos build,
# as well as all necessary third-party libraries
if (USE_TRILINOS)
set(SLIBS "${SLIBS} -L${LIB_TRILINOS} ${Trilinos_LIBRARIES} ${Trilinos_TPL_LIBRARY_DIRS} ${Trilinos_TPL_LIBRARIES}")
endif()

# Add Albany libraries. These are defined in the ALBANY_LINK_LIBS env var that was included above
if (USE_ALBANY)
set(SLIBS "${SLIBS} ${ALBANY_LINK_LIBS}")
endif()

# Add MOAB libraries. These are defined in the MOAB_LINK_LIBS env var that was included above
# if (USE_MOAB)
# set(SLIBS "${SLIBS} ${IMESH_LIBS}")
Expand Down
36 changes: 36 additions & 0 deletions components/cmake/find_dep_packages.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# This file is for finding pacakges needed by E3SM. It should be included
# from the main CMakeLists.txt file.
#
# Finding the correct packages will likely depend on the ${Package}_ROOT
# environment variable being set by config_machines.xml for your machine. Note
# that is environment var is case sensitive.

# Machine env vars should follow the following pattern:
# <env name="Package_ROOT">$SHELL{if [ -z "$Package_ROOT" ]; then echo /default/install/location; else echo "$Package_ROOT"; fi}</env>
#
# This will allow users to easily specify a different location for all their cases by
# simply setting ${Package}_ROOT in their shell.

# Kokkos' find_package needs to come before other find_packages
# that may define Kokkos targets so we can avoid duplicate target
# errors.
if (USE_KOKKOS)

# Kokkos will be built in the sharedlibs if Kokkos_ROOT is
# unset.
if (NOT DEFINED ENV{Kokkos_ROOT})
set(ENV{Kokkos_ROOT} ${INSTALL_SHAREDPATH})
endif()

find_package(Kokkos REQUIRED)
endif()

# Albany depends on Trilinos
if (USE_ALBANY OR USE_TRILINOS)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is fine, but for the record, I think that USE_ALBANY here is redundant. find_package(ALBANY REQUIRED) will also call find_package(Trilinos REQUIRED). If a CMake package is installed well, it is responsible of finding all its TPLs when you call find_package(YOUR_PKG).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like this is a deficiency with the Albany cmake system. I tried without USE_ALBANY check for Trilinos and I got this error:

CMake Error at /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc/lib64/Albany/cmake/albany-targets.cmake:61 (set_target_properties):
  The link interface of target "albanyLib" contains:

    Panzer::all_libs

  but the target was not found.  Possible reasons include:

    * There is a typo in the target name.
    * A find_package call is missing for an IMPORTED target.
    * An ALIAS target is missing.

Call Stack (most recent call first):
  /global/common/software/e3sm/mali_tpls/albany-e3sm-serial-release-gcc/lib64/Albany/cmake/AlbanyConfig.cmake:28 (include)
  cmake/find_dep_packages.cmake:45 (find_package)
  CMakeLists.txt:124 (include)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, we are probably not packaging albany that well.. :/

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am going through our cmake stuff, and indeed we lack a lot of stuff to consider us "good cmake citizens"... I will fix it at some point.

find_package(Trilinos REQUIRED)
endif()

if (USE_ALBANY)
find_package(Albany REQUIRED)
endif()

2 changes: 2 additions & 0 deletions components/mpas-albany-landice/src/landice.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
list(APPEND CPPDEFS "-DCORE_LANDICE")
list(APPEND INCLUDES "${CMAKE_BINARY_DIR}/core_landice/shared" "${CMAKE_BINARY_DIR}/core_landice/analysis_members" "${CMAKE_BINARY_DIR}/core_landice/mode_forward")

list(APPEND LIBRARIES ${Albany_LIBRARIES})

#
# Check if building with LifeV, Albany, and/or PHG external libraries
#
Expand Down
4 changes: 2 additions & 2 deletions share/build/buildlib.kokkos
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ OR
###############################################################################
def buildlib(bldroot, installpath, case):
###############################################################################
installed_kokkos_dir = os.environ.get("KOKKOS_PATH")
installed_kokkos_dir = os.environ.get("Kokkos_ROOT")
if installed_kokkos_dir is not None:
# We are trying to use a pre-installed kokkos. Look for the relevant folders/libs,
# and if all looks good, return. Otherwise, crap out
Expand All @@ -65,7 +65,7 @@ def buildlib(bldroot, installpath, case):
# cmake build system will do that soon enough, so any error will be caught there.
return
else:
print ("no value foudn in env for KOKKOS_PATH. building from scratch")
print ("no value foudn in env for Kokkos_ROOT. building from scratch")

srcroot = case.get_value("SRCROOT")
ekat_dir = os.path.join(srcroot, "externals", "ekat")
Expand Down