From 67e21808c31c4b4632b25f77e99bef27d45e8807 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 13 Oct 2023 16:04:07 -0600 Subject: [PATCH 01/15] PIO: Remove dependence on CIME/Tools/Makefile We need to remove all dependencies on CIME/Tools/Makefile so we can change our macros to use cmake-style names for things. PIO/Scorpio was the first thing I picked. The situation with spio was strange. It was using CIME/Tools/Makefile to call CMake. It translated the Makefile-style settings to Cmake using a lot of extra code in the Makefile to set up CMAKE_OPTS. We change the python wrapper build script, buildlib.spio, to instead just set up the cmake options in python, allowing us to skip the CIME/Tools/Makefile. On top of removing the dependency, I think this makes the overall spio build a lot easier to understand, although it was a bit tedious pulling things out of the macros. Once the macros are in proper cmake-stype, I think we could just pass them directly to scorpio's CMake build via -C and things will be even simpler. [BFB] --- .../crayclang-scream_crusher-scream-gpu.cmake | 6 +- .../machines/cmake_macros/intel_chicoma.cmake | 11 -- .../machines/cmake_macros/nag_cascade.cmake | 11 -- .../machines/cmake_macros/pathscale.cmake | 19 -- share/build/buildlib.spio | 177 +++++++++--------- 5 files changed, 92 insertions(+), 132 deletions(-) delete mode 100644 cime_config/machines/cmake_macros/intel_chicoma.cmake delete mode 100644 cime_config/machines/cmake_macros/nag_cascade.cmake delete mode 100644 cime_config/machines/cmake_macros/pathscale.cmake diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake index 20a7b89907d1..3cca240d9cc5 100644 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake +++ b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake @@ -8,9 +8,9 @@ endif() set(PIO_FILESYSTEM_HINTS "gpfs") string(APPEND CXX_LIBS " -lstdc++") -SET(CMAKE_C_COMPILER "cc" CACHE STRING "") -SET(CMAKE_Fortran_COMPILER "ftn" CACHE STRING "") -SET(CMAKE_CXX_COMPILER "hipcc" CACHE STRING "") +set(CMAKE_C_COMPILER "cc") +set(CMAKE_Fortran_COMPILER "ftn") +set(CMAKE_CXX_COMPILER "hipcc") string(APPEND CXXFLAGS " -I${MPICH_DIR}/include") string(APPEND LDFLAGS " -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.16/gtl/lib -lmpi_gtl_hsa") diff --git a/cime_config/machines/cmake_macros/intel_chicoma.cmake b/cime_config/machines/cmake_macros/intel_chicoma.cmake deleted file mode 100644 index 7cc9dad8c1b1..000000000000 --- a/cime_config/machines/cmake_macros/intel_chicoma.cmake +++ /dev/null @@ -1,11 +0,0 @@ -set(PIO_FILESYSTEM_HINTS "lustre") -if (MPILIB STREQUAL impi) - set(MPICC "mpiicc") - set(MPICXX "mpiicpc") - set(MPIFC "mpiifort") -endif() -if (NOT MPILIB STREQUAL impi) - set(MPICXX "mpic++") -endif() -string(APPEND SLIBS " -lpthread") -string(APPEND CXX_LIBS " -lstdc++") diff --git a/cime_config/machines/cmake_macros/nag_cascade.cmake b/cime_config/machines/cmake_macros/nag_cascade.cmake deleted file mode 100644 index 3302b14814f8..000000000000 --- a/cime_config/machines/cmake_macros/nag_cascade.cmake +++ /dev/null @@ -1,11 +0,0 @@ -string(APPEND CPPDEFS " -DnoI8") -if (DEBUG) - string(APPEND FFLAGS " -C=all -g -O0 -v") -endif() -if (DEBUG AND COMP_NAME STREQUAL eam) - string(APPEND FFLAGS " -C=all -g -nan -O0 -v") -endif() -if (MPILIB STREQUAL mvapich2) - set(MPI_PATH "$ENV{MPI_LIB}") -endif() -set(PIO_FILESYSTEM_HINTS "lustre") diff --git a/cime_config/machines/cmake_macros/pathscale.cmake b/cime_config/machines/cmake_macros/pathscale.cmake deleted file mode 100644 index 679500e2b1c3..000000000000 --- a/cime_config/machines/cmake_macros/pathscale.cmake +++ /dev/null @@ -1,19 +0,0 @@ -if (compile_threaded) - string(APPEND CFLAGS " -mp") -endif() -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRPATHSCALE") -string(APPEND FC_AUTO_R8 " -r8") -string(APPEND FFLAGS " -O -extend_source -ftpp -fno-second-underscore -funderscoring -byteswapio") -if (compile_threaded) - string(APPEND FFLAGS " -mp") -endif() -if (DEBUG) - string(APPEND FFLAGS " -g -trapuv -Wuninitialized") -endif() -string(APPEND FFLAGS_NOOPT " -O0") -set(HAS_F2008_CONTIGUOUS "FALSE") -if (compile_threaded) - string(APPEND LDFLAGS " -mp") -endif() -set(MPICC "mpicc") -set(MPIFC "mpif90") diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index ac25e2fff95d..4addd6d8fcb6 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -7,15 +7,25 @@ sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) import glob, re from standard_script_setup import * from CIME import utils -from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy +from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy, run_cmd from CIME.build import get_standard_makefile_args from CIME.case import Case logger = logging.getLogger(__name__) +############################################################################### +def extract_from_macros(macro_dump, varname): +############################################################################### + look_for = f"{varname} :=" + for line in macro_dump.splitlines(): + if look_for in line: + return line.split(":=")[-1].strip() + return "" + +############################################################################### def parse_command_line(args, description): - ############################################################################### +############################################################################### parser = argparse.ArgumentParser( usage="""\n{0} [--debug] OR @@ -50,29 +60,25 @@ OR ############################################################################### def buildlib(bldroot, installpath, case): - ############################################################################### - cime_model = case.get_value("MODEL") - caseroot = case.get_value("CASEROOT") - exeroot = case.get_value("EXEROOT") +############################################################################### + caseroot = case.get_value("CASEROOT") + exeroot = case.get_value("EXEROOT") pio_version = case.get_value("PIO_VERSION") - srcroot = case.get_value("SRCROOT") - scorpio_src_root_dir = None - if cime_model == "e3sm": - scorpio_src_root_dir = os.path.join(srcroot, "externals") - # Scorpio classic is derived from PIO1 - scorpio_classic_dir = "scorpio_classic" - # Scorpio is derived from PIO2 - scorpio_dir = "scorpio" - scorpio_classic_src_dir = os.path.join( - scorpio_src_root_dir, scorpio_classic_dir - ) - scorpio_src_dir = os.path.join(scorpio_src_root_dir, scorpio_dir) - if ( - not os.path.isdir(scorpio_src_root_dir) - or not os.path.isdir(scorpio_classic_src_dir) - or not os.path.isdir(scorpio_src_dir) - ): - scorpio_src_root_dir = None + srcroot = case.get_value("SRCROOT") + debug = case.get_value("DEBUG") + mpilib = case.get_value("MPILIB") + gmake = case.get_value("GMAKE") + gmake_j = case.get_value("GMAKE_J") + + scorpio_src_root_dir = os.path.join(srcroot, "externals") + # Scorpio classic is derived from PIO1 + scorpio_classic_dir = "scorpio_classic" + # Scorpio is derived from PIO2 + scorpio_dir = "scorpio" + scorpio_classic_src_dir = os.path.join(scorpio_src_root_dir, scorpio_classic_dir) + scorpio_new_src_dir = os.path.join(scorpio_src_root_dir, scorpio_dir) + scorpio_src_dir = scorpio_classic_src_dir if pio_version == 1 else scorpio_new_src_dir + expect(os.path.isdir(scorpio_src_dir), "Missing scorpio dir " + scorpio_src_dir) # find_package(ADIOS2) appears to work better with ADIOS2_DIR than # ADIOS2_ROOT. This is a workaround @@ -86,80 +92,75 @@ def buildlib(bldroot, installpath, case): # also defined in the environment. In this case we # will use the installed pio and not build it here. installed_pio_version = os.environ.get("PIO_VERSION_MAJOR") - logger.info( - "pio_version_major = {} pio_version = {}".format( - installed_pio_version, pio_version - ) - ) + logger.info(f"pio_version_major = {installed_pio_version} pio_version = {pio_version}") if installed_pio_version is not None and int(installed_pio_version) == pio_version: logger.info("Using installed PIO library") _set_pio_valid_values(case, os.environ.get("PIO_TYPENAME_VALID_VALUES")) return pio_model = "pio{}".format(pio_version) - pio_dir = os.path.join(bldroot, pio_model) - if not os.path.isdir(pio_dir): - os.makedirs(pio_dir) - casetools = case.get_value("CASETOOLS") - if scorpio_src_root_dir: - # Use old genf90 until "short" type is supported - cmake_opts = ( - '"-D GENF90_PATH=' - + os.path.join(scorpio_src_root_dir, scorpio_dir, "src/genf90") - + '" ' - ) - elif pio_version == 1: - cmake_opts = '"-D GENF90_PATH=$CIMEROOT/CIME/non_py/externals/genf90 "' + pio_bld_dir = os.path.join(bldroot, pio_model) + if not os.path.isdir(pio_bld_dir): + os.makedirs(pio_bld_dir) + + # Compute cmake args + # Use old genf90 until "short" type is supported + cmake_opts = "-Wno-dev " + cmake_opts = f"-DGENF90_PATH={scorpio_src_dir}/src/genf90 " + if "ADIOS2_ROOT" in os.environ: + cmake_opts += "-DWITH_ADIOS2:BOOL=ON " + if debug: + cmake_opts += "-DPIO_ENABLE_LOGGING=ON " + # Case changes for NetCDF/NETCDF forces us to do this. For other packages + # what's already in the env should work + if "NETCDF_PATH" in os.environ: + cmake_opts += f"-DNetCDF_PATH:PATH={os.environ['NETCDF_PATH']} " + if "NETCDF_C_PATH" in os.environ: + cmake_opts += f"-DNetCDF_C_PATH:PATH={os.environ['NETCDF_C_PATH']} " + if "NETCDF_FORTRAN_PATH" in os.environ: + cmake_opts += f"-DNetCDF_Fortran_PATH:PATH={os.environ['NETCDF_FORTRAN_PATH']} " + if "PNETCDF_PATH" in os.environ: + cmake_opts += f"-DPnetCDF_PATH:PATH={os.environ['PNETCDF_PATH']} " else: - cmake_opts = '"-D GENF90_PATH=' + srcroot + '/libraries/parallelio/scripts/ "' - - stdargs = get_standard_makefile_args(case, shared_lib=True) - - gmake_vars = ( - "CASEROOT={caseroot} COMP_NAME={pio_model} " - "USER_CMAKE_OPTS={cmake_opts} " - "PIO_LIBDIR={pio_dir} CASETOOLS={casetools} " - "USER_CPPDEFS=-DTIMING".format( - caseroot=caseroot, - pio_model=pio_model, - cmake_opts=cmake_opts, - pio_dir=pio_dir, - casetools=casetools, - ) - ) - - if scorpio_src_root_dir is not None: - gmake_vars += ( - " IO_LIB_SRCROOT={scorpio_src_root_dir} " - " IO_LIB_v1_SRCDIR={scorpio_classic_dir} " - " IO_LIB_v2_SRCDIR={scorpio_dir} ".format( - scorpio_src_root_dir=scorpio_src_root_dir, - scorpio_classic_dir=scorpio_classic_dir, - scorpio_dir=scorpio_dir, - ) - ) - - gmake_opts = ( - "{pio_dir}/Makefile -C {pio_dir} " - " {gmake_vars} {stdargs} -f {casetools}/Makefile".format( - pio_dir=pio_dir, gmake_vars=gmake_vars, casetools=casetools, stdargs=stdargs - ) - ) + cmake_opts += "-DWITH_PNETCDF:LOGICAL=FALSE -DPIO_USE_MPIIO:LOGICAL=FALSE " + if mpilib == "mpi-serial": + cmake_opts += f"-DPIO_USE_MPISERIAL=TRUE -DMPISERIAL_PATH={installpath} " + cmake_opts += "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON " + cmake_opts += f"-DGPTL_PATH:STRING={installpath} " + cmake_opts += "-DPIO_ENABLE_TESTS:BOOL=OFF -DPIO_USE_MALLOC:BOOL=ON " + cmake_opts += f"-DUSER_CMAKE_MODULE_PATH:LIST={cimeroot}/CIME/non_py/src/CMake " + + # Hacky way of getting stuff from Macros. Should scorpio just load the E3SM + # macros once they are using cmake names for things? + make_args = get_standard_makefile_args(case, shared_lib=True) + macro_dump = run_cmd(f"make -f Macros.make COMP_NAME=spio {make_args} -p")[1] + fflags = extract_from_macros(macro_dump, "FFLAGS") + cflags = extract_from_macros(macro_dump, "CFLAGS") + cxxflags = extract_from_macros(macro_dump, "CXXFLAGS") + cppdefs = extract_from_macros(macro_dump, "CPPDEFS") + ldflags = extract_from_macros(macro_dump, "LDFLAGS") + if mpilib == "mpi-serial": + fc = extract_from_macros(macro_dump, "SFC") + cc = extract_from_macros(macro_dump, "SCC") + cxx = extract_from_macros(macro_dump, "SCXX") + else: + fc = extract_from_macros(macro_dump, "MPIFC") + cc = extract_from_macros(macro_dump, "MPICC") + cxx = extract_from_macros(macro_dump, "MPICXX") - gmake_cmd = case.get_value("GMAKE") + cppdefs += " -DTIMING" + cmake_opts += f"-DCMAKE_Fortran_FLAGS:STRING='{fflags} {cppdefs}' " + cmake_opts += f"-DCMAKE_C_FLAGS:STRING='{cflags} {cppdefs}' " + cmake_opts += f"-DCMAKE_CXX_FLAGS:STRING='{cxxflags} {cppdefs}' " # This runs the pio cmake command from the cime case Makefile logger.info("Configuring SCORPIO") - cmd = "{} {}".format(gmake_cmd, gmake_opts) - run_bld_cmd_ensure_logging(cmd, logger, from_dir=pio_dir) + cmd = f"FC={fc} CC={cc} CXX={cxx} LDFLAGS='{ldflags}' cmake {cmake_opts} {scorpio_src_dir}" + run_bld_cmd_ensure_logging(cmd, logger, from_dir=pio_bld_dir) # This runs the pio make command from the cmake generated Makefile logger.info("Building SCORPIO") - run_bld_cmd_ensure_logging( - "{} -j {}".format(gmake_cmd, case.get_value("GMAKE_J")), - logger, - from_dir=pio_dir, - ) + run_bld_cmd_ensure_logging(f"{gmake} -j {gmake_j}", logger, from_dir=pio_bld_dir) if pio_version == 1: expect_string = "D_NETCDF;" @@ -177,7 +178,7 @@ def buildlib(bldroot, installpath, case): netcdf4_parallel_found = False adios_found = False - cache_file = open(os.path.join(pio_dir,"CMakeCache.txt"), "r") + cache_file = open(os.path.join(pio_bld_dir,"CMakeCache.txt"), "r") for line in cache_file: if re.search(expect_string, line): expect_string_found = True @@ -193,13 +194,13 @@ def buildlib(bldroot, installpath, case): installed_lib_time = 0 if os.path.isfile(installed_lib): installed_lib_time = os.path.getmtime(installed_lib) - newlib = os.path.join(pio_dir, "pio", "libpio.a") + newlib = os.path.join(pio_bld_dir, "pio", "libpio.a") newlib_time = os.path.getmtime(newlib) if newlib_time > installed_lib_time: logger.info("Installing pio version 1") safe_copy(newlib, installed_lib) for glob_to_copy in ("*.h", "*.mod"): - for item in glob.glob(os.path.join(pio_dir, "pio", glob_to_copy)): + for item in glob.glob(os.path.join(pio_bld_dir, "pio", glob_to_copy)): safe_copy(item, "{}/include".format(installpath)) else: globs_to_copy = [ @@ -216,7 +217,7 @@ def buildlib(bldroot, installpath, case): globs_to_copy.append(os.path.join("tools","adios2pio-nm","adios2pio-nm.exe")) for glob_to_copy in globs_to_copy: installed_file_time = 0 - for item in glob.glob(os.path.join(pio_dir, glob_to_copy)): + for item in glob.glob(os.path.join(pio_bld_dir, glob_to_copy)): if item.endswith(".a") or item.endswith(".so"): installdir = "lib" else: From d9c80c3c83d1e3d92030d490edd266de50169b18 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 16 Oct 2023 10:56:52 -0600 Subject: [PATCH 02/15] Fix circleCI to refer to correct PIO build name --- .circleci/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run.sh b/.circleci/run.sh index 593a13f93c90..7a625ebadb14 100755 --- a/.circleci/run.sh +++ b/.circleci/run.sh @@ -57,7 +57,7 @@ rc=$? if [ $rc -ne 0 ]; then print_bldlog "e3sm" print_bldlog "csm_share" - print_bldlog "pio" + print_bldlog "spio" print_bldlog "mct" print_bldlog "gptl" exit $rc From 6b75a3a6b243b1377cc7f552675f47b7ae16e8a3 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 16 Oct 2023 13:28:48 -0600 Subject: [PATCH 03/15] Fix cmake_opts mistake --- share/build/buildlib.spio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index 4addd6d8fcb6..0a8457d21c2b 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -106,7 +106,7 @@ def buildlib(bldroot, installpath, case): # Compute cmake args # Use old genf90 until "short" type is supported cmake_opts = "-Wno-dev " - cmake_opts = f"-DGENF90_PATH={scorpio_src_dir}/src/genf90 " + cmake_opts += f"-DGENF90_PATH={scorpio_src_dir}/src/genf90 " if "ADIOS2_ROOT" in os.environ: cmake_opts += "-DWITH_ADIOS2:BOOL=ON " if debug: From 94822ce248be67a8de71e23dbf2e59fd014fe175 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Mon, 16 Oct 2023 13:45:54 -0600 Subject: [PATCH 04/15] Try setting hdf5 path explicitly --- share/build/buildlib.spio | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index 0a8457d21c2b..c9065681bc48 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -119,6 +119,11 @@ def buildlib(bldroot, installpath, case): cmake_opts += f"-DNetCDF_C_PATH:PATH={os.environ['NETCDF_C_PATH']} " if "NETCDF_FORTRAN_PATH" in os.environ: cmake_opts += f"-DNetCDF_Fortran_PATH:PATH={os.environ['NETCDF_FORTRAN_PATH']} " + + if "HDF5_PATH" in os.environ: + cmake_opts += f"-DHDF5_PATH:STRING={os.environ['HDF5_PATH']} " + + if "PNETCDF_PATH" in os.environ: cmake_opts += f"-DPnetCDF_PATH:PATH={os.environ['PNETCDF_PATH']} " else: From a6997334cb43dfd4dbfc43d1f3889ed267fffa0b Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 18 Oct 2023 13:20:42 -0600 Subject: [PATCH 05/15] Prefer ROOT over PATH for env for cmake --- cime_config/machines/config_machines.xml | 38 ++++++++++++------------ share/build/buildlib.spio | 4 --- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index e5454f25c91d..7caa293ea543 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1264,7 +1264,7 @@ /usr/local/packages/netcdf-parallel /usr/local/packages/pnetcdf - /usr/local/packages/hdf5-parallel + /usr/local/packages/hdf5-parallel /usr/local/packages/cmake/bin:/usr/local/packages/mpich/bin:/usr/local/packages/hdf5-parallel/bin:/usr/local/packages/netcdf-parallel/bin:/usr/local/packages/pnetcdf/bin:$ENV{PATH} /usr/local/packages/mpich/lib:/usr/local/packages/szip/lib:/usr/local/packages/hdf5-parallel/lib:/usr/local/packages/netcdf-parallel/lib:/usr/local/packages/pnetcdf/lib @@ -1617,7 +1617,7 @@ /soft/apps/packages/climate/mpich/3.3.2/gcc-8.2.0/bin:/soft/apps/packages/climate/cmake/3.18.4/bin:/soft/apps/packages/climate/gmake/bin:$ENV{PATH} - /soft/apps/packages/climate/hdf5/1.8.16-parallel/mpich-3.3.2/gcc-8.2.0 + /soft/apps/packages/climate/hdf5/1.8.16-parallel/mpich-3.3.2/gcc-8.2.0 /soft/apps/packages/climate/netcdf/4.4.1c-4.2cxx-4.4.4f-parallel/mpich-3.3.2/gcc-8.2.0 @@ -1626,9 +1626,9 @@ /soft/apps/packages/climate/openmpi/2.1.5/gcc-8.2.0/bin:/soft/apps/packages/climate/cmake/3.18.4/bin:/soft/apps/packages/climate/gmake/bin:$ENV{PATH} - /soft/apps/packages/climate/zlib/1.2.11/gcc-8.2.0-static - /soft/apps/packages/climate/szip/2.1/gcc-8.2.0-static - /soft/apps/packages/climate/hdf5/1.8.12-parallel/openmpi-2.1.5/gcc-8.2.0-static + /soft/apps/packages/climate/zlib/1.2.11/gcc-8.2.0-static + /soft/apps/packages/climate/szip/2.1/gcc-8.2.0-static + /soft/apps/packages/climate/hdf5/1.8.12-parallel/openmpi-2.1.5/gcc-8.2.0-static /soft/apps/packages/climate/netcdf/4.7.4c-4.3.1cxx-4.4.4f-parallel/openmpi-2.1.5/gcc-8.2.0-static-hdf5-1.8.12-pnetcdf-1.12.0 /soft/apps/packages/climate/pnetcdf/1.12.0/openmpi-2.1.5/gcc-8.2.0 @@ -1705,8 +1705,8 @@ /nfs/gce/projects/climate/software/mpich/3.4.2/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/mpich/3.4.2/gcc-11.1.0/bin:$ENV{PATH} - /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/zlib-1.2.11-smoyzzo - /nfs/gce/projects/climate/software/hdf5/1.12.1/mpich-3.4.2/gcc-11.1.0 + /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/zlib-1.2.11-smoyzzo + /nfs/gce/projects/climate/software/hdf5/1.12.1/mpich-3.4.2/gcc-11.1.0 /nfs/gce/projects/climate/software/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-3.4.2/gcc-11.1.0 /nfs/gce/projects/climate/software/pnetcdf/1.12.2/mpich-3.4.2/gcc-11.1.0 @@ -1714,8 +1714,8 @@ /nfs/gce/projects/climate/software/openmpi/4.1.3/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/openmpi/4.1.3/gcc-11.1.0/bin:$ENV{PATH} - /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/zlib-1.2.11-smoyzzo - /nfs/gce/projects/climate/software/hdf5/1.12.1/openmpi-4.1.3/gcc-11.1.0 + /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/zlib-1.2.11-smoyzzo + /nfs/gce/projects/climate/software/hdf5/1.12.1/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/pnetcdf/1.12.2/openmpi-4.1.3/gcc-11.1.0 @@ -1792,8 +1792,8 @@ /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/bin:$ENV{PATH} - /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/zlib-1.2.11-p7dmb5p - /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/hdf5/1.12.1/mpich-4.0/gcc-11.1.0 + /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/zlib-1.2.11-p7dmb5p + /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/hdf5/1.12.1/mpich-4.0/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-4.0/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/pnetcdf/1.12.2/mpich-4.0/gcc-11.1.0 @@ -1801,8 +1801,8 @@ /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/openmpi/4.1.3/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/openmpi/4.1.3/gcc-11.1.0/bin:$ENV{PATH} - /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/zlib-1.2.11-p7dmb5p - /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/hdf5/1.12.1/openmpi-4.1.3/gcc-11.1.0 + /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/zlib-1.2.11-p7dmb5p + /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/hdf5/1.12.1/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/pnetcdf/1.12.2/openmpi-4.1.3/gcc-11.1.0 @@ -2456,7 +2456,7 @@ $SHELL{dirname $(dirname $(which pnetcdf_version))} - $SHELL{which h5dump | xargs dirname | xargs dirname} + $SHELL{which h5dump | xargs dirname | xargs dirname} 128M @@ -3375,7 +3375,7 @@ 0.05 0 - $SHELL{dirname $(dirname $(which h5diff))} + $SHELL{dirname $(dirname $(which h5diff))} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $ENV{MKLROOT} @@ -4207,7 +4207,7 @@ $ENV{OLCF_NETLIB_LAPACK_ROOT} Generic $ENV{OLCF_ESSL_ROOT} - $ENV{OLCF_HDF5_ROOT} + $ENV{OLCF_HDF5_ROOT} $ENV{OLCF_PARALLEL_NETCDF_ROOT} 0 @@ -4383,7 +4383,7 @@ $ENV{OLCF_NETLIB_LAPACK_ROOT} Generic $ENV{OLCF_ESSL_ROOT} - $ENV{OLCF_HDF5_ROOT} + $ENV{OLCF_HDF5_ROOT} 0 @@ -4642,7 +4642,7 @@ 0.2 0.20 - $SHELL{dirname $(dirname $(which h5diff))} + $SHELL{dirname $(dirname $(which h5diff))} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf-config))} @@ -4732,7 +4732,7 @@ 0.2 0.20 - $SHELL{dirname $(dirname $(which h5diff))} + $SHELL{dirname $(dirname $(which h5diff))} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf-config))} diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index c9065681bc48..ba535606d18a 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -120,10 +120,6 @@ def buildlib(bldroot, installpath, case): if "NETCDF_FORTRAN_PATH" in os.environ: cmake_opts += f"-DNetCDF_Fortran_PATH:PATH={os.environ['NETCDF_FORTRAN_PATH']} " - if "HDF5_PATH" in os.environ: - cmake_opts += f"-DHDF5_PATH:STRING={os.environ['HDF5_PATH']} " - - if "PNETCDF_PATH" in os.environ: cmake_opts += f"-DPnetCDF_PATH:PATH={os.environ['PNETCDF_PATH']} " else: From 2871f56f4824ab355039c919caa6cb91a3ce01bc Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 11:40:05 -0600 Subject: [PATCH 06/15] More help for scorpio to find HDF5 --- share/build/buildlib.spio | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index ba535606d18a..16fe1c134fce 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -4,7 +4,7 @@ import sys, os, logging, argparse cimeroot = os.getenv("CIMEROOT") sys.path.append(os.path.join(cimeroot, "CIME", "Tools")) -import glob, re +import glob, re, shutil from standard_script_setup import * from CIME import utils from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy, run_cmd @@ -120,6 +120,18 @@ def buildlib(bldroot, installpath, case): if "NETCDF_FORTRAN_PATH" in os.environ: cmake_opts += f"-DNetCDF_Fortran_PATH:PATH={os.environ['NETCDF_FORTRAN_PATH']} " + # scorpio needs a little help finding hdf5. The FindHDF5 in scopio uses the + # env var "HDF5" instead of "HDF5_ROOT". If HDF5_ROOT is not set, but we have + # h5dump in our path, we can use that info to set up HDF5. + which_h5dump = shutil.which('h5dump') + if "HDF5_ROOT" in os.environ: + os.environ["HDF5"] = os.environ["HDF5_ROOT"] + elif which_h5dump is not None: + os.environ["HDF5"] = os.path.dirname(os.path.dirname(which_h5dump)) + + if "HDF5" in os.environ: + cmake_opts += "-DWITH_HDF5=On " + if "PNETCDF_PATH" in os.environ: cmake_opts += f"-DPnetCDF_PATH:PATH={os.environ['PNETCDF_PATH']} " else: From 9b15d1f5832b1c0e8485e9275c44b635bc14f41d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 11:57:01 -0600 Subject: [PATCH 07/15] Fix for adios/gptl --- share/build/buildlib.spio | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index 16fe1c134fce..50aaa497b3a7 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -148,9 +148,10 @@ def buildlib(bldroot, installpath, case): make_args = get_standard_makefile_args(case, shared_lib=True) macro_dump = run_cmd(f"make -f Macros.make COMP_NAME=spio {make_args} -p")[1] fflags = extract_from_macros(macro_dump, "FFLAGS") - cflags = extract_from_macros(macro_dump, "CFLAGS") - cxxflags = extract_from_macros(macro_dump, "CXXFLAGS") - cppdefs = extract_from_macros(macro_dump, "CPPDEFS") + # Setting GPTL_PATH is currently insufficient scorpio to find gptl headers? + cflags = extract_from_macros(macro_dump, "CFLAGS") + f" -I{installpath}/include" + cxxflags = extract_from_macros(macro_dump, "CXXFLAGS") + f" -I{installpath}/include" + cppdefs = extract_from_macros(macro_dump, "CPPDEFS") + f" -I{installpath}/include" ldflags = extract_from_macros(macro_dump, "LDFLAGS") if mpilib == "mpi-serial": fc = extract_from_macros(macro_dump, "SFC") From d729e2ee8de0c079dacc1b61419de14bc786ed19 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 12:19:38 -0600 Subject: [PATCH 08/15] szip/zlib --- share/build/buildlib.spio | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index 50aaa497b3a7..f36af8ee0c9d 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -132,6 +132,12 @@ def buildlib(bldroot, installpath, case): if "HDF5" in os.environ: cmake_opts += "-DWITH_HDF5=On " + # Same deal with libz and szip + if "ZLIB_ROOT" in os.environ: + os.environ["ZLIB"] = os.environ["ZLIB_ROOT"] + if "SZIP_ROOT" in os.environ: + os.environ["SZIP"] = os.environ["SZIP_ROOT"] + if "PNETCDF_PATH" in os.environ: cmake_opts += f"-DPnetCDF_PATH:PATH={os.environ['PNETCDF_PATH']} " else: From ff15482bc96c9140dc459db078de3290511022b9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 12:53:52 -0600 Subject: [PATCH 09/15] Fix zlib/libz --- share/build/buildlib.spio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index f36af8ee0c9d..a5637bba6c77 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -134,7 +134,7 @@ def buildlib(bldroot, installpath, case): # Same deal with libz and szip if "ZLIB_ROOT" in os.environ: - os.environ["ZLIB"] = os.environ["ZLIB_ROOT"] + os.environ["LIBZ"] = os.environ["ZLIB_ROOT"] # LIBZ/ZLIB if "SZIP_ROOT" in os.environ: os.environ["SZIP"] = os.environ["SZIP_ROOT"] From 2ac88ca2e1d1c2925eb1ccd1a062891b4caa9716 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 13:36:42 -0600 Subject: [PATCH 10/15] Add the hdf5 dep to PIO --- components/cmake/modules/FindPIO.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index 1642dfb03b2b..b94fad1e1bb3 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -27,6 +27,7 @@ endif() list(APPEND PIOLIBS "${INSTALL_SHAREDPATH}/lib/libgptl.a") find_package(NETCDF REQUIRED) +find_package(HDF5 REQUIRED COMPONENTS C HL) # Not all machines/PIO installations use ADIOS but, for now, # we can assume that an MPI case with ADIOS2_ROOT set is probably @@ -46,6 +47,9 @@ else() list(APPEND PIOLIBS MPI::MPI_C MPI::MPI_Fortran) endif() +message("JGF HDF5_libs: ${HDF5_LIBRARIES}") +list(APPEND PIOLIBS ${HDF5_HL_LIBRARIES} ${HDF5_LIBRARIES}) + # Create the interface library, and set target properties add_library(spio INTERFACE) target_link_libraries(spio INTERFACE ${PIOLIBS}) From 02d87e0f815f5340f5892254013d40ae5debb426 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 13:47:38 -0600 Subject: [PATCH 11/15] Be less aggressive about turning hdf5 on for scorpio --- components/cmake/modules/FindPIO.cmake | 10 +++++++--- share/build/buildlib.spio | 7 ++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index b94fad1e1bb3..3a647b1297dd 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -27,7 +27,10 @@ endif() list(APPEND PIOLIBS "${INSTALL_SHAREDPATH}/lib/libgptl.a") find_package(NETCDF REQUIRED) -find_package(HDF5 REQUIRED COMPONENTS C HL) +# Check if scorpio has hdf5 enabled +if (DEFINED ENV{HDF5_ROOT}) + find_package(HDF5 REQUIRED COMPONENTS C HL) +endif() # Not all machines/PIO installations use ADIOS but, for now, # we can assume that an MPI case with ADIOS2_ROOT set is probably @@ -47,8 +50,9 @@ else() list(APPEND PIOLIBS MPI::MPI_C MPI::MPI_Fortran) endif() -message("JGF HDF5_libs: ${HDF5_LIBRARIES}") -list(APPEND PIOLIBS ${HDF5_HL_LIBRARIES} ${HDF5_LIBRARIES}) +if (DEFINED ENV{HDF5_ROOT}) + list(APPEND PIOLIBS ${HDF5_HL_LIBRARIES} ${HDF5_LIBRARIES}) +endif() # Create the interface library, and set target properties add_library(spio INTERFACE) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index a5637bba6c77..677232f51327 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -122,12 +122,13 @@ def buildlib(bldroot, installpath, case): # scorpio needs a little help finding hdf5. The FindHDF5 in scopio uses the # env var "HDF5" instead of "HDF5_ROOT". If HDF5_ROOT is not set, but we have - # h5dump in our path, we can use that info to set up HDF5. + # h5dump in our path, we can use that info to set up HDF5. For now, we will + # assume only HDF5_ROOT is what matters. which_h5dump = shutil.which('h5dump') if "HDF5_ROOT" in os.environ: os.environ["HDF5"] = os.environ["HDF5_ROOT"] - elif which_h5dump is not None: - os.environ["HDF5"] = os.path.dirname(os.path.dirname(which_h5dump)) + # elif which_h5dump is not None: + # os.environ["HDF5"] = os.path.dirname(os.path.dirname(which_h5dump)) if "HDF5" in os.environ: cmake_opts += "-DWITH_HDF5=On " From 9d1bd9d40ade12531697b9c687b75c6855d50db7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 13:59:35 -0600 Subject: [PATCH 12/15] Turn off hdf5 on summit for scorpio --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 7caa293ea543..68678095ef9e 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4207,7 +4207,7 @@ $ENV{OLCF_NETLIB_LAPACK_ROOT} Generic $ENV{OLCF_ESSL_ROOT} - $ENV{OLCF_HDF5_ROOT} + $ENV{OLCF_PARALLEL_NETCDF_ROOT} 0 From dc2b58ba26533e78a04fa59a8fbb7f942f0c2b54 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 15:11:59 -0600 Subject: [PATCH 13/15] Add WITH_HDF5_SCORPIO support --- share/build/buildlib.spio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index 677232f51327..be3242adda7c 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -130,7 +130,7 @@ def buildlib(bldroot, installpath, case): # elif which_h5dump is not None: # os.environ["HDF5"] = os.path.dirname(os.path.dirname(which_h5dump)) - if "HDF5" in os.environ: + if "WITH_HDF5_SCORPIO" in os.environ: cmake_opts += "-DWITH_HDF5=On " # Same deal with libz and szip From 213d5e5c05d31c213af3b07f155a080354c165a5 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 19 Oct 2023 15:21:05 -0600 Subject: [PATCH 14/15] Revert "Turn off hdf5 on summit for scorpio" This reverts commit 9d1bd9d40ade12531697b9c687b75c6855d50db7. --- cime_config/machines/config_machines.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 68678095ef9e..7caa293ea543 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -4207,7 +4207,7 @@ $ENV{OLCF_NETLIB_LAPACK_ROOT} Generic $ENV{OLCF_ESSL_ROOT} - + $ENV{OLCF_HDF5_ROOT} $ENV{OLCF_PARALLEL_NETCDF_ROOT} 0 From 464321fc075cbb7e5ac369671d08faa10b63aff8 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 20 Oct 2023 09:12:39 -0600 Subject: [PATCH 15/15] Fix macros extraction and hdf5 bool cmake setting --- share/build/buildlib.spio | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index be3242adda7c..57423c51bde3 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -18,7 +18,7 @@ def extract_from_macros(macro_dump, varname): ############################################################################### look_for = f"{varname} :=" for line in macro_dump.splitlines(): - if look_for in line: + if line.startswith(look_for): return line.split(":=")[-1].strip() return "" @@ -131,7 +131,7 @@ def buildlib(bldroot, installpath, case): # os.environ["HDF5"] = os.path.dirname(os.path.dirname(which_h5dump)) if "WITH_HDF5_SCORPIO" in os.environ: - cmake_opts += "-DWITH_HDF5=On " + cmake_opts += "-DWITH_HDF5:BOOL=ON " # Same deal with libz and szip if "ZLIB_ROOT" in os.environ: