From 6a2bebe06909386fdce735bd0d94b2c6750e8219 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 8 Aug 2024 13:09:08 -0400 Subject: [PATCH 1/5] Delete crusher from config_machines. --- cime_config/machines/config_machines.xml | 295 ----------------------- 1 file changed, 295 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 889fc03834e..84cbaacd550 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1094,301 +1094,6 @@ - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu - mpich - cli115 - /lustre/orion/cli115/world-shared/crusher - .* - /lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli115/world-shared/e3sm/baselines/crusher/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - 8 - 8 - 8 - TRUE - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - -c $ENV{OMP_NUM_THREADS} - $ENV{NTASKS_PER_GPU} - $ENV{GPU_BIND_ARGS} - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/15.0.1 - - - craype craype/2.7.20 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.4.0 - - - craype-accel-amd-gfx90a - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - gcc gcc/11.2.0 - - - craype-accel-amd-gfx90a - rocm/5.4.0 - - - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0.25 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} - --gpu-bind=closest - romio_cb_read=disable - 0 - - - 10 - 3 - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 56 - 56 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --threads-per-core=1 - -c $ENV{OMP_NUM_THREADS} - -m *:block - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - - PrgEnv-cray PrgEnv-cray/8.3.3 - cce cce/14.0.0 - - - - - PrgEnv-cray PrgEnv-amd/8.3.3 - amd amd/5.1.0 - - - - - PrgEnv-cray PrgEnv-gnu/8.3.3 - - - cray-mpich/8.1.12 - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-libsci/21.08.1.2 - cray-hdf5-parallel/1.12.1.1 - cray-netcdf-hdf5parallel/4.8.1.1 - cray-parallel-netcdf/1.12.1.7 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - - - - $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} - - - - 0 - - - - 128M - spread - threads - - - - - Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs - .*crusher.* - Linux - crayclang-scream - mpich - CLI115 - /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher - /lustre/orion/cli115/world-shared/e3sm/inputdata - /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 - $CIME_OUTPUT_ROOT/archive/$CASE - /lustre/orion/cli133/world-shared/e3sm/baselines/$COMPILER - /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc - 8 - 1 - slurm - e3sm - 64 - 8 - TRUE - - - srun - - -l -K -n {{ total_tasks }} -N {{ num_nodes }} - - - --gpus-per-node=8 --gpu-bind=closest - -c $ENV{OMP_NUM_THREADS} - - - - - - - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/libexec/lmod perl - module - module - /usr/share/lmod/lmod/libexec/lmod python - - - PrgEnv-cray - - craype-accel-amd-gfx90a - rocm/5.1.0 - - cce/14.0.3 - - - cray-python/3.9.4.2 - subversion/1.14.0 - git/2.31.1 - cmake/3.21.3 - zlib/1.2.11 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 - - - - $CIME_OUTPUT_ROOT/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - 0.1 - 0 - - $ENV{NETCDF_DIR} - $ENV{PNETCDF_DIR} - 0 - - 1 - romio_cb_read=disable - - - - 128M - spread - threads - - - Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* From bcc5b62f37e14c69a62cd7f57c11a9ec5d4ba5d8 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 8 Aug 2024 14:09:47 -0400 Subject: [PATCH 2/5] Deleted crusher files from cime_config/machines. --- .../Depends.crusher-gpu.crayclang.cmake | 43 --------- .../cmake_macros/amdclang_crusher.cmake | 12 --- .../cmake_macros/amdclanggpu_crusher.cmake | 26 ----- .../crayclang-scream_crusher-scream-cpu.cmake | 12 --- .../crayclang-scream_crusher-scream-gpu.cmake | 28 ------ .../crayclang-scream_crusher-scream.cmake | 12 --- .../cmake_macros/crayclang_crusher.cmake | 10 -- .../cmake_macros/crayclanggpu_crusher.cmake | 63 ------------- .../machines/cmake_macros/gnu_crusher.cmake | 15 --- .../cmake_macros/gnugpu_crusher.cmake | 25 ----- cime_config/machines/syslog.crusher | 94 ------------------- 11 files changed, 340 deletions(-) delete mode 100644 cime_config/machines/Depends.crusher-gpu.crayclang.cmake delete mode 100644 cime_config/machines/cmake_macros/amdclang_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclang_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/gnu_crusher.cmake delete mode 100644 cime_config/machines/cmake_macros/gnugpu_crusher.cmake delete mode 100755 cime_config/machines/syslog.crusher diff --git a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake b/cime_config/machines/Depends.crusher-gpu.crayclang.cmake deleted file mode 100644 index e41d959b52b..00000000000 --- a/cime_config/machines/Depends.crusher-gpu.crayclang.cmake +++ /dev/null @@ -1,43 +0,0 @@ -set(CICE_F90 - ice_FY.F90 - ice_aerosol.F90 - ice_age.F90 - ice_atmo.F90 - ice_blocks.F90 - ice_calendar.F90 - ice_diagnostics.F90 - ice_distribution.F90 - ice_domain.F90 - ice_domain_size.F90 - ice_dyn_evp.F90 - ice_fileunits.F90 - ice_flux.F90 - ice_forcing.F90 - ice_grid.F90 - ice_history.F90 - ice_history_fields.F90 - ice_init.F90 - ice_itd.F90 - ice_kinds_mod.F90 - ice_lvl.F90 - ice_mechred.F90 - ice_meltpond.F90 - ice_ocean.F90 - ice_orbital.F90 - ice_probability.F90 - ice_probability_tools.F90 - ice_read_write.F90 - ice_restoring.F90 - ice_shortwave.F90 - ice_spacecurve.F90 - ice_state.F90 - ice_step_mod.F90 - ice_therm_itd.F90 - ice_therm_vertical.F90 - ice_transport_driver.F90 - ice_transport_remap.F90 - ice_work.F90) - -foreach(ITEM IN LISTS CICE_F90) - e3sm_add_flags("cice/src/source/${ITEM}" "-O0") -endforeach() diff --git a/cime_config/machines/cmake_macros/amdclang_crusher.cmake b/cime_config/machines/cmake_macros/amdclang_crusher.cmake deleted file mode 100644 index 2df5074d11f..00000000000 --- a/cime_config/machines/cmake_macros/amdclang_crusher.cmake +++ /dev/null @@ -1,12 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") diff --git a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake deleted file mode 100644 index 6f3c0074798..00000000000 --- a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake +++ /dev/null @@ -1,26 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd") - -set(MPICXX "hipcc") -set(SCXX "hipcc") -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake deleted file mode 100644 index f8176ccb3d0..00000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-cpu.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -string(APPEND CPPDEFS " -DCPRCRAY") - - diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake deleted file mode 100644 index 18564d4d301..00000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream-gpu.cmake +++ /dev/null @@ -1,28 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() - -set(PIO_FILESYSTEM_HINTS "gpfs") - -set(MPICXX "hipcc") -set(SCXX "hipcc") - -string(APPEND CMAKE_CXX_FLAGS " -I${MPICH_DIR}/include") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L${MPICH_DIR}/lib -lmpi -L/opt/cray/pe/mpich/8.1.16/gtl/lib -lmpi_gtl_hsa") - -# For YAKL's -lroctx64 -lrocfft; the rocm module doesn't set this. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib") - -# 'NOT DEBUG': this resolves a crash in mct in docn init -# 'DEBUG' casee, too: resolves a build error in elm/src/main/elm_varctl.F90 due to several OpenACC syntax errors -#if (NOT DEBUG) - string(APPEND CMAKE_C_FLAGS " -O2 -hnoacc -hfp0 -hipa0") - string(APPEND CMAKE_Fortran_FLAGS " -O2 -hnoacc -hfp0 -hipa0") -#endif() - -string(APPEND CPPDEFS " -DCPRCRAY") - -#set(SCREAM_MPI_ON_DEVICE OFF CACHE STRING "See SCREAM issue #2080.") diff --git a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake b/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake deleted file mode 100644 index 2a0bfd6217c..00000000000 --- a/cime_config/machines/cmake_macros/crayclang-scream_crusher-scream.cmake +++ /dev/null @@ -1,12 +0,0 @@ -if (compile_threaded) - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -if (COMP_NAME STREQUAL elm) - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") - -set(PIO_FILESYSTEM_HINTS "gpfs") diff --git a/cime_config/machines/cmake_macros/crayclang_crusher.cmake b/cime_config/machines/cmake_macros/crayclang_crusher.cmake deleted file mode 100644 index 7a5fb412cbb..00000000000 --- a/cime_config/machines/cmake_macros/crayclang_crusher.cmake +++ /dev/null @@ -1,10 +0,0 @@ -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero") -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -string(APPEND CMAKE_Fortran_FLAGS " -em -ef") diff --git a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake deleted file mode 100644 index 653938b81da..00000000000 --- a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake +++ /dev/null @@ -1,63 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() - -if (compile_threaded) - string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") - string(APPEND CMAKE_C_FLAGS " -fopenmp") - string(APPEND CMAKE_CXX_FLAGS " -fopenmp") - string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") -endif() -string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g") -string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g") -string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG") -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_R16 -DCPRCRAY") -string(APPEND CMAKE_Fortran_FLAGS " -f free -em") -if (NOT compile_threaded) - # -M1077 flag used to suppress message about OpenMP directives - # that are ignored for non-threaded builds. (-h omp inactive) - # Details: `explain ftn-1077` - string(APPEND CMAKE_Fortran_FLAGS " -M1077") -endif() -set(HAS_F2008_CONTIGUOUS "TRUE") - -# -Wl,--allow-shlib-undefined was added to address rocm 5.4.3 Fortran linker issue: -# /opt/rocm-5.4.3/lib/libhsa-runtime64.so.1: undefined reference to `std::condition_variable::wait(std::unique_lock&)@GLIBCXX_3.4.30' -# AMD started building with GCC 12.2.0, which brings in a GLIBCXX symbol that isn't in CCE's default GCC toolchain. -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefined") - -# Switch to O3 for better performance -# Using O2 to ensure passing tests -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") - -if (COMP_NAME STREQUAL elm) - # See Land NaNs in conditionals: https://github.com/E3SM-Project/E3SM/issues/4996 - string(APPEND CMAKE_Fortran_FLAGS " -hfp0") -endif() -# -em -ef generates modulename.mod (lowercase files) to support -# Scorpio installs -# Disable ipa and zero initialization are for other NaN isues: -# https://github.com/E3SM-Project/E3SM/pull/5208 -string(APPEND CMAKE_Fortran_FLAGS " -hipa0 -hzero -em -ef -hnoacc") - -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/cmake_macros/gnu_crusher.cmake b/cime_config/machines/cmake_macros/gnu_crusher.cmake deleted file mode 100644 index 9b242243989..00000000000 --- a/cime_config/machines/cmake_macros/gnu_crusher.cmake +++ /dev/null @@ -1,15 +0,0 @@ -set(MPICC "cc") -set(MPICXX "CC") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "CC") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") diff --git a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake b/cime_config/machines/cmake_macros/gnugpu_crusher.cmake deleted file mode 100644 index bb213014b2f..00000000000 --- a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake +++ /dev/null @@ -1,25 +0,0 @@ -set(MPICC "cc") -set(MPICXX "hipcc") -set(MPIFC "ftn") -set(SCC "cc") -set(SCXX "hipcc") -set(SFC "ftn") - -string(APPEND CPPDEFS " -DLINUX") -if (COMP_NAME STREQUAL gptl) - string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") -endif() -string(APPEND CMAKE_Fortran_FLAGS " -Wno-implicit-interface") - -string(APPEND CMAKE_C_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2") -string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - -string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Wl,--copy-dt-needed-entries -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ") - -string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On") - -set(USE_HIP "TRUE") -string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip") diff --git a/cime_config/machines/syslog.crusher b/cime_config/machines/syslog.crusher deleted file mode 100755 index 25ef50e1e95..00000000000 --- a/cime_config/machines/syslog.crusher +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/csh -f -# pm-gpu syslog script: -# mach_syslog - -set sample_interval = $1 -set jid = $2 -set lid = $3 -set run = $4 -set timing = $5 -set dir = $6 - -# Wait until job task-to-node mapping information is output before saving output file. -# Target length was determined empirically (maximum number of lines before job mapping -# information starts + number of nodes), and it may need to be adjusted in the future. -# (Note that calling script 'touch'es the e3sm log file before spawning this script, so that 'wc' does not fail.) -set nnodes = `scontrol show jobid $jid | grep -F NumNodes | sed 's/ *NumNodes=\([0-9]*\) .*/\1/' ` -@ target_lines = 150 + $nnodes -sleep 10 -set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -while ($outlth < $target_lines) - sleep 60 - set outlth = `wc \-l $run/e3sm.log.$lid | sed 's/ *\([0-9]*\) *.*/\1/' ` -end - -set TimeLimit = `scontrol show jobid $jid | grep -F TimeLimit | sed 's/^ *RunTime=.*TimeLimit=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set limit_hours = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set limit_mins = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set limit_secs = `echo $TimeLimit | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$limit_hours" == "X") set limit_hours = 0 -if ("X$limit_mins" == "X") set limit_mins = 0 -if ("X$limit_secs" == "X") set limit_secs = 0 -@ limit = 3600 * $limit_hours + 60 * $limit_mins + $limit_secs - -set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` -set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` -set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` -set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` -if ("X$runt_hours" == "X") set runt_hours = 0 -if ("X$runt_mins" == "X") set runt_mins = 0 -if ("X$runt_secs" == "X") set runt_secs = 0 -@ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - -@ remaining = $limit - $runt -cat > $run/Walltime.Remaining < $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining -endif - -while ($remaining > 0) - echo "Wallclock time remaining: $remaining" >> $dir/atm.log.$lid.step - grep -Fa -e "nstep" -e "model date" $run/*atm.log.$lid | tail -n 4 >> $dir/atm.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/lnd.log.$lid.step - grep -Fa -e "timestep" -e "model date" $run/*lnd.log.$lid | tail -n 4 >> $dir/lnd.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ocn.log.$lid.step - grep -Fa -e "timestep" -e "Step number" -e "model date" $run/*ocn.log.$lid | tail -n 4 >> $dir/ocn.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/ice.log.$lid.step - grep -Fa -e "timestep" -e "istep" -e "model date" $run/*ice.log.$lid | tail -n 4 >> $dir/ice.log.$lid.step - echo "Wallclock time remaining: $remaining" >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*rof.log.$lid | tail -n 4 >> $dir/rof.log.$lid.step - grep -Fa "model date" $run/*cpl.log.$lid > $dir/cpl.log.$lid.step-all - echo "Wallclock time remaining: $remaining" >> $dir/cpl.log.$lid.step - tail -n 4 $dir/cpl.log.$lid.step-all >> $dir/cpl.log.$lid.step - /bin/cp --preserve=timestamps -u $timing/* $dir - # sqs -w -a | grep "^[0-9]* *R *"> $dir/sqswr.$lid.$remaining - squeue -t R -o "%.10i %.15P %.20j %.10u %.7a %.2t %.6D %.8C %.10M %.10l" > $dir/squeuef.$lid.$remaining - squeue -s | grep -v -F extern > $dir/squeues.$lid.$remaining - # squeue -t R -o "%.10i %R" > $dir/squeueR.$lid.$remaining - chmod a+r $dir/* - # sleep $sample_interval - set sleep_remaining = $sample_interval - while ($sleep_remaining > 120) - sleep 120 - @ sleep_remaining = $sleep_remaining - 120 - end - sleep $sleep_remaining - set RunTime = `scontrol show jobid $jid | grep -F RunTime | sed 's/^ *RunTime=\([0-9]*:[0-9]*:[0-9]*\) .*/\1/' ` - set runt_hours = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\1/' ` - set runt_mins = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\2/' ` - set runt_secs = `echo $RunTime | sed 's/^0*\([0-9]*\):0*\([0-9]*\):0*\([0-9]*\)/\3/' ` - if ("X$runt_hours" == "X") set runt_hours = 0 - if ("X$runt_mins" == "X") set runt_mins = 0 - if ("X$runt_secs" == "X") set runt_secs = 0 - @ runt = 3600 * $runt_hours + 60 * $runt_mins + $runt_secs - @ remaining = $limit - $runt - cat > $run/Walltime.Remaining << EOF2 -$remaining $sample_interval -EOF2 - -end From ea640364780427fe3f6eac0e5bf6a8c1f58a0459 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 22 Aug 2024 14:32:05 -0400 Subject: [PATCH 3/5] Successful build and run of ne30pg2_ne30pg2 with PrgEnv-gnu and rocm/6.2.0 on Frontier. --- .../machines/cmake_macros/craygnu-hipcc.cmake | 37 +++++++++++++++++++ cime_config/machines/config_machines.xml | 30 +++++++-------- components/eamxx/CMakeLists.txt | 4 +- .../machine-files/frontier-scream-gpu.cmake | 2 +- 4 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 cime_config/machines/cmake_macros/craygnu-hipcc.cmake diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake new file mode 100644 index 00000000000..18f5300fded --- /dev/null +++ b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake @@ -0,0 +1,37 @@ +set(MPICC "cc") +set(MPICXX "hipcc") # Needs MPICH_CXX to use hipcc +set(MPIFC "ftn") # Linker needs to be the Cray wrapper ftn, not mpif90 +set(SCC "cc") +set(SCXX "hipcc") +set(SFC "ftn") + +string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") +if (COMP_NAME STREQUAL gptl) + string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") +endif() +string(APPEND CMAKE_Fortran_FLAGS " -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fallow-argument-mismatch") + +string(APPEND CMAKE_C_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=invalid,zero,overflow") +string(APPEND CMAKE_Fortran_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace -fcheck=bounds -ffpe-trap=zero,overflow") +string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g -Wall -fbacktrace") + +string(APPEND CMAKE_C_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_CXX_FLAGS_RELEASE " -g -O2") +string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -g -O2") + +if (COMP_NAME STREQUAL csm_share) + string(APPEND CMAKE_C_FLAGS " -std=c99") +endif() +string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") +string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") + +string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") + +set(E3SM_LINK_WITH_FORTRAN "TRUE") +string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") + +string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") + +set(USE_HIP "TRUE") +string(APPEND CMAKE_HIP_FLAGS "$ENV{CXXFLAGS} --offload-arch=gfx90a -munsafe-fp-atomics") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 84cbaacd550..88fad58bf3e 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1098,7 +1098,7 @@ Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - crayclang-scream + craygnu-hipcc mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1135,23 +1135,21 @@ module module /usr/share/lmod/lmod/libexec/lmod python - + - cpe/22.12 + + PrgEnv-gnu + cpe/24.07 craype-accel-amd-gfx90a - rocm/5.4.0 - libunwind/1.6.2 - cce/15.0.1 - libfabric/1.15.2.0 - craype/2.7.20 - cray-mpich/8.1.26 - cray-python/3.9.13.1 - subversion/1.14.1 - git/2.36.1 - cmake/3.21.3 - cray-hdf5-parallel/1.12.2.1 - cray-netcdf-hdf5parallel/4.9.0.1 - cray-parallel-netcdf/1.12.3.1 + rocm/6.2.0 + libunwind + cray-python + subversion + git + cmake + cray-hdf5-parallel + cray-netcdf-hdf5parallel + cray-parallel-netcdf darshan-runtime diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index 398efa3bc2d..efaf77514d9 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -550,9 +550,9 @@ DisableMpiCxxBindings() if (SCREAM_DOUBLE_PRECISION) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") set(SCREAM_Fortran_FLAGS -real-size 64) - elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") + #elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") #find what is crayclang id in cmake - set(SCREAM_Fortran_FLAGS -s default32 -eZ) + #set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake index 14d1d501160..8704d848334 100644 --- a/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake +++ b/components/eamxx/cmake/machine-files/frontier-scream-gpu.cmake @@ -6,4 +6,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/hip.cmake) set(SCREAM_MPIRUN_EXE "srun" CACHE STRING "") set(SCREAM_MACHINE "frontier-scream-gpu" CACHE STRING "") -set(CMAKE_CXX_FLAGS "--amdgpu-target=gfx90a -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) +set(CMAKE_CXX_FLAGS "-Wno-mismatched-tags --offload-arch=gfx90a -munsafe-fp-atomics -fno-gpu-rdc -I$ENV{MPICH_DIR}/include" CACHE STRING "" FORCE) From dba1054f184ea62fbfe5eb07889e3ffd38a17c82 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Mon, 26 Aug 2024 10:53:31 -0400 Subject: [PATCH 4/5] Changed Frontier build to use OpenMP and Adios2. --- cime_config/machines/cmake_macros/craygnu-hipcc.cmake | 9 +++++++-- cime_config/machines/config_machines.xml | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake index 18f5300fded..8322c5d3a94 100644 --- a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake +++ b/cime_config/machines/cmake_macros/craygnu-hipcc.cmake @@ -25,12 +25,17 @@ endif() string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -ffixed-form") string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -ffree-form") -string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF") - set(E3SM_LINK_WITH_FORTRAN "TRUE") string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include") string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{ROCM_PATH}/lib -lamdhip64") +if (compile_threaded) + string(APPEND CMAKE_C_FLAGS " -fopenmp") + string(APPEND CMAKE_Fortran_FLAGS " -fopenmp") + string(APPEND CMAKE_CXX_FLAGS " -fopenmp=libgomp") + string(APPEND CMAKE_EXE_LINKER_FLAGS " -fopenmp") +endif() + string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On -DKokkos_ENABLE_OPENMP=Off") set(USE_HIP "TRUE") diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 88fad58bf3e..2448e2f797f 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1176,8 +1176,8 @@ threads - - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} From cc81b1f6565ef058eed6340a365e9eadc4d2a759 Mon Sep 17 00:00:00 2001 From: "White, Trey" Date: Thu, 29 Aug 2024 20:21:38 -0400 Subject: [PATCH 5/5] Add SCREAM_SYSTEM_WORKAROUND_P3_PART2. Return support for crayclang-scream. Rename craygnuamdgpu. --- ...raygnu-hipcc.cmake => craygnuamdgpu.cmake} | 2 +- cime_config/machines/config_machines.xml | 29 +++++++++++++++++-- components/eamxx/CMakeLists.txt | 5 ++-- .../p3/disp/p3_main_impl_part2_disp.cpp | 7 ++++- 4 files changed, 35 insertions(+), 8 deletions(-) rename cime_config/machines/cmake_macros/{craygnu-hipcc.cmake => craygnuamdgpu.cmake} (97%) diff --git a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake similarity index 97% rename from cime_config/machines/cmake_macros/craygnu-hipcc.cmake rename to cime_config/machines/cmake_macros/craygnuamdgpu.cmake index 8322c5d3a94..6cb79c0146b 100644 --- a/cime_config/machines/cmake_macros/craygnu-hipcc.cmake +++ b/cime_config/machines/cmake_macros/craygnuamdgpu.cmake @@ -5,7 +5,7 @@ set(SCC "cc") set(SCXX "hipcc") set(SFC "ftn") -string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU") +string(APPEND CPPDEFS " -DLINUX -DFORTRANUNDERSCORE -DNO_R16 -DCPRGNU -DSCREAM_SYSTEM_WORKAROUND_P3_PART2") if (COMP_NAME STREQUAL gptl) string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY") endif() diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 2448e2f797f..e2d4a50cab5 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1098,7 +1098,7 @@ Frontier. AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs. .*frontier.* CNL - craygnu-hipcc + craygnuamdgpu,crayclang-scream mpich cli115 /lustre/orion/proj-shared/cli115 @@ -1135,7 +1135,7 @@ module module /usr/share/lmod/lmod/libexec/lmod python - + PrgEnv-gnu @@ -1152,6 +1152,25 @@ cray-parallel-netcdf darshan-runtime + + + cpe/22.12 + craype-accel-amd-gfx90a + rocm/5.4.0 + libunwind/1.6.2 + cce/15.0.1 + libfabric/1.15.2.0 + craype/2.7.20 + cray-mpich/8.1.26 + cray-python/3.9.13.1 + subversion/1.14.1 + git/2.36.1 + cmake/3.21.3 + cray-hdf5-parallel/1.12.2.1 + cray-netcdf-hdf5parallel/4.9.0.1 + cray-parallel-netcdf/1.12.3.1 + darshan-runtime + $CIME_OUTPUT_ROOT/$CASE/run @@ -1176,9 +1195,13 @@ threads - + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /sw/frontier/spack-envs/cpe24.07-gpu/opt/gcc-13.2/adios2-2.10.0-cyq7kvlfq5nixmw5unih2gjdkhvitava; else echo "$ADIOS2_ROOT"; fi} + + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /lustre/orion/cli115/world-shared/frontier/3rdparty/adios2/2.9.1/cray-mpich-8.1.26/crayclang-scream-14.0.0; else echo "$ADIOS2_ROOT"; fi} + + diff --git a/components/eamxx/CMakeLists.txt b/components/eamxx/CMakeLists.txt index efaf77514d9..4c80aa55b4f 100644 --- a/components/eamxx/CMakeLists.txt +++ b/components/eamxx/CMakeLists.txt @@ -550,9 +550,8 @@ DisableMpiCxxBindings() if (SCREAM_DOUBLE_PRECISION) if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") set(SCREAM_Fortran_FLAGS -real-size 64) - #elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" OR "${CMAKE_CXX_COMPILER_ID}" MATCHES "HIPCC") - #find what is crayclang id in cmake - #set(SCREAM_Fortran_FLAGS -s default32 -eZ) + elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Cray") + set(SCREAM_Fortran_FLAGS -s default32 -eZ) else() set(SCREAM_Fortran_FLAGS -fdefault-real-8 -fdefault-double-8) endif() diff --git a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp index 2b619d54bf3..7dddaf343e8 100644 --- a/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp +++ b/components/eamxx/src/physics/p3/disp/p3_main_impl_part2_disp.cpp @@ -10,6 +10,9 @@ namespace p3 { * this file, #include p3_functions.hpp instead. */ +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize off +#endif template <> void Functions ::p3_main_part2_disp( @@ -130,7 +133,9 @@ ::p3_main_part2_disp( if (!hydrometeorsPresent(i)) return; }); } - +#ifdef SCREAM_SYSTEM_WORKAROUND_P3_PART2 +#pragma clang optimize on +#endif } // namespace p3 } // namespace scream