From 04b755de79bca03ca16473f9d0d7a23ce24b09aa Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Mon, 4 Mar 2024 16:54:17 -0800 Subject: [PATCH 1/2] This update reflects the new directory structure on LC for storing input. The switch from climdat to e3sm in the dir name. --- cime_config/machines/config_machines.xml | 97 ++----------------- .../eamxx/cmake/machine-files/lassen.cmake | 9 -- .../cmake/machine-files/quartz-intel.cmake | 2 +- .../eamxx/cmake/machine-files/quartz.cmake | 2 +- .../cmake/machine-files/ruby-intel.cmake | 2 +- .../eamxx/cmake/machine-files/ruby.cmake | 2 +- .../eamxx/cmake/machine-files/syrah.cmake | 13 --- .../eamxx/data/scream_default_remap.yaml | 2 +- components/eamxx/scripts/machines_specs.py | 8 +- 9 files changed, 19 insertions(+), 118 deletions(-) delete mode 100644 components/eamxx/cmake/machine-files/lassen.cmake delete mode 100644 components/eamxx/cmake/machine-files/syrah.cmake diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index f3aa5cfa6149..af4b689a90d2 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2897,83 +2897,6 @@ - - LLNL Linux Cluster, Linux, 4 V100 GPUs/node, 44 IBM P9 cpu cores/node - lassen.* - LINUX - gnugpu - spectrum-mpi - cbronze - /usr/workspace/$USER/e3sm_scratch - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 - /usr/workspace/$USER/archive/$CASE - /usr/gdata/climdat/baselines/$COMPILER - 16 - lsf - donahue5 -at- llnl.gov - 40 - 40 - - - - - jsrun - - -X 1 - $SHELL{if [ {{ total_tasks }} -eq 1 ];then echo --nrs 1 --rs_per_host 1;else echo --nrs $NUM_RS --rs_per_host $RS_PER_NODE;fi} - --tasks_per_rs $SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} - -d plane:$SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} - --cpu_per_rs $ENV{CPU_PER_RS} - --gpu_per_rs $ENV{GPU_PER_RS} - --bind packed:smt:$ENV{OMP_NUM_THREADS} - --latency_priority $ENV{LTC_PRT} - --stdio_mode prepended - $ENV{JSRUN_THREAD_VARS} - $ENV{SMPIARGS} - - - - /usr/share/lmod/lmod/init/env_modules_python.py - /usr/share/lmod/lmod/init/perl - /usr/share/lmod/lmod/init/sh - /usr/share/lmod/lmod/init/csh - module - module - /usr/share/lmod/lmod/libexec/lmod python - /usr/share/lmod/lmod/libexec/lmod perl - - - git - gcc/8.3.1 - cuda/11.8.0 - cmake/3.16.8 - spectrum-mpi - python/3.7.2 - - - /p/gpfs1/$USER/e3sm_scratch/$CASE/run - $CIME_OUTPUT_ROOT/$CASE/bld - - - - - -E OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -E OMP_PROC_BIND=spread -E OMP_PLACES=threads -E OMP_STACKSIZE=256M - - - y - /usr/gdata/climdat/netcdf/bin:$ENV{PATH} - /usr/gdata/climdat/netcdf/lib:$ENV{LD_LIBRARY_PATH} - /usr/gdata/climdat/netcdf - 2 - 20 - 2 - gpu-cpu - $SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} - --smpiargs="-gpu" - - - LLNL Linux Cluster, Linux (pgi), 56 pes/node, batch system is Slurm LINUX @@ -2981,11 +2904,11 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/ruby - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/gdata/e3sm/ccsm3data/inputdata + /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/climdat/tools/cprnc + /usr/gdata/e3sm/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov @@ -3013,7 +2936,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/climdat/install/quartz/modulefiles + /usr/gdata/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3024,7 +2947,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/climdat/install/quartz/netcdf-fortran/ + /usr/gdata/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 @@ -3036,11 +2959,11 @@ mpich cbronze /p/lustre2/$USER/e3sm_scratch/quartz - /usr/gdata/climdat/ccsm3data/inputdata - /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 + /usr/gdata/e3sm/ccsm3data/inputdata + /usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER - /usr/gdata/climdat/tools/cprnc + /usr/gdata/e3sm/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov @@ -3068,7 +2991,7 @@ intel-classic/2021.6.0-magic mvapich2/2.3.7 cmake/3.19.2 - /usr/gdata/climdat/install/quartz/modulefiles + /usr/gdata/e3sm/install/quartz/modulefiles hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 @@ -3079,7 +3002,7 @@ $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld - /usr/gdata/climdat/install/quartz/netcdf-fortran/ + /usr/gdata/e3sm/install/quartz/netcdf-fortran/ /usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0 diff --git a/components/eamxx/cmake/machine-files/lassen.cmake b/components/eamxx/cmake/machine-files/lassen.cmake deleted file mode 100644 index 36b69c7f0253..000000000000 --- a/components/eamxx/cmake/machine-files/lassen.cmake +++ /dev/null @@ -1,9 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -set(NetCDF_PATH /usr/gdata/climdat/netcdf CACHE STRING "") -set(NetCDF_Fortran_PATH /usr/gdata/climdat/netcdf CACHE STRING "") -set(LAPACK_LIBRARIES /usr/lib64/liblapack.so CACHE STRING "") -set(CMAKE_CXX_FLAGS "-DTHRUST_IGNORE_CUB_VERSION_CHECK" CACHE STRING "" FORCE) - -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata/" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/quartz-intel.cmake b/components/eamxx/cmake/machine-files/quartz-intel.cmake index 753c782702db..defd8cbb2d33 100644 --- a/components/eamxx/cmake/machine-files/quartz-intel.cmake +++ b/components/eamxx/cmake/machine-files/quartz-intel.cmake @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) -execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) +execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/quartz.cmake b/components/eamxx/cmake/machine-files/quartz.cmake index ee9a3dcbffd3..e4b4fcbd8a57 100644 --- a/components/eamxx/cmake/machine-files/quartz.cmake +++ b/components/eamxx/cmake/machine-files/quartz.cmake @@ -16,4 +16,4 @@ elseif ("${COMPILER}" STREQUAL "gnu") set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/" CACHE STRING "" FORCE) endif() -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "") +set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/ruby-intel.cmake b/components/eamxx/cmake/machine-files/ruby-intel.cmake index 63fff478fdaf..9c6318da4952 100644 --- a/components/eamxx/cmake/machine-files/ruby-intel.cmake +++ b/components/eamxx/cmake/machine-files/ruby-intel.cmake @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE) option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON) set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE) -execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate) +execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate) diff --git a/components/eamxx/cmake/machine-files/ruby.cmake b/components/eamxx/cmake/machine-files/ruby.cmake index d0a9de4baf4b..77d6e6618c71 100644 --- a/components/eamxx/cmake/machine-files/ruby.cmake +++ b/components/eamxx/cmake/machine-files/ruby.cmake @@ -12,4 +12,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "") +set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "") diff --git a/components/eamxx/cmake/machine-files/syrah.cmake b/components/eamxx/cmake/machine-files/syrah.cmake deleted file mode 100644 index f03fc1e9d469..000000000000 --- a/components/eamxx/cmake/machine-files/syrah.cmake +++ /dev/null @@ -1,13 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) -common_setup() - -include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake) -include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake) - -# Enable Sandy Bridge arch in Kokkos -option(Kokkos_ARCH_SNB "" ON) - -set(CMAKE_CXX_FLAGS "-w -cxxlib=/usr/tce/packages/gcc/gcc-8.3.1/rh" CACHE STRING "" FORCE) -set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/ -mkl" CACHE STRING "" FORCE) - -set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata/" CACHE STRING "") diff --git a/components/eamxx/data/scream_default_remap.yaml b/components/eamxx/data/scream_default_remap.yaml index 8bf47386c76d..3be4dc13244c 100644 --- a/components/eamxx/data/scream_default_remap.yaml +++ b/components/eamxx/data/scream_default_remap.yaml @@ -4,7 +4,7 @@ filename_prefix: ${CASE}.scream.arm_sites.hi Averaging Type: Instant Max Snapshots Per File: 744 # One output every 31 days #remap_file: /g/g17/donahue5/Code/e3sm/scream-docs/regional_output_sites/20221123_ARM_sites_map.nc -remap_file: /usr/gdata/climdat/ccsm3data/inputdata/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc +remap_file: /usr/gdata/e3sm/ccsm3data/inputdata/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc Fields: Physics ${PHYSICS_GRID_TYPE}: Field Names: diff --git a/components/eamxx/scripts/machines_specs.py b/components/eamxx/scripts/machines_specs.py index cd717cba6b97..4b2d61745082 100644 --- a/components/eamxx/scripts/machines_specs.py +++ b/components/eamxx/scripts/machines_specs.py @@ -31,17 +31,17 @@ "", "/sems-data-store/ACME/baselines/scream/master-baselines"), "lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'", - "export PATH=/usr/gdata/climdat/netcdf/bin:$PATH", - "export LD_LIBRARY_PATH=/usr/gdata/climdat/netcdf/lib:$LD_LIBRARY_PATH", + "export PATH=/usr/gdata/e3sm/netcdf/bin:$PATH", + "export LD_LIBRARY_PATH=/usr/gdata/e3sm/netcdf/lib:$LD_LIBRARY_PATH", ], ["mpicxx","mpifort","mpicc"], "bsub -Ip -qpdebug", ""), - "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "ruby-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), - "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], + "quartz-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"], ["mpicxx","mpifort","mpicc"], "salloc --partition=pdebug", ""), From cf7f4bee951b06dc51e47c431ee4fa6639347513 Mon Sep 17 00:00:00 2001 From: Aaron Donahue Date: Mon, 15 Apr 2024 16:53:14 -0700 Subject: [PATCH 2/2] add missing subversion module to machine configs for quartz and ruby --- cime_config/machines/config_machines.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index af4b689a90d2..c5b3d0d562d7 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -2942,6 +2942,7 @@ netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 screamML-venv/0.0.1 + subversion $CIME_OUTPUT_ROOT/$CASE/run @@ -2997,6 +2998,7 @@ netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 screamML-venv/0.0.1 + subversion $CIME_OUTPUT_ROOT/$CASE/run