Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

This update reflects the new directory structure on LC for storing input #6281

Merged
merged 2 commits into from
Apr 25, 2024
+21 −118
Merged

This update reflects the new directory structure on LC for storing input #6281

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
04b755d
This update reflects the new directory structure on LC for storing in…
AaronDonahue Mar 5, 2024
cf7f4be
add missing subversion module to machine configs for quartz and ruby
AaronDonahue Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 10 additions & 87 deletions cime_config/machines/config_machines.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2897,95 +2897,18 @@
</environment_variables>
</machine>

<machine MACH="lassen">
<DESC>LLNL Linux Cluster, Linux, 4 V100 GPUs/node, 44 IBM P9 cpu cores/node</DESC>
<NODENAME_REGEX>lassen.*</NODENAME_REGEX>
<OS>LINUX</OS>
<COMPILERS>gnugpu</COMPILERS>
<MPILIBS>spectrum-mpi</MPILIBS>
<PROJECT>cbronze</PROJECT>
<CIME_OUTPUT_ROOT>/usr/workspace/$USER/e3sm_scratch</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/usr/gdata/climdat/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/usr/gdata/climdat/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/usr/workspace/$USER/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/usr/gdata/climdat/baselines/$COMPILER</BASELINE_ROOT>
<GMAKE_J>16</GMAKE_J>
<BATCH_SYSTEM>lsf</BATCH_SYSTEM>
<SUPPORTED_BY>donahue5 -at- llnl.gov</SUPPORTED_BY>
<MAX_TASKS_PER_NODE>40</MAX_TASKS_PER_NODE>
<MAX_MPITASKS_PER_NODE>40</MAX_MPITASKS_PER_NODE>
<mpirun mpilib="mpi-serial">
<executable/>
</mpirun>
<mpirun mpilib="spectrum-mpi">
<executable>jsrun</executable>
<arguments>
<arg name="exit_on_error">-X 1</arg>
<arg name="num_rs">$SHELL{if [ {{ total_tasks }} -eq 1 ];then echo --nrs 1 --rs_per_host 1;else echo --nrs $NUM_RS --rs_per_host $RS_PER_NODE;fi}</arg>
<arg name="tasks_per_rs">--tasks_per_rs $SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc}</arg>
<arg name="distribute">-d plane:$SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc}</arg>
<arg name="cpu_per_rs">--cpu_per_rs $ENV{CPU_PER_RS}</arg>
<arg name="gpu_per_rs">--gpu_per_rs $ENV{GPU_PER_RS}</arg>
<arg name="task_bind">--bind packed:smt:$ENV{OMP_NUM_THREADS}</arg>
<arg name="latency_priority">--latency_priority $ENV{LTC_PRT}</arg>
<arg name="stdio_mode">--stdio_mode prepended</arg>
<arg name="thread_vars">$ENV{JSRUN_THREAD_VARS}</arg>
<arg name="smpiargs">$ENV{SMPIARGS}</arg>
</arguments>
</mpirun>
<module_system type="module">
<init_path lang="python">/usr/share/lmod/lmod/init/env_modules_python.py</init_path>
<init_path lang="perl">/usr/share/lmod/lmod/init/perl</init_path>
<init_path lang="sh">/usr/share/lmod/lmod/init/sh</init_path>
<init_path lang="csh">/usr/share/lmod/lmod/init/csh</init_path>
<cmd_path lang="csh">module</cmd_path>
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="python">/usr/share/lmod/lmod/libexec/lmod python</cmd_path>
<cmd_path lang="perl">/usr/share/lmod/lmod/libexec/lmod perl</cmd_path>
<modules compiler="gnugpu">
<command name="--force purge"/>
<command name="load">git</command>
<command name="load">gcc/8.3.1</command>
<command name="load">cuda/11.8.0</command>
<command name="load">cmake/3.16.8</command>
<command name="load">spectrum-mpi</command>
<command name="load">python/3.7.2</command>
</modules>
</module_system>
<RUNDIR>/p/gpfs1/$USER/e3sm_scratch/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables BUILD_THREADED="FALSE">
<env name="JSRUN_THREAD_VARS"> </env>
</environment_variables>
<environment_variables BUILD_THREADED="TRUE">
<env name="JSRUN_THREAD_VARS">-E OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -E OMP_PROC_BIND=spread -E OMP_PLACES=threads -E OMP_STACKSIZE=256M</env>
</environment_variables>
<environment_variables compiler="gnugpu">
<env name="LLNL_USE_OMPI_VARS">y</env>
<env name="PATH">/usr/gdata/climdat/netcdf/bin:$ENV{PATH}</env>
<env name="LD_LIBRARY_PATH">/usr/gdata/climdat/netcdf/lib:$ENV{LD_LIBRARY_PATH}</env>
<env name="NETCDF_PATH">/usr/gdata/climdat/netcdf</env>
<env name="RS_PER_NODE">2</env>
<env name="CPU_PER_RS">20</env>
<env name="GPU_PER_RS">2</env>
<env name="LTC_PRT">gpu-cpu</env>
<env name="NUM_RS">$SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc}</env>
<env name="SMPIARGS">--smpiargs="-gpu"</env>
</environment_variables>
</machine>

<machine MACH="ruby">
<DESC>LLNL Linux Cluster, Linux (pgi), 56 pes/node, batch system is Slurm</DESC>
<OS>LINUX</OS>
<COMPILERS>intel</COMPILERS>
<MPILIBS>mpich</MPILIBS>
<PROJECT>cbronze</PROJECT>
<CIME_OUTPUT_ROOT>/p/lustre2/$USER/e3sm_scratch/ruby</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/usr/gdata/climdat/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/usr/gdata/climdat/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DIN_LOC_ROOT>/usr/gdata/e3sm/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/usr/gdata/climdat/tools/cprnc</CCSM_CPRNC>
<CCSM_CPRNC>/usr/gdata/e3sm/tools/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>lc_slurm</BATCH_SYSTEM>
<SUPPORTED_BY>donahue5 -at- llnl.gov</SUPPORTED_BY>
Expand Down Expand Up @@ -3013,7 +2936,7 @@
<command name="load">intel-classic/2021.6.0-magic</command>
<command name="load">mvapich2/2.3.7</command>
<command name="load">cmake/3.19.2</command>
<command name="use --append">/usr/gdata/climdat/install/quartz/modulefiles</command>
<command name="use --append">/usr/gdata/e3sm/install/quartz/modulefiles</command>
<command name="load">hdf5/1.12.2</command>
<command name="load">netcdf-c/4.9.0</command>
<command name="load">netcdf-fortran/4.6.0</command>
Expand All @@ -3024,7 +2947,7 @@
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables compiler="intel">
<env name="NETCDF_PATH">/usr/gdata/climdat/install/quartz/netcdf-fortran/</env>
<env name="NETCDF_PATH">/usr/gdata/e3sm/install/quartz/netcdf-fortran/</env>
<env name="PNETCDF_PATH">/usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0</env>
</environment_variables>
</machine>
Expand All @@ -3036,11 +2959,11 @@
<MPILIBS>mpich</MPILIBS>
<PROJECT>cbronze</PROJECT>
<CIME_OUTPUT_ROOT>/p/lustre2/$USER/e3sm_scratch/quartz</CIME_OUTPUT_ROOT>
<DIN_LOC_ROOT>/usr/gdata/climdat/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/usr/gdata/climdat/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DIN_LOC_ROOT>/usr/gdata/e3sm/ccsm3data/inputdata</DIN_LOC_ROOT>
<DIN_LOC_ROOT_CLMFORC>/usr/gdata/e3sm/ccsm3data/inputdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
<DOUT_S_ROOT>/p/lustre2/$USER/archive/$CASE</DOUT_S_ROOT>
<BASELINE_ROOT>/p/lustre2/$USER/ccsm_baselines/$COMPILER</BASELINE_ROOT>
<CCSM_CPRNC>/usr/gdata/climdat/tools/cprnc</CCSM_CPRNC>
<CCSM_CPRNC>/usr/gdata/e3sm/tools/cprnc</CCSM_CPRNC>
<GMAKE_J>8</GMAKE_J>
<BATCH_SYSTEM>lc_slurm</BATCH_SYSTEM>
<SUPPORTED_BY>donahue5 -at- llnl.gov</SUPPORTED_BY>
Expand Down Expand Up @@ -3068,7 +2991,7 @@
<command name="load">intel-classic/2021.6.0-magic</command>
<command name="load">mvapich2/2.3.7</command>
<command name="load">cmake/3.19.2</command>
<command name="use --append">/usr/gdata/climdat/install/quartz/modulefiles</command>
<command name="use --append">/usr/gdata/e3sm/install/quartz/modulefiles</command>
<command name="load">hdf5/1.12.2</command>
<command name="load">netcdf-c/4.9.0</command>
<command name="load">netcdf-fortran/4.6.0</command>
Expand All @@ -3079,7 +3002,7 @@
<RUNDIR>$CIME_OUTPUT_ROOT/$CASE/run</RUNDIR>
<EXEROOT>$CIME_OUTPUT_ROOT/$CASE/bld</EXEROOT>
<environment_variables compiler="intel">
<env name="NETCDF_PATH">/usr/gdata/climdat/install/quartz/netcdf-fortran/</env>
<env name="NETCDF_PATH">/usr/gdata/e3sm/install/quartz/netcdf-fortran/</env>
<env name="PNETCDF_PATH">/usr/tce/packages/parallel-netcdf/parallel-netcdf-1.12.3-mvapich2-2.3.7-intel-classic-2021.6.0</env>
</environment_variables>
</machine>
Expand Down
9 changes: 0 additions & 9 deletions components/eamxx/cmake/machine-files/lassen.cmake
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion components/eamxx/cmake/machine-files/quartz-intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE
set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE)
option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON)
set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE)
execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate)
execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate)
2 changes: 1 addition & 1 deletion components/eamxx/cmake/machine-files/quartz.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@ elseif ("${COMPILER}" STREQUAL "gnu")
set(CMAKE_EXE_LINKER_FLAGS "-L/usr/tce/packages/gcc/gcc-8.3.1/rh/lib/gcc/x86_64-redhat-linux/8/" CACHE STRING "" FORCE)
endif()

set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "")
set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "")
2 changes: 1 addition & 1 deletion components/eamxx/cmake/machine-files/ruby-intel.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ set(PYTHON_EXECUTABLE "/usr/tce/packages/python/python-3.9.12/bin/python3" CACHE
set(PYTHON_LIBRARIES "/usr/lib64/libpython3.9.so.1.0" CACHE STRING "" FORCE)
option (SCREAM_ENABLE_ML_CORRECTION "Whether to enable ML correction parametrization" ON)
set(HDF5_DISABLE_VERSION_CHECK 1 CACHE STRING "" FORCE)
execute_process(COMMAND source /usr/WS1/climdat/python_venv/3.9.2/screamML/bin/activate)
execute_process(COMMAND source /usr/WS1/e3sm/python_venv/3.9.2/screamML/bin/activate)
2 changes: 1 addition & 1 deletion components/eamxx/cmake/machine-files/ruby.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,4 @@ include (${EKAT_MACH_FILES_PATH}/kokkos/openmp.cmake)

include (${EKAT_MACH_FILES_PATH}/mpi/srun.cmake)

set(SCREAM_INPUT_ROOT "/usr/gdata/climdat/ccsm3data/inputdata" CACHE STRING "")
set(SCREAM_INPUT_ROOT "/usr/gdata/e3sm/ccsm3data/inputdata" CACHE STRING "")
13 changes: 0 additions & 13 deletions components/eamxx/cmake/machine-files/syrah.cmake
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion components/eamxx/data/scream_default_remap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ filename_prefix: ${CASE}.scream.arm_sites.hi
Averaging Type: Instant
Max Snapshots Per File: 744 # One output every 31 days
#remap_file: /g/g17/donahue5/Code/e3sm/scream-docs/regional_output_sites/20221123_ARM_sites_map.nc
remap_file: /usr/gdata/climdat/ccsm3data/inputdata/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc
remap_file: /usr/gdata/e3sm/ccsm3data/inputdata/atm/scream/maps/map_ne30np4_to_ne4pg2_mono.20220714.nc
Fields:
Physics ${PHYSICS_GRID_TYPE}:
Field Names:
Expand Down
8 changes: 4 additions & 4 deletions components/eamxx/scripts/machines_specs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,17 @@
"",
"/sems-data-store/ACME/baselines/scream/master-baselines"),
"lassen" : (["module --force purge", "module load git gcc/8.3.1 cuda/11.8.0 cmake/3.16.8 spectrum-mpi python/3.7.2", "export LLNL_USE_OMPI_VARS='y'",
"export PATH=/usr/gdata/climdat/netcdf/bin:$PATH",
"export LD_LIBRARY_PATH=/usr/gdata/climdat/netcdf/lib:$LD_LIBRARY_PATH",
"export PATH=/usr/gdata/e3sm/netcdf/bin:$PATH",
"export LD_LIBRARY_PATH=/usr/gdata/e3sm/netcdf/lib:$LD_LIBRARY_PATH",
],
["mpicxx","mpifort","mpicc"],
"bsub -Ip -qpdebug",
""),
"ruby-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"],
"ruby-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"],
["mpicxx","mpifort","mpicc"],
"salloc --partition=pdebug",
""),
"quartz-intel" : (["module --force purge", "module use --append /usr/gdata/climdat/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"],
"quartz-intel" : (["module --force purge", "module use --append /usr/gdata/e3sm/install/quartz/modulefiles", "module load StdEnv cmake/3.19.2 mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.7 hdf5/1.12.2 netcdf-c/4.9.0 netcdf-fortran/4.6.0 parallel-netcdf/1.12.3 python/3.9.12 screamML-venv/0.0.1"],
["mpicxx","mpifort","mpicc"],
"salloc --partition=pdebug",
""),
Expand Down