Skip to content

Commit

Permalink
Merge pull request #3 from beharrop/cre_update_v1
Browse files Browse the repository at this point in the history
Rebase code to maint-1.0 branch with updated machine files
  • Loading branch information
beharrop authored Oct 17, 2023
2 parents ddf15ee + 0fca3e2 commit 41aaaf6
Show file tree
Hide file tree
Showing 37 changed files with 1,338 additions and 373 deletions.
8 changes: 8 additions & 0 deletions .circleci/checkout.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#!/bin/bash

# Replace all ssh URLs to submodules with HTTP URLs
sed -i 's/[email protected]:/https:\/\/github.com\//' .gitmodules
git submodule update --init
cd components/mpas-source
sed -i 's/[email protected]:/https:\/\/github.com\//' .gitmodules
git submodule update --init --recursive
13 changes: 13 additions & 0 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
version: 2.1
jobs:
build:
machine:
image: ubuntu-2004:202010-01
steps:
- checkout
- run: bash .circleci/setup.sh
- run: bash .circleci/checkout.sh
- run: wget https://dabdceba-6d04-11e5-ba46-22000b92c6ec.e.globus.org/containers/public/e3sm.sif
- run:
command: singularity exec --hostname singularity e3sm.sif .circleci/run.sh
no_output_timeout: 60m
70 changes: 70 additions & 0 deletions .circleci/run.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#!/bin/bash

case="master.A_WCYCL1850.ne4_oQU240.baseline"
compset="A_WCYCL1850"
res="ne4_oQU240"

log_dir="${HOME}/projects/e3sm/scratch/$case"

shopt -s extglob
shopt -s nullglob

function command_yellow
{
echo -e "\e[33m\$ $1\e[0m"
eval $1
return $?
}

function command_yellow_rc
{
command_yellow "$1"
rc=$?
if [ $rc -ne 0 ]; then
exit $rc
fi
}

function print_bldlog
{
bldlog=$1
bldlog_path=(${log_dir}/bld/$bldlog.bldlog.+([[:digit:]])-+([[:digit:]]))
if [ -n "${bldlog_path}" ]; then
command_yellow "cat ${bldlog_path}"
fi
}

function print_runlog
{
runlog=$1
runlog_path=(${log_dir}/run/$runlog.log.+([[:digit:]])-+([[:digit:]]))
if [ -n "${runlog_path}" ]; then
command_yellow "cat ${runlog_path}"
return 1
else
return 0
fi
}

command_yellow_rc "mkdir -p $HOME/projects/e3sm/cesm-inputdata"
command_yellow_rc "cd cime/scripts"
command_yellow_rc "./create_newcase --case $case --compset $compset --res $res"
command_yellow_rc "cd $case"
command_yellow_rc "./xmlchange GMAKE_J=2"
command_yellow_rc "./case.setup"
command_yellow "./case.build -v"
rc=$?
if [ $rc -ne 0 ]; then
print_bldlog "e3sm"
print_bldlog "csm_share"
print_bldlog "pio"
print_bldlog "mct"
print_bldlog "gptl"
exit $rc
fi

# Unfortunately, case.submit always return 0, even if it fails.
# To find out if it succeeded, we check if the run log has been gzipped.
#command_yellow "./case.submit"
#print_runlog "e3sm"
#exit $?
14 changes: 14 additions & 0 deletions .circleci/setup.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
#!/bin/bash

go version

# Install Singularity
export VERSION=3.5.3 && # adjust this as necessary \
wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz && \
tar -xzf singularity-${VERSION}.tar.gz && \
cd singularity
./mconfig && \
make -C ./builddir && \
sudo make -C ./builddir install

singularity --version
5 changes: 5 additions & 0 deletions cime/config/e3sm/allactive/config_compsets.xml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@
<lname>SSP585_CAM5%CMIP6_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>A_WCYCLSSP585_CMIP6_HR</alias>
<lname>SSP585_CAM5%CMIP6-HR_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV</lname>
</compset>

<compset>
<alias>A_WCYCL1950S_CMIP6_LR</alias>
<lname>1950_CAM5%CMIP6-LR_CLM45%SPBC_MPASCICE%SPUNUP_MPASO%SPUNUP_MOSART_SGLC_SWAV</lname>
Expand Down
165 changes: 165 additions & 0 deletions cime/config/e3sm/allactive/config_pesall.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4771,6 +4771,41 @@
</rootpe>
</pes>
</mach>
<mach name="chrysalis">
<pes compset="any" pesize="any">
<comment>default,4nodes*64tasks*1thread</comment>
<ntasks>
<ntasks_atm>256</ntasks_atm>
<ntasks_lnd>256</ntasks_lnd>
<ntasks_rof>256</ntasks_rof>
<ntasks_ice>256</ntasks_ice>
<ntasks_ocn>256</ntasks_ocn>
<ntasks_glc>256</ntasks_glc>
<ntasks_wav>256</ntasks_wav>
<ntasks_cpl>256</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="mira|cetus">
Expand Down Expand Up @@ -7550,6 +7585,35 @@
</rootpe>
</pes>
</mach>
<mach name="chrysalis">
<pes compset="any" pesize="any">
<comment>chrysalis: any compset on ne4 grid</comment>
<ntasks>
<ntasks_atm>96</ntasks_atm>
<ntasks_ice>96</ntasks_ice>
<ntasks_cpl>96</ntasks_cpl>
<ntasks_lnd>96</ntasks_lnd>
<ntasks_rof>96</ntasks_rof>
<ntasks_ocn>96</ntasks_ocn>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_ice>1</nthrds_ice>
<nthrds_cpl>1</nthrds_cpl>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ocn>1</nthrds_ocn>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_ice>0</rootpe_ice>
<rootpe_cpl>0</rootpe_cpl>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ocn>0</rootpe_ocn>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%ne30np4_l%ne30np4_oi%oEC60to30.*_r%r05_m%oEC60to30.*_g%null_w%null">
<mach name="anvil">
Expand Down Expand Up @@ -7835,6 +7899,107 @@
</rootpe>
</pes>
</mach>
<mach name="chrysalis">
<pes compset=".*CAM5.+CLM45.+MPASCICE.+MPASO.+MOSART.+SGLC.+SWAV" pesize="S">
<comment> -compset A_WCYCL* -res ne30_oEC* on 30 nodes: 7.4 sypd </comment>
<ntasks>
<ntasks_atm>1350</ntasks_atm>
<ntasks_lnd>128</ntasks_lnd>
<ntasks_rof>128</ntasks_rof>
<ntasks_ice>1280</ntasks_ice>
<ntasks_ocn>512</ntasks_ocn>
<ntasks_cpl>1408</ntasks_cpl>
<ntasks_glc>1</ntasks_glc>
<ntasks_wav>1</ntasks_wav>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>1280</rootpe_lnd>
<rootpe_rof>1280</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>1408</rootpe_ocn>
<rootpe_cpl>0</rootpe_cpl>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
</rootpe>
</pes>
<pes compset=".*CAM5.+CLM45.+MPASCICE.+MPASO.+MOSART.+SGLC.+SWAV" pesize="M">
<comment> -compset A_WCYCL* -res ne30_oEC* on 60 nodes: 13.3 sypd </comment>
<ntasks>
<ntasks_atm>2700</ntasks_atm>
<ntasks_lnd>192</ntasks_lnd>
<ntasks_rof>192</ntasks_rof>
<ntasks_ice>2560</ntasks_ice>
<ntasks_ocn>1088</ntasks_ocn>
<ntasks_cpl>2752</ntasks_cpl>
<ntasks_glc>1</ntasks_glc>
<ntasks_wav>1</ntasks_wav>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>2560</rootpe_lnd>
<rootpe_rof>2560</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>2752</rootpe_ocn>
<rootpe_cpl>0</rootpe_cpl>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
</rootpe>
</pes>
<pes compset=".*CAM5.+CLM45.+MPASCICE.+MPASO.+MOSART.+SGLC.+SWAV" pesize="L">
<comment> -compset A_WCYCL* -res ne30_oEC* on 115 nodes: 21.5 sypd </comment>
<ntasks>
<ntasks_atm>5400</ntasks_atm>
<ntasks_lnd>320</ntasks_lnd>
<ntasks_rof>320</ntasks_rof>
<ntasks_ice>5120</ntasks_ice>
<ntasks_ocn>1920</ntasks_ocn>
<ntasks_cpl>5440</ntasks_cpl>
<ntasks_glc>1</ntasks_glc>
<ntasks_wav>1</ntasks_wav>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>5120</rootpe_lnd>
<rootpe_rof>5120</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>5440</rootpe_ocn>
<rootpe_cpl>0</rootpe_cpl>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%ne4np4_l%ne4np4_oi%oQU240_r%r05_m%oQU240_g%null_w%null">
<mach name="any">
Expand Down
29 changes: 25 additions & 4 deletions cime/config/e3sm/machines/config_batch.xml
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,12 @@
</queues>
</batch_system>

<batch_system MACH="chrysalis" type="slurm" >
<queues>
<queue walltimemax="03:00:00" nodemin="1" nodemax="512" default="true" strict="true">compute</queue>
</queues>
</batch_system>

<batch_system MACH="bebop" type="slurm" >
<queues>
<queue walltimemax="00:30:00" nodemin="1" nodemax="64" strict="true">debug</queue>
Expand All @@ -296,7 +302,18 @@
</queues>
</batch_system>

<batch_system MACH="edison" type="nersc_slurm" >
<batch_system MACH="pm-cpu" type="nersc_slurm">
<directives>
<directive> --constraint=cpu</directive>
</directives>
<queues>
<queue walltimemax="00:30:00" nodemin="1" nodemax="8" strict="true">debug</queue>
<queue walltimemax="00:35:00" nodemin="1" nodemax="4096" strict="true">preempt</queue>
<queue walltimemax="01:00:00" nodemin="1" nodemax="4096" default="true">regular</queue>
</queues>
</batch_system>

<batch_system MACH="edison" type="nersc_slurm" >
<queues>
<queue walltimemax="00:30:00" nodemin="1" nodemax="512" strict="true">debug</queue>
<queue walltimemax="01:30:00" nodemin="1" nodemax="5519" default="true">regular</queue>
Expand Down Expand Up @@ -496,11 +513,15 @@
<batch_system MACH="summit" type="lsf" >
<directives>
<directive>-P {{ project }}</directive>
<directive>-alloc_flags gpumps</directive>
<directive>-alloc_flags smt2</directive>
</directives>
<directives>
<directive>-alloc_flags "gpumps smt$SHELL{echo "(`./xmlquery --value MAX_TASKS_PER_NODE`+41)/42"|bc}"</directive>
</directives>
<queues>
<queue walltimemax="02:00" nodemin="0" nodemax="64" default="true">batch</queue>
<queue walltimemax="02:00" nodemin="1" nodemax="4608" strict="true" default="true">batch</queue>
<queue walltimemax="02:00" nodemin="1" nodemax="54" strict="true" default="false">batch-hm</queue>
<queue walltimemax="02:00" nodemin="1" nodemax="91" strict="true" default="false">killable</queue>
<queue walltimemax="02:00" nodemin="1" nodemax="4608" strict="true" default="false">debug</queue>
</queues>
</batch_system>

Expand Down
Loading

0 comments on commit 41aaaf6

Please sign in to comment.