Merge pull request #59 from xylar/add_pm-cpu

Add support for pm-cpu

mache/discover.py

@@ -1,5 +1,6 @@
 import socket
 import warnings
+import os
 
 
 def discover_machine():
@@ -12,6 +13,7 @@ def discover_machine():
         The name of the current machine
     """
     hostname = socket.gethostname()
+    machine = None
     if hostname.startswith('acme1'):
         machine = 'acme1'
     elif hostname.startswith('andes'):
@@ -27,11 +29,28 @@ def discover_machine():
     elif hostname.startswith('cooley'):
         machine = 'cooley'
     elif hostname.startswith('cori'):
-        warnings.warn('defaulting to cori-haswell.  Use -m cori-knl if you'
-                      ' wish to run on KNL.')
+        warnings.warn('defaulting to cori-haswell.  Explicitly specify '
+                      'cori-knl as the machine if you wish to run on KNL.')
         machine = 'cori-haswell'
     elif hostname.startswith('gr-fe'):
         machine = 'grizzly'
-    else:
-        machine = None
+    elif 'NERSC_HOST' in os.environ:
+        hostname = os.environ['NERSC_HOST']
+        if hostname == 'perlmutter':
+            # perlmutter's hostname is too generic to detect, so relying on
+            # $NERSC_HOST
+
+            warnings.warn('defaulting to pm-cpu.  Explicitly specify '
+                          'pm-gpu as the machine if you wish to run on '
+                          'GPUs.')
+            machine = 'pm-cpu'
+        elif hostname == 'unknown':
+            raise ValueError(
+                'You appear to have $NERSC_HOST=unknown.  This typically '
+                'indicates that you \n'
+                'have an outdated .bash_profile.ext or similar.  Please '
+                'either delete or \n'
+                'edit that file so it no longer defines $NERSC_HOST, log out, '
+                'log back in, \n'
+                'and try again.')
     return machine

mache/machine_info.py

@@ -72,7 +72,7 @@ def __init__(self, machine=None):
         if machine is None:
             machine = discover_machine()
             if machine is None:
-                raise ValueError('Unable to discover machine form host name')
+                raise ValueError('Unable to discover machine from host name')
         self.machine = machine
 
         self.config = self._get_config()

mache/machines/pm-cpu.cfg

@@ -0,0 +1,77 @@
+# Options related to deploying an e3sm-unified conda environment on supported
+# machines
+[e3sm_unified]
+
+# the unix group for permissions for the e3sm-unified conda environment
+group = e3sm
+
+# the compiler set to use for system libraries and MPAS builds
+compiler = gnu
+
+# the system MPI library to use for intel18 compiler
+mpi = mpich
+
+# the path to the directory where activation scripts, the base environment, and
+# system libraries will be deployed
+base_path = /global/common/software/e3sm/anaconda_envs
+
+
+# config options related to data needed by diagnostics software such as
+# e3sm_diags and MPAS-Analysis
+[diagnostics]
+
+# The base path to the diagnostics directory
+base_path = /global/cfs/cdirs/e3sm/diagnostics
+
+# the unix group for permissions for diagnostics
+group = e3sm
+
+
+# config options associated with web portals
+[web_portal]
+
+# The path to the base of the web portals
+base_path = /global/cfs/cdirs/e3sm/www
+
+# The base URL that corresponds to the base path
+base_url = https://portal.nersc.gov/cfs/e3sm
+
+
+# The parallel section describes options related to running jobs in parallel
+[parallel]
+
+# parallel system of execution: slurm, cobalt or single_node
+system = slurm
+
+# whether to use mpirun or srun to run a task
+parallel_executable = srun
+
+# cores per node on the machine
+cores_per_node = 256
+
+# account for running diagnostics jobs
+account = e3sm
+
+# available constraint(s) (default is the first)
+constraints = cpu
+
+# quality of service (default is the first)
+qos = regular, premium, debug
+
+# Config options related to spack environments
+[spack]
+
+# whether to load modules from the spack yaml file before loading the spack
+# environment
+modules_before = False
+
+# whether to load modules from the spack yaml file after loading the spack
+# environment
+modules_after = False
+
+
+# config options related to synchronizing files
+[sync]
+
+# the full hostname of the machine
+hostname = perlmutter-p1.nersc.gov

mache/spack/pm-cpu_gnu_mpich.csh

@@ -0,0 +1,41 @@
+module rm PrgEnv-gnu
+module rm PrgEnv-nvidia
+module rm cudatoolkit
+module rm craype-accel-nvidia80
+module rm craype-accel-host
+module rm perftools-base
+module rm perftools
+module rm darshan
+
+module load PrgEnv-gnu/8.3.3
+module load gcc/11.2.0
+module load craype-accel-host
+{% if e3sm_lapack %}
+module load cray-libsci
+{% endif %}
+module load craype
+module rm cray-mpich
+module load cray-mpich/8.1.15
+{% if e3sm_hdf5_netcdf %}
+module rm cray-hdf5-parallel
+module rm cray-netcdf-hdf5parallel
+module rm cray-parallel-netcdf
+module load cray-hdf5-parallel/1.12.1.1
+module load cray-netcdf-hdf5parallel/4.8.1.1
+module load cray-parallel-netcdf/1.12.2.1
+{% endif %}
+module load cmake/3.22.0
+
+{% if e3sm_hdf5_netcdf %}
+setenv NETCDF_C_PATH $(dirname $(dirname $(which nc-config)))
+setenv NETCDF_FORTRAN_PATH $(dirname $(dirname $(which nf-config)))
+setenv PNETCDF_PATH $(dirname $(dirname $(which pnetcdf_version)))
+{% endif %}
+setenv MPICH_ENV_DISPLAY 1
+setenv MPICH_VERSION_DISPLAY 1
+setenv OMP_STACKSIZE 128M
+setenv OMP_PROC_BIND spread
+setenv OMP_PLACES threads
+setenv HDF5_USE_FILE_LOCKING FALSE
+setenv PERL5LIB /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
+setenv FI_CXI_RX_MATCH_MODE software

mache/spack/pm-cpu_gnu_mpich.sh

@@ -0,0 +1,41 @@
+module rm PrgEnv-gnu
+module rm PrgEnv-nvidia
+module rm cudatoolkit
+module rm craype-accel-nvidia80
+module rm craype-accel-host
+module rm perftools-base
+module rm perftools
+module rm darshan
+
+module load PrgEnv-gnu/8.3.3
+module load gcc/11.2.0
+module load craype-accel-host
+{% if e3sm_lapack %}
+module load cray-libsci
+{% endif %}
+module load craype
+module rm cray-mpich
+module load cray-mpich/8.1.15
+{% if e3sm_hdf5_netcdf %}
+module rm cray-hdf5-parallel
+module rm cray-netcdf-hdf5parallel
+module rm cray-parallel-netcdf
+module load cray-hdf5-parallel/1.12.1.1
+module load cray-netcdf-hdf5parallel/4.8.1.1
+module load cray-parallel-netcdf/1.12.2.1
+{% endif %}
+module load cmake/3.22.0
+
+{% if e3sm_hdf5_netcdf %}
+export NETCDF_C_PATH=$(dirname $(dirname $(which nc-config)))
+export NETCDF_FORTRAN_PATH=$(dirname $(dirname $(which nf-config)))
+export PNETCDF_PATH=$(dirname $(dirname $(which pnetcdf_version)))
+{% endif %}
+export MPICH_ENV_DISPLAY=1
+export MPICH_VERSION_DISPLAY=1
+export OMP_STACKSIZE=128M
+export OMP_PROC_BIND=spread
+export OMP_PLACES=threads
+export HDF5_USE_FILE_LOCKING=FALSE
+export PERL5LIB=/global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch
+export FI_CXI_RX_MATCH_MODE=software

mache/spack/pm-cpu_gnu_mpich.yaml

@@ -0,0 +1,137 @@
+spack:
+  specs:
+  - cmake
+  - gcc
+  - mpich
+{% if e3sm_lapack %}
+  - cray-libsci
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+  - hdf5
+  - netcdf-c
+  - netcdf-fortran
+  - parallel-netcdf
+{% endif %}
+{{ specs }}
+  concretizer:
+    unify: true
+  packages:
+    all:
+      compiler: [[email protected]]
+      providers:
+        mpi: [[email protected]]
+{% if e3sm_lapack %}
+        lapack: [[email protected]]
+{% endif %}
+    bzip2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    curl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    gettext:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    openssl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    perl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    tar:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    xz:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /global/common/software/nersc/pm-2021q4/sw/cmake-3.22.0
+        modules:
+        - cmake/3.22.0
+      buildable: false
+    gcc:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/gcc/11.2.0/
+        modules:
+        - gcc/11.2.0
+      buildable: false
+    mpich:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/mpich/8.1.15/ofi/gnu/9.1
+        modules:
+        - cray-mpich/8.1.15
+      buildable: false
+{% if e3sm_lapack %}
+    cray-libsci:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/libsci/21.08.1.2/GNU/9.1/x86_64
+        modules:
+        - mkl/2019.0.4
+      buildable: false
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+    hdf5:
+      externals:
+      - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.1.1/GNU/9.1
+        modules:
+        - cray-hdf5-parallel/1.12.1.1
+      buildable: false
+    parallel-netcdf:
+      externals:
+      - spec: [email protected]+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.2.1/GNU/9.1/
+        modules:
+        - cray-parallel-netcdf/1.12.2.1
+      buildable: false
+    netcdf-c:
+      externals:
+      - spec: [email protected]+mpi+parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.8.1.1/GNU/9.1
+        modules:
+        - cray-netcdf-hdf5parallel/4.8.1.1
+      buildable: false
+    netcdf-fortran:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.8.1.1/GNU/9.1
+        modules:
+        - cray-netcdf-hdf5parallel/4.8.1.1
+      buildable: false
+{% endif %}
+  config:
+    install_missing_compilers: false
+  compilers:
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      target: x86_64
+      modules: []
+      environment: {}
+      extra_rpaths:
+      - /opt/cray/libfabric/1.15.0.0/lib64