diff --git a/Jenkinsfile b/Jenkinsfile index 04eb4fb417..f3865140e4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,20 +20,30 @@ pipeline { } } stages { - stage('Create environment') { + stage('Create python environment') { steps { sh """ - python3 -m venv env - source env/bin/activate + python3 -m venv .venv + source .venv/bin/activate pip install -r requirements.txt """ } } + stage('Create uenv') { + steps { + sh """ + git clone -b fix/jenkins https://github.com/eth-cscs/uenv.git + ./uenv/install --yes --destdir=$WORKSPACE + source $WORKSPACE/etc/profile.d/uenv.sh + uenv repo create + uenv image pull mch/v8:rc2 + """ + } + } stage('Bootstrap spack') { // Bootstrapping spack is a separate stage to avoid problems with concurrently bootstrapping spack in the tests. steps { sh """ - source env/bin/activate source ./setup-env.sh spack spec gnuconfig """ @@ -42,7 +52,7 @@ pipeline { stage('Unit Tests') { steps { sh """ - source env/bin/activate + source .venv/bin/activate python3 test/unit_test.py """ } @@ -50,18 +60,20 @@ pipeline { stage('Integration Tests') { steps { sh """ - source env/bin/activate - source ./setup-env.sh /mch-environment/v7 - pytest -v -n auto test/integration_test.py + source $WORKSPACE/etc/profile.d/uenv.sh + source ./setup-env.sh /user-environment + source .venv/bin/activate + uenv run mch/v8:rc2 -- pytest -v -n auto test/integration_test.py """ } } stage('System Tests') { steps { sh """ - source env/bin/activate - source ./setup-env.sh /mch-environment/v7 - pytest -v -n auto test/system_test.py + source $WORKSPACE/etc/profile.d/uenv.sh + source ./setup-env.sh /user-environment + source .venv/bin/activate + uenv run mch/v8:rc2 -- pytest -v -n auto test/system_test.py """ } } diff --git a/docs/CodeDevelopment.rst b/docs/CodeDevelopment.rst index cc3c7caac9..ce73988233 100644 --- a/docs/CodeDevelopment.rst +++ b/docs/CodeDevelopment.rst @@ -3,42 +3,6 @@ Develop packages Spack offers several options for package development. Depending on your workflow, one or the other option is preferred. -Also some packages like ICON or COSMO have their own -development workflow which is maintained by C2SM. - -Plain dev-build ---------------- - -This is the easiest way to build local sources. -Enter the root of your source repository and execute: - -.. code-block:: console - - $ spack dev-build --until build @ - -This will build the package as is. The downside of this approach is that -you need to go through all phases of a package build. - -Dev-build in combination with build-env ---------------------------------------- - -We assume that developers of a package are familiar with its build system. -Therefore, we reccomend to use Spack to set up the environment for the package. -Building and testing should be done with the package's build and test system. - -.. code-block:: console - - # Load Spack! - $ spack dev-build --before build @develop # stops dev-build before executing the phase 'build' - $ spack build-env @develop -- bash # nests a bash shell with the build env vars loaded - # Work on the package! - # Use the build system of the package! (e.g. 'make') - # Use the testing infrastructure of the package! - $ exit # to exit the nested bash - -If you want multiple dev-builds at the same time, label them with separate ``@``. -The identifier ``@develop`` is common in the Spack documentation but you can use any string. - Environments with Spack develop ------------------------------- @@ -95,3 +59,15 @@ To deactivate a Spack environment, type Most of the Spack commands are sensitive to environments, see `Spack docs `__. +Plain dev-build +--------------- + +This is the easiest way to build local sources. +Enter the root of your source repository and execute: + +.. code-block:: console + + $ spack dev-build --until build @ + +This will build the package as is. The downside of this approach is that +you need to go through all phases of a package build. diff --git a/docs/QuickStart.rst b/docs/QuickStart.rst index e1ad39f7aa..7b17770d9f 100644 --- a/docs/QuickStart.rst +++ b/docs/QuickStart.rst @@ -11,19 +11,17 @@ To set up a Spack instance, clone the repository using a specific Spack tag (lat $ git clone --depth 1 --recurse-submodules --shallow-submodules -b $SPACK_TAG https://github.com/C2SM/spack-c2sm.git -To load it into your command line, execute +To load it into your command line, execute one of the following commands: .. code-block:: console $ . spack-c2sm/setup-env.sh + $ . spack-c2sm/setup-env.sh /user-environment + $ . spack-c2sm/setup-env.sh /mch-environment/v6 + $ . spack-c2sm/setup-env.sh /mch-environment/v7 + $ . spack-c2sm/setup-env.sh any_other_upstream -This auto-detects your machine and configures your instance for it. -You can force a machine with an argument. The name has to match a folder in sysconfigs. - -.. code-block:: console - - $ . spack-c2sm/setup-env.sh balfrin - +This will make upstream installation from user-environment available in spack-c2sm. Local machines and Containers ----------------------------- @@ -89,16 +87,6 @@ For ICON, they are located in ``config/cscs/spack//__< They work with a special Spack tag, that is provided in the ICON repository at ``config/cscs/SPACK_TAG_*``. So make sure you clone Spack with the specified tag. -.. tip:: - **On Balfrin:** - In case your Spack environment requires Python, a compatability issue - with `openssl` and `git` appears. - - ``/usr/bin/ssh: symbol lookup error: /usr/bin/ssh: undefined symbol: EVP_KDF_CTX_free, version OPENSSL_1_1_1d`` - - To circumvent that simply do - ``spack load git`` prior to activation of the environment. - To activate the Spack environment, type .. code-block:: console @@ -127,15 +115,25 @@ Out-of-source builds are possible as follows: .. code-block:: console - $ mkdir cpu && cd cpu - $ cp -r ../config . - $ spack env activate -d config/cscs/spack/v0.20.1.5/daint_cpu_nvhpc + $ mkdir cpu + $ spack env activate config/cscs/spack/v0.20.1.5/daint_cpu_nvhpc + $ # tell spack to build icon in folder cpu + $ spack develop --path $(pwd) --build-directory cpu icon@develop $ spack install -.. attention:: - Out-of-source build for AutotoolsPackages is not supported by Spack. - The implementation for ICON relies on some hacks inside package.py and - only works if the build-folder is located inside the Git repo of ICON. +By executing the commands above, spack will add some lines directly into ``spack.yaml``: + +.. code-block:: yaml + + spack: + packages: + icon: + package_attributes: + build_directory: /scratch/mch/juckerj/icon-nwp/cpu + +Any further ``spack install`` command will use the build directory specified in the ``spack.yaml`` file. +In case you want to change the build directory, edit the ``spack.yaml`` file or remove the ``build_directory`` line +and run ``spack concretize -f`` afterwards. COSMO ----- diff --git a/repos/c2sm/packages/cosmo-eccodes-definitions/package.py b/repos/c2sm/packages/cosmo-eccodes-definitions/package.py index 8de1cfbb70..1666160a2d 100644 --- a/repos/c2sm/packages/cosmo-eccodes-definitions/package.py +++ b/repos/c2sm/packages/cosmo-eccodes-definitions/package.py @@ -1,25 +1,3 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# ---------------------------------------------------------------------------- -# If you submit this package back to Spack as a pull request, -# please first remove this boilerplate and all FIXME comments. -# -# This is a template package file for Spack. We've put "FIXME" -# next to all the things you'll want to change. Once you've handled -# them, you can save this file and test your package like this: -# -# spack install cosmo-eccodes-definitions -# -# You can edit this file again by typing: -# -# spack edit cosmo-eccodes-definitions -# -# See the Spack documentation for more information on packaging. -# ---------------------------------------------------------------------------- - from spack import * @@ -30,27 +8,21 @@ class CosmoEccodesDefinitions(Package): url = "https://github.com/COSMO-ORG/eccodes-cosmo-resources.git" git = 'https://github.com/COSMO-ORG/eccodes-cosmo-resources.git' - maintainers = ['petrabaumann'] + maintainers = ['huppd,lxavier'] + version('2.36.0.3', tag='v2.36.0.3') version('2.25.0.3', tag='v2.25.0.3') version('2.25.0.2', tag='v2.25.0.2') version('2.25.0.1', tag='v2.25.0.1') - version('2.19.0.7', tag='v2.19.0.7') - version('2.19.0.6', tag='v2.19.0.6') - version('2.19.0.5', tag='v2.19.0.5') - version('2.19.0.4', tag='v2.19.0.4') - version('2.19.0.3', tag='v2.19.0.3') - version('2.19.0.2', tag='v2.19.0.2') - version('2.19.0.1', tag='v2.19.0.1') version('2.18.0.1', tag='v2.18.0.1') depends_on('eccodes') - depends_on('eccodes@2.25.0', + depends_on('eccodes@2.36.4', type=('build', 'link', 'run'), - when='@2.25.0.1:') - depends_on('eccodes@2.19.0', + when='@2.36.0.3') + depends_on('eccodes@2.25.0', type=('build', 'link', 'run'), - when='@2.19.0.1:2.19.0.7') + when='@2.25.0.1:2.25.0.3') depends_on('eccodes@2.18.0', type=('build', 'link', 'run'), when='@2.18.0.1') diff --git a/repos/c2sm/packages/cosmo/package.py b/repos/c2sm/packages/cosmo/package.py deleted file mode 100644 index 2abb680196..0000000000 --- a/repos/c2sm/packages/cosmo/package.py +++ /dev/null @@ -1,329 +0,0 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import subprocess, itertools, os -from spack import * - - -class Cosmo(MakefilePackage): - """COSMO: Numerical Weather Prediction Model. Needs access to private GitHub.""" - - homepage = "http://www.cosmo-model.org" - url = "https://github.com/COSMO-ORG/cosmo/archive/6.0.tar.gz" - git = 'git@github.com:COSMO-ORG/cosmo.git' - apn_git = 'git@github.com:MeteoSwiss-APN/cosmo.git' - c2sm_git = 'git@github.com:C2SM-RCM/cosmo.git' - empa_git = 'git@github.com:C2SM-RCM/cosmo-ghg.git' - maintainers = ['mjaehn', 'juckerj'] - - version('org-master', branch='master') - version('6.0', tag='6.0') - - version('apn-mch', git=apn_git, branch='mch') - version('5.09a.mch1.2.p2', git=apn_git, tag='5.09a.mch1.2.p2') - - version('6.1_2023.11', git=c2sm_git, tag='6.1_2023.11') - version('c2sm-master', git=c2sm_git, branch='master') - version('c2sm-features', git=c2sm_git, branch='c2sm-features') - version('empa-ghg', git=empa_git, branch='c2sm') - - # pass spec from spec to test_cosmo.py in yaml-format - # There are three different types of test_cosmo.py around: - - # COSMO-ORG - patch('patches/org-master/spec_as_yaml/patch.test_cosmo', - when='@org-master') - patch('patches/org-master/spec_as_yaml/patch.test_cosmo', when='@6.0') - # APN-MCH - patch('patches/apn-mch/spec_as_yaml/patch.test_cosmo', when='@apn-mch') - patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.test_cosmo', - when='@5.09a.mch1.2.p2') - - # pass spec from spec to serialize_cosmo.py in yaml-format - - # There are two different types of serialize_cosmo.py around: - - # COSMO-ORG - patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', - when='@org-master +serialize') - patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', - when='@6.0 +serialize') - patch('patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo', - when='@c2sm-features +serialize') - patch('patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo', - when='@empa-ghg +serialize') - - # APN-MCH - patch('patches/apn-mch/spec_as_yaml/patch.serialize_cosmo', - when='@apn-mch +serialize') - patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.serialize_cosmo', - when='@5.09a.mch1.2.p2 +serialize') - - # build dependency - depends_on('perl@5.16.3:', type='build') - depends_on('libgrib1', type='build') - - # build and link dependency - depends_on('mpi +fortran') - depends_on('netcdf-fortran') - depends_on('netcdf-c +mpi') - depends_on('jasper@1.900.1') - depends_on('eccodes +fortran') - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - depends_on('eccodes %nvhpc', when='%nvhpc') - depends_on('eccodes %gcc', when='%gcc') - - # run dependency - depends_on('slurm', type='run') - - depends_on('cosmo-eccodes-definitions', type=('build', 'run')) - depends_on('serialbox +fortran ^python@2:2.9', - when='+serialize', - type=('build', 'link', 'run')) - depends_on('oasis', when='+oasis', type=('build', 'link', 'run')) - - variant('serialize', - default=False, - description='Build with serialization enabled') - variant('parallel', default=True, description='Build parallel COSMO') - variant('debug', default=False, description='Build debug mode') - variant('real_type', - default='double', - description='Build with double or single precision enabled', - values=('double', 'float'), - multi=False) - variant('slave', default='none', description='Build on slave') - variant('pollen', default=False, description='Build with pollen enabled') - variant('verbose', - default=False, - description='Build cosmo with verbose enabled') - variant('set_version', - default=False, - description='Pass cosmo tag version to Makefile') - variant('gt1', - default=False, - description='Build dycore with gridtools 1.1.3') - variant('cuda_arch', - default='none', - description='Build with cuda_arch', - values=('80', '70', '60', '37'), - multi=False) - variant('oasis', - default=False, - description='Build with the unified oasis interface') - - conflicts( - '@dev-build', - msg= - "Please use only official versions listed with 'spack info cosmo'. Even when using 'devbuildcosmo'. C2SM introduced 'dev-build' to avoid name conflicts with the upstream instance. Since spack-c2sm v0.18.1.0 this is not relevant anymore." - ) - conflicts('+pollen', when='@org-master,master') - # - ML - A conflict should be added there if the oasis variant is - # chosen and the version is neither c2sm-features nor - # dev-build. The problem is that this doesn't seem possible in a - # native spack way. Hence the dirty check at the beginning of - # setup_build_environment method - - build_directory = 'cosmo/ACC' - - # v0.20.1 does not fully support builds of COSMO - # users should continue using v0.18.1.x on Daint - # once support on Alps has been clarified finalize - # recipe and remove line below. - manual_download = True - - def setup_build_environment(self, env): - - # - ML - Dirty conflict check (see above) - if self.spec.variants['oasis'].value and self.spec.version not in ( - Version('c2sm-features')): - raise InstallError( - '+oasis variant only compatible with the @c2sm-features versions' - ) - - self.setup_run_environment(env) - - # Check mpi provider - self.mpi_spec = self.spec['mpi'] - - grib_prefix = self.spec['eccodes'].prefix - env.set( - 'GRIBAPIL', - str(self.spec['eccodes:fortran'].libs.ld_flags) + ' ' + - str(self.spec['jasper'].libs.ld_flags)) - grib_inc_dir_path = os.path.join(grib_prefix, 'include') - if os.path.exists(grib_inc_dir_path): - env.set('GRIBAPII', '-I' + grib_inc_dir_path) - else: - env.set('GRIBAPII', '') - - # libaec - if 'libaec' in self.spec: - env.set('AECL', '-L' + self.spec['libaec'].prefix + '/lib64 -laec') - env.set('AECI', '-I' + self.spec['libaec'].prefix + '/include') - - # Netcdf library - env.set( - 'NETCDFL', - '-L' + self.spec['netcdf-fortran'].prefix + '/lib -lnetcdff -L' + - self.spec['netcdf-c'].prefix + '/lib -lnetcdf') - env.set('NETCDFI', - '-I' + self.spec['netcdf-fortran'].prefix + '/include') - - # Grib1 library - if self.compiler.name == 'gcc': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_gnu') - elif self.compiler.name == 'cce': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_cray') - elif self.compiler.name == 'nvhpc': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_pgi') - else: - env.set( - 'GRIBDWDL', '-L' + self.spec['libgrib1'].prefix + - '/lib -lgrib1_' + self.compiler.name) - env.set('GRIBDWDI', '-I' + self.spec['libgrib1'].prefix + '/include') - - # MPI library - if self.mpi_spec.name == 'openmpi': - env.set('MPIL', '-L' + self.mpi_spec.prefix + ' -lmpi') - - else: - env.set('MPIL', '-L' + self.spec['mpi'].prefix + ' -lmpich') - - env.set('MPII', '-I' + self.mpi_spec.prefix + '/include') - - # Serialbox library - if '+serialize' in self.spec: - env.set('SERIALBOX', self.spec['serialbox'].prefix) - env.set('SERIALBOXL', - self.spec['serialbox:fortran,c'].libs.ld_flags) - env.set('SERIALBOXI', - '-I' + self.spec['serialbox'].prefix + '/include') - - # OASIS library - if '+oasis' in self.spec: - oasis_prefix = self.spec['oasis'].prefix - env.set( - 'PSMILEL', - '-L{:s}/lib -lpsmile.MPI1 -lscrip -lmct -lmpeu'.format( - oasis_prefix)) - env.set( - 'PSMILEI', - '-I{0:s}/build/lib/psmile.MPI1 -I{0:s}/build/lib/mct'.format( - oasis_prefix)) - env.set('MPPIOI', '-I{:s}/build/lib/mct'.format(oasis_prefix)) - - # Linker flags - if self.compiler.name in ('pgi', - 'nvhpc') and '~cppdycore' in self.spec: - env.set('LFLAGS', '-lstdc++') - - # Compiler & linker variables - if self.compiler.name in ('pgi', 'nvhpc'): - env.set('F90', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') - env.set('LD', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') - else: - env.set('F90', self.mpi_spec.mpifc) - env.set('LD', self.mpi_spec.mpifc) - - @property - def build_targets(self): - build = [] - if self.spec.variants['pollen'].value: - build.append('POLLEN=1') - if self.spec.variants['oasis'].value: - build.append('COUP_OAS=1') - if self.spec.variants['real_type'].value == 'float': - build.append('SINGLEPRECISION=1') - if '+serialize' in self.spec: - build.append('SERIALIZE=1') - if self.spec.variants['verbose'].value: - build.append('VERBOSE=1') - if '+set_version' in self.spec: - build.append('COSMO_VERSION=' + self.spec.format('{version}')) - MakeFileTarget = '' - if '+parallel' in self.spec: - MakeFileTarget += 'par' - else: - MakeFileTarget += 'seq' - if '+debug' in self.spec: - MakeFileTarget += 'debug' - else: - MakeFileTarget += 'opt' - build.append(MakeFileTarget) - - return build - - def edit(self, spec, prefix): - with working_dir(self.build_directory): - OptionsFileName = 'Options' - if self.compiler.name == 'gcc': - OptionsFileName += '.gnu' - elif self.compiler.name in ('pgi', 'nvhpc'): - OptionsFileName += '.pgi' - elif self.compiler.name == 'cce': - OptionsFileName += '.cray' - OptionsFileName += '.cpu' - OptionsFile = FileFilter(OptionsFileName) - - makefile = FileFilter('Makefile') - makefile.filter('/Options.*', '/' + OptionsFileName) - if self.spec.version == Version('empa-ghg'): - if '~serialize' in spec: - makefile.filter( - 'TARGET :=.*', - 'TARGET := {0}'.format('cosmo-ghg_cpu')) - else: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo-ghg')) - else: - if '~serialize' in spec: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo_cpu')) - else: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo')) - - # Pre-processor flags - if self.mpi_spec.name == 'mpich': - OptionsFile.filter( - 'PFLAGS = -Mpreprocess.*', - 'PFLAGS = -Mpreprocess -DNO_MPI_HOST_DATA') - - def install(self, spec, prefix): - - with working_dir(self.build_directory): - mkdir(prefix.bin) - if self.spec.version == Version('empa-ghg'): - if '+serialize' in spec: - install('cosmo-ghg_serialize', prefix.bin) - else: - install('cosmo-ghg_cpu', prefix.bin) - install('cosmo-ghg_cpu', 'test/testsuite') - else: - if '+serialize' in spec: - install('cosmo_serialize', prefix.bin) - else: - install('cosmo_cpu', prefix.bin) - install('cosmo_cpu', 'test/testsuite') - - @run_after('install') - @on_package_attributes(run_tests=True) - def test(self): - with open('spec.yaml', mode='w') as f: - f.write(self.spec.to_yaml()) - try: - subprocess.run([ - self.build_directory + '/test/tools/test_cosmo.py', '-s', - 'spec.yaml', '-b', - str('.') - ], - stderr=subprocess.STDOUT, - check=True) - except: - raise InstallError('Testsuite failed') diff --git a/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 b/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 deleted file mode 120000 index 34c563540d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 +++ /dev/null @@ -1 +0,0 @@ -apn-mch/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo deleted file mode 100644 index 39aa5e1fc2..0000000000 --- a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py -index 624fd295..fc15c712 100755 ---- a/cosmo/ACC/test/tools/serialize_cosmo.py -+++ b/cosmo/ACC/test/tools/serialize_cosmo.py -@@ -15,7 +15,14 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) - args=parser.parse_args() - -- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ - cosmo_spec.concretize() - - serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo deleted file mode 100644 index 40bc6a90d1..0000000000 --- a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo +++ /dev/null @@ -1,21 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py -index 733bde4a..2a166bbd 100755 ---- a/cosmo/ACC/test/tools/test_cosmo.py -+++ b/cosmo/ACC/test/tools/test_cosmo.py -@@ -16,7 +16,15 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') - args=parser.parse_args() - -- test_cosmo(Spec(args.spec), args.base_directory) -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ -+ test_cosmo(cosmo_spec, args.base_directory) - - def test_cosmo(cosmo_spec, base_directory): - diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo deleted file mode 120000 index bfd9c3d65d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1 +0,0 @@ -../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-master b/repos/c2sm/packages/cosmo/patches/c2sm-master deleted file mode 120000 index 71b99193f9..0000000000 --- a/repos/c2sm/packages/cosmo/patches/c2sm-master +++ /dev/null @@ -1 +0,0 @@ -org-master/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo deleted file mode 120000 index bfd9c3d65d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1 +0,0 @@ -../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo deleted file mode 100644 index 8c72160de2..0000000000 --- a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py -index 624fd295..e93b96f6 100755 ---- a/cosmo/ACC/test/tools/serialize_cosmo.py -+++ b/cosmo/ACC/test/tools/serialize_cosmo.py -@@ -15,7 +15,13 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) - args=parser.parse_args() - -- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) - cosmo_spec.concretize() - - serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo deleted file mode 100644 index 8e91d15f45..0000000000 --- a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py -index 733bde4a..c2c60f2b 100755 ---- a/cosmo/ACC/test/tools/test_cosmo.py -+++ b/cosmo/ACC/test/tools/test_cosmo.py -@@ -16,7 +16,16 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') - args=parser.parse_args() - -- test_cosmo(Spec(args.spec), args.base_directory) -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ -+ -+ test_cosmo(cosmo_spec, args.base_directory) - - def test_cosmo(cosmo_spec, base_directory): - diff --git a/repos/c2sm/packages/eccodes/package.py b/repos/c2sm/packages/eccodes/package.py deleted file mode 100644 index f27782f4b8..0000000000 --- a/repos/c2sm/packages/eccodes/package.py +++ /dev/null @@ -1,10 +0,0 @@ -from spack.package import * - -from spack.pkg.builtin.eccodes import Eccodes as SpackEccodes - - -class Eccodes(SpackEccodes): - - version('2.19.0', - sha256= - 'a1d080aed1b17a9d4e3aecccc5a328c057830cd4d54f451f5498b80b24c46404') diff --git a/repos/c2sm/packages/fdb-fortran/package.py b/repos/c2sm/packages/fdb-fortran/package.py deleted file mode 100644 index 422f21b261..0000000000 --- a/repos/c2sm/packages/fdb-fortran/package.py +++ /dev/null @@ -1,33 +0,0 @@ -from spack import * - - -class FdbFortran(CMakePackage): - """An experimental Fortran interface to ECMWF's FDB (Fields DataBase).""" - - homepage = 'https://github.com/MeteoSwiss/fdb-fortran' - git = 'https://github.com/MeteoSwiss/fdb-fortran.git' - maintainers = ['victoria-cherkas'] - - version('0.1.0', tag='0.1.0') - - depends_on('cmake@3.10:', type='build') - depends_on('eckit') - depends_on('metkit') - depends_on('eccodes +fortran') - depends_on('fdb@5.11.0:') - - @property - def libs(self): - return find_libraries("libfdbf", - root=self.prefix, - shared=False, - recursive=True) - - def cmake_args(self): - args = [ - self.define('Deckit_DIR', self.spec['eckit'].prefix), - self.define('Dmetkit_DIR', self.spec['metkit'].prefix), - self.define('Deccodes_DIR', self.spec['eccodes'].prefix), - self.define('Dfdb5_DIR', self.spec['fdb'].prefix), - ] - return args diff --git a/repos/c2sm/packages/fdb/package.py b/repos/c2sm/packages/fdb/package.py deleted file mode 100644 index 459f692f5a..0000000000 --- a/repos/c2sm/packages/fdb/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack.package import * - -from spack.pkg.builtin.fdb import Fdb as SpackFdb - - -class Fdb(SpackFdb): - - # This section can be removed when the following commit - # https://github.com/spack/spack/commit/8871bd5ba5c58562b8c20baa00f125aeccba586f - # is included in a release on spack and this is used by spack-c2sm. - depends_on("eckit@1.24.4:", when="@5.11.22:") - - # This section can be removed when the following PR - # https://github.com/spack/spack/pull/42874 - # is included in a release on spack and this is used by spack-c2sm. - @property - def libs(self): - return find_libraries("libfdb5", - root=self.prefix, - shared=True, - recursive=True) diff --git a/repos/c2sm/packages/flexpart-cosmo/package.py b/repos/c2sm/packages/flexpart-cosmo/package.py index 79df4300b2..5b380ed7ba 100644 --- a/repos/c2sm/packages/flexpart-cosmo/package.py +++ b/repos/c2sm/packages/flexpart-cosmo/package.py @@ -19,13 +19,11 @@ class FlexpartCosmo(MakefilePackage): version('main', branch='main') depends_on('eccodes +fortran') - depends_on('netcdf-fortran') + # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on('netcdf-fortran %gcc') depends_on('makedepf90') - conflicts('%gcc@:10') - conflicts('%nvhpc') - conflicts('%pgi') - conflicts('%cce') + requires('%gcc@11:') build_directory = 'src' diff --git a/repos/c2sm/packages/flexpart-ifs/package.py b/repos/c2sm/packages/flexpart-ifs/package.py index cc3f5c7751..2eea6ddb43 100644 --- a/repos/c2sm/packages/flexpart-ifs/package.py +++ b/repos/c2sm/packages/flexpart-ifs/package.py @@ -10,15 +10,13 @@ class FlexpartIfs(MakefilePackage): maintainers = ['pirmink'] version('main', branch='main') - version('fdb', tag='10.4.4_fdb') version('10.4.4', tag='10.4.4') depends_on('eccodes +fortran') - depends_on('netcdf-fortran') - depends_on('fdb-fortran', when='@fdb') + # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on('netcdf-fortran %gcc') - conflicts('%nvhpc') - conflicts('%pgi') + requires('%gcc') build_directory = 'src' @@ -29,12 +27,6 @@ def setup_build_environment(self, env): '-I' + self.spec['netcdf-fortran'].prefix.include) env.set('NETCDF_FORTRAN_LD_FLAGS', self.spec['netcdf-fortran'].libs.ld_flags) - if self.spec.satisfies('@fdb'): - env.set('FDB_DIR', self.spec['fdb'].prefix) - env.set('FDB_LD_FLAGS', self.spec['fdb'].libs.ld_flags) - env.set('FDB_FORTRAN_DIR', self.spec['fdb-fortran'].prefix) - env.set('FDB_FORTRAN_LD_FLAGS', - self.spec['fdb-fortran'].libs.ld_flags) def build(self, spec, prefix): with working_dir(self.build_directory): diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py new file mode 100755 index 0000000000..599e71ac00 --- /dev/null +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -0,0 +1,107 @@ +from spack.pkg.c2sm.icon import Icon as IconC2sm + +import os +import re +from collections import defaultdict +import spack.error as error + + +def validate_variant_dsl(pkg, name, value): + set_mutual_excl = set(['substitute', 'verify', 'serialize']) + set_input_var = set(value) + if len(set_mutual_excl.intersection(set_input_var)) > 1: + raise error.SpecError( + 'Cannot have more than one of (substitute, verify, serialize) in the same build' + ) + + +class IconExclaim(IconC2sm): + git = 'git@github.com:C2SM/icon-exclaim.git', + + maintainers('jonasjucker', 'huppd') + + version('icon-dsl', branch='icon-dsl', submodules=True) + + # EXCLAIM-GT4Py specific features: + dsl_values = ('substitute', 'verify', 'serialize', 'fused', 'nvtx', 'lam') + variant('dsl', + default='none', + validator=validate_variant_dsl, + values=('none', ) + dsl_values, + description='Build with GT4Py dynamical core', + multi=True) + + for x in dsl_values: + depends_on('py-icon4py', when='dsl={0}'.format(x)) + depends_on('py-gridtools-cpp', when='dsl={0}'.format(x)) + depends_on('boost', when='dsl={0}'.format(x)) + conflicts('^python@:3.9,3.11:', when='dsl={0}'.format(x)) + + def setup_build_environment(self, env): + super().set_build_envirionment(self, env) + # help cmake to build dsl-stencils + if 'none' not in self.spec.variants['dsl'].value: + env.set("CUDAARCHS", self.spec.variants['cuda_arch'].value[0]) + env.unset("CUDAHOSTCXX") + env.set("BOOST_ROOT", self.spec['boost'].prefix) + + def configure_args(self): + args = super().configure_args() + super_libs = args.pop() + + libs = LibraryList([]) + flags = defaultdict(list) + + # Check for DSL variants and set corresponding Liskov options + dsl = self.spec.variants['dsl'].value + if dsl != ('none', ): + if 'substitute' in dsl: + args.append('--enable-liskov=substitute') + elif 'verify' in dsl: + args.append('--enable-liskov=verify') + elif 'serialize' in dsl: + raise error.UnsupportedPlatformError( + 'serialize mode is not supported yet by icon-liskov') + + if 'lam' in dsl: + args.append('--enable-dsl-local') + if 'nvtx' in dsl: + args.append('--enable-nvtx') + if 'fused' in dsl: + raise error.UnsupportedPlatformError( + 'liskov does not support fusing just yet') + + flags['LOC_GT4PY'].append(self.spec['py-gt4py'].prefix) + flags['LOC_ICON4PY_BIN'].append(self.spec['py-icon4py'].prefix) + + flags['LOC_ICON4PY_ATM_DYN_ICONAM'].append( + self.spec['py-icon4py:atm_dyn_iconam'].headers.directories[0]) + + if self.spec['py-icon4py'].version < Version("0.0.4"): + flags['LOC_ICON4PY_UTILS'].append( + os.path.dirname( + self.spec['py-icon4py:utils'].headers.directories[0])) + else: + flags['LOC_ICON4PY_TOOLS'].append( + self.spec['py-icon4py:tools'].headers.directories[0]) + if self.spec['py-icon4py'].version > Version("0.0.7"): + flags['LOC_ICON4PY_DIFFUSION'].append( + self.spec['py-icon4py:diffusion'].headers. + directories[0]) + flags['LOC_ICON4PY_INTERPOLATION'].append( + self.spec['py-icon4py:interpolation'].headers. + directories[0]) + if self.spec['py-icon4py'].version > Version("0.0.8"): + flags['LOC_ICON4PY_ADVECTION'].append( + self.spec['py-icon4py:advection'].headers. + directories[0]) + flags['LOC_GRIDTOOLS'].append( + self.spec['py-gridtools-cpp:data'].headers.directories[0]) + flags['GT4PYNVCFLAGS'] = flags['NVCFLAGS'] + + args.extend([ + "{0}={1}".format(name, " ".join(value)) + for name, value in flags.items() + ]) + args.append(f"{super_libs} {libs.link_flags}") + return args diff --git a/repos/c2sm/packages/icon-ham/package.py b/repos/c2sm/packages/icon-ham/package.py index 038cc18a5b..8ba3b6222b 100644 --- a/repos/c2sm/packages/icon-ham/package.py +++ b/repos/c2sm/packages/icon-ham/package.py @@ -1,8 +1,10 @@ from spack import * -from spack.pkg.c2sm.icon import Icon as C2SMIcon +from spack.pkg.builtin.icon import Icon as SpackIcon -class IconHam(C2SMIcon): +class IconHam(SpackIcon): + + maintainers('stelliom') @run_before('build') def generate_hammoz_nml(self): diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 96c5582f49..facf46bf0c 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -1,21 +1,13 @@ -import os, subprocess, glob, re +from spack.pkg.builtin.icon import Icon as SpackIcon +import os +import re +import glob from collections import defaultdict - -from llnl.util import lang, filesystem, tty -from spack.util.environment import is_system_path, dump_environment -from spack.util.executable import which_string, which +from llnl.util import tty +from spack.util.environment import is_system_path import spack.error as error -def validate_variant_dsl(pkg, name, value): - set_mutual_excl = set(['substitute', 'verify', 'serialize']) - set_input_var = set(value) - if len(set_mutual_excl.intersection(set_input_var)) > 1: - raise error.SpecError( - 'Cannot have more than one of (substitute, verify, serialize) in the same build' - ) - - def check_variant_fcgroup(fcgroup): pattern = re.compile(r"^[A-Z]+\..+\..") # fcgroup is False as default @@ -37,73 +29,38 @@ def check_variant_extra_config_args(extra_config_arg): return False -class Icon(AutotoolsPackage, CudaPackage): - """Icosahedral Nonhydrostatic Weather and Climate Model.""" - - homepage = "https://www.icon-model.org" - url = "https://gitlab.dkrz.de/icon/icon-model/-/archive/icon-2024.01-public/icon-model-icon-2024.01-public.tar.gz" - git = 'git@gitlab.dkrz.de:icon/icon.git' +class Icon(SpackIcon): + git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' - maintainers = ['jonasjucker', 'dominichofer'] + maintainers('jonasjucker', 'huppd') version('develop', submodules=True) - version("2024.01-1", tag="icon-2024.01-1", submodules=True) - version('exclaim-master', - branch='master', - git='git@github.com:C2SM/icon-exclaim.git', - submodules=True) - version('exclaim', - branch='icon-dsl', - git='git@github.com:C2SM/icon-exclaim.git', - submodules=True) - version('nwp-master', - git='git@gitlab.dkrz.de:icon/icon-nwp.git', + version("2024.01-1", + tag="icon-2024.01-1", + git='git@gitlab.dkrz.de:icon/icon.git', submodules=True) + version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) + version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) + version('nwp-master', submodules=True) # The variants' default follow those of ICON # as described here # https://gitlab.dkrz.de/icon/icon/-/blob/icon-2024.01/configure?ref_type=tags#L1492-1638 # Model Features: - variant('atmo', - default=True, - description='Enable the atmosphere component') - variant('les', - default=True, - description='Enable the Large-Eddy Simulation component') - variant('upatmo', - default=True, - description='Enable the upper atmosphere component') - variant('ocean', default=True, description='Enable the ocean component') - variant('jsbach', default=True, description='Enable the land component') - variant('waves', - default=False, - description='Enable the surface wave component') - variant('coupling', default=True, description='Enable the coupling') - variant('aes', default=True, description='Enable the AES physics package') - variant('nwp', default=True, description='Enable the NWP physics package') - variant('ecrad', - default=False, - description='Enable usage of the ECMWF radiation scheme') - variant('rte-rrtmgp', - default=True, - description='Enable usage of the RTE+RRTMGP toolbox ' - 'for radiation calculations') variant('dace', default=False, description='Enable the DACE modules for data assimilation') + requires("+mpi", when="+dace") + variant('emvorado', default=False, description='Enable the radar forward operator EMVORADO') - variant('art', - default=False, - description='Enable the aerosols and reactive trace component ART') + requires("+mpi", when="+emvorado") + variant('art-gpl', default=False, description='Enable GPL-licensed code parts of the ART component') - variant('comin', - default=False, - description='Enable the ICON community interfaces') variant( 'acm-license', default=False, @@ -111,9 +68,6 @@ class Icon(AutotoolsPackage, CudaPackage): 'Enable code parts that require accepting the ACM Software License') # Infrastructural Features: - variant('mpi', - default=True, - description='Enable MPI (parallelization) support') variant( 'active-target-sync', default=False, @@ -123,33 +77,10 @@ class Icon(AutotoolsPackage, CudaPackage): variant('async-io-rma', default=True, description='Enable remote memory access (RMA) for async I/O') - variant('mpi-gpu', - default=False, - description='Enable usage of the GPU-aware MPI features') - variant('openmp', default=False, description='Enable OpenMP support') - variant('gpu', - default='no', - values=('openacc+cuda', 'no'), - description='Enable GPU support') variant('realloc-buf', default=False, description='Enable reallocatable communication buffer') - variant('grib2', default=False, description='Enable GRIB2 I/O') - variant('parallel-netcdf', - default=False, - description='Enable usage of the parallel features of NetCDF') variant('sct', default=False, description='Enable the SCT timer') - variant('yaxt', default=False, description='Enable the YAXT data exchange') - - serialization_values = ('read', 'perturb', 'create') - variant('serialization', - default='none', - values=('none', ) + serialization_values, - description='Enable the Serialbox2 serialization') - variant('testbed', - default=False, - description='Enable ICON Testbed infrastructure') - variant( 'extra-config-args', default='none', @@ -158,26 +89,22 @@ class Icon(AutotoolsPackage, CudaPackage): description= 'Inject any configure argument not yet available as variant\nUse this feature cautiously, as injecting non-variant configure arguments may potentially disrupt the build process' ) - variant('comin', - default=False, - description='Enable usage of ComIn toolbox ' - 'for building plugins.') # Optimization Features: - variant('loop-exchange', default=True, description='Enable loop exchange') + variant('loop-exchange', default=False, description='Enable loop exchange') variant('vectorized-lrtm', default=False, description='Enable the parallelization-invariant version of LRTM') - variant('mixed-precision', - default=False, - description='Enable mixed precision dycore') variant( 'pgi-inlib', default=False, description= 'Enable PGI/NVIDIA cross-file function inlining via an inline library') variant('nccl', default=False, description='Enable NCCL for communication') + variant('cuda-graphs', default=False, description='Enable CUDA graphs.') + requires('%nvhpc@23.3:', when='+cuda-graphs') + variant( 'fcgroup', default='none', @@ -191,92 +118,37 @@ class Icon(AutotoolsPackage, CudaPackage): variant('silent-rules', default=True, description='Enable silent-rules for build-process') - - variant( - 'pytorch', - default=False, - description= - 'Build with pytorch for inference with machine-learning models. Experimental, needs non-standard codebase!' - ) - variant( 'eccodes-definitions', default=False, description= 'Enable extension of eccodes with center specific definition files') - # EXCLAIM-GT4Py specific features: - dsl_values = ('substitute', 'verify', 'serialize', 'fused', 'nvtx', 'lam') - variant('dsl', - default='none', - validator=validate_variant_dsl, - values=('none', ) + dsl_values, - description='Build with GT4Py dynamical core', - multi=True) - - for x in dsl_values: - depends_on('py-icon4py', when='dsl={0}'.format(x)) - depends_on('py-gridtools-cpp', when='dsl={0}'.format(x)) - depends_on('boost', when='dsl={0}'.format(x)) - conflicts('^python@:3.9,3.11:', when='dsl={0}'.format(x)) - - depends_on('pytorch-fortran', when='+pytorch') - - depends_on('libfyaml', when='+coupling') - depends_on('libxml2', when='+art') - - for x in serialization_values: + depends_on('cosmo-eccodes-definitions', + type='run', + when='+eccodes-definitions') + with when('+emvorado'): + depends_on('eccodes +fortran') + depends_on('hdf5 +szip +hl +fortran') + depends_on('zlib') # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + depends_on('eccodes %nvhpc', when='%nvhpc') + depends_on('eccodes %gcc', when='%gcc') + + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + for __x in SpackIcon.serialization_values: depends_on('serialbox+fortran %nvhpc', - when='serialization={0} %nvhpc'.format(x)) + when='serialization={0} %nvhpc'.format(__x)) depends_on('serialbox+fortran %gcc', - when='serialization={0} %gcc'.format(x)) - - depends_on('eccodes +fortran', when='+emvorado') - depends_on('eccodes', when='+grib2') - depends_on('cosmo-eccodes-definitions', - type=('build', 'run'), - when='+eccodes-definitions') - - depends_on('lapack') - depends_on('blas') - depends_on('netcdf-fortran') + when='serialization={0} %gcc'.format(__x)) - depends_on('netcdf-c') - depends_on('netcdf-c+mpi', when='+parallel-netcdf') + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + depends_on('netcdf-fortran %nvhpc', when='%nvhpc') + depends_on('netcdf-fortran %gcc', when='%gcc') - depends_on('hdf5 +szip +hl +fortran', when='+emvorado') depends_on('hdf5 +szip', when='+sct') - depends_on('zlib', when='+emvorado') - depends_on('mpi', when='+mpi') - - depends_on('python', type='build') - depends_on('perl', type='build') - depends_on('cmake@3.18:', type='build') - - conflicts('+dace', when='~mpi') - conflicts('+emvorado', when='~mpi') - conflicts('+cuda', when='%gcc') - - # The gpu=openacc+cuda relies on the cuda variant - conflicts('~cuda', when='gpu=openacc+cuda') - conflicts('+cuda', when='gpu=no') - - conflicts('+cuda-graphs', when='%cce') - conflicts('+cuda-graphs', when='%gcc') - conflicts('+cuda-graphs', when='%intel') - conflicts('+cuda-graphs', when='%pgi') - conflicts('+cuda-graphs', when='%nvhpc@:23.2') - - conflicts('+loop-exchange', when='gpu=openacc+cuda') - - # Flag to mark if we build out-of-source - # Needed to trigger sync of input files for experiments - out_of_source_build = False - out_of_source_configure_directory = '' - # patch_libtool is a function from Autotoolspackage. # For BB we cannot use it because it finds all files # named "libtool". spack-c2sm is cloned into icon-repo, @@ -284,226 +156,39 @@ class Icon(AutotoolsPackage, CudaPackage): # also the folder where libtool package itself is installed. patch_libtool = False - def setup_build_environment(self, env): - # help cmake to build dsl-stencils - if 'none' not in self.spec.variants['dsl'].value: - env.set("CUDAARCHS", self.spec.variants['cuda_arch'].value[0]) - env.unset("CUDAHOSTCXX") - env.set("BOOST_ROOT", self.spec['boost'].prefix) - def configure_args(self): - args = ['--disable-rpaths'] - flags = defaultdict(list) + args = super().configure_args() + super_libs = args.pop() + libs = LibraryList([]) + flags = defaultdict(list) for x in [ - 'atmo', - 'les', - 'upatmo', - 'ocean', - 'jsbach', - 'waves', - 'coupling', - 'aes', - 'nwp', - 'ecrad', - 'rte-rrtmgp', 'dace', 'emvorado', - 'art', 'art-gpl', - 'comin', 'acm-license', - 'mpi', 'active-target-sync', 'async-io-rma', - 'mpi-gpu', - 'openmp', 'realloc-buf', - 'grib2', 'parallel-netcdf', 'sct', - 'yaxt', - 'testbed', 'loop-exchange', 'vectorized-lrtm', - 'mixed-precision', 'pgi-inlib', 'nccl', 'cuda-graphs', 'silent-rules', - 'comin', ]: args += self.enable_or_disable(x) - if self.compiler.name == 'gcc': - flags['CFLAGS'].append('-g') - flags['ICON_CFLAGS'].append('-O3') - flags['ICON_BUNDLED_CFLAGS'].append('-O2') - flags['FCFLAGS'].extend([ - '-g', - '-fmodule-private', - '-fimplicit-none', - '-fmax-identifier-length=63', - '-Wall', - '-Wcharacter-truncation', - '-Wconversion', - '-Wunderflow', - '-Wunused-parameter', - '-Wno-surprising', - '-fall-intrinsics', - ]) - flags['ICON_FCFLAGS'].extend([ - '-O2', '-fbacktrace', '-fbounds-check', - '-fstack-protector-all', '-finit-real=nan', - '-finit-integer=-2147483648', '-finit-character=127' - ]) - flags['ICON_OCEAN_FCFLAGS'].append('-O3') - - # Version-specific workarounds: - fc_version = self.compiler.version - if fc_version >= ver(10): - flags['ICON_FCFLAGS'].append('-fallow-argument-mismatch') - flags['ICON_OCEAN_FCFLAGS'].append('-fallow-argument-mismatch') - if '+ecrad' in self.spec: - # For externals/ecrad/ifsaux/random_numbers_mix.F90: - flags['ICON_ECRAD_FCFLAGS'].append('-fallow-invalid-boz') - elif self.compiler.name == 'intel': - flags['CFLAGS'].extend( - ['-g', '-gdwarf-4', '-O3', '-qno-opt-dynamic-align', '-ftz']) - flags['FCFLAGS'].extend( - ['-g', '-gdwarf-4', '-traceback', '-fp-model source']) - flags['ICON_FCFLAGS'].extend( - ['-O2', '-assume realloc_lhs', '-ftz']) - flags['ICON_OCEAN_FCFLAGS'].extend([ - '-O3', '-assume norealloc_lhs', '-reentrancy threaded', - '-qopt-report-file=stdout', '-qopt-report=0', - '-qopt-report-phase=vec' - ]) - args.append('--enable-intel-consistency') - elif self.compiler.name == 'nag': - flags['CFLAGS'].append('-g') - flags['ICON_CFLAGS'].append('-O3') - flags['ICON_BUNDLED_CFLAGS'].append('-O2') - flags['FCFLAGS'].extend([ - '-g', '-Wc,-g', '-O0', '-colour', '-f2008', '-w=uep', - '-float-store', '-nan' - ]) - if '~openmp' in self.spec: - # The -openmp option is incompatible with the -gline option: - flags['FCFLAGS'].append('-gline') - flags['ICON_FCFLAGS'].extend([ - '-Wc,-pipe', - '-Wc,--param,max-vartrack-size=200000000', - '-Wc,-mno-fma', - # Spack compiler wrapper (see the respective compilers.yaml) - # injects '-mismatch', which is incompatible with '-C=calls' - # Therefore, we specify the following flags instead of a single - # '-C=all', which implies '-C=calls'. - '-C=alias', - '-C=array', - '-C=bits', - '-C=dangling', - '-C=do', - '-C=intovf', - '-C=present', - '-C=pointer', - '-C=recursion' - ]) - flags['ICON_BUNDLED_FCFLAGS'] = [] - elif self.compiler.name in ['pgi', 'nvhpc']: - flags['CFLAGS'].extend(['-g', '-O2']) - flags['FCFLAGS'].extend( - ['-g', '-O', '-Mrecursive', '-Mallocatable=03', '-Mbackslash']) - - if self.spec.variants['gpu'].value == 'openacc+cuda': - flags['FCFLAGS'].extend([ - '-acc=verystrict', '-Minfo=accel,inline', - '-gpu=cc{0}'.format( - self.spec.variants['cuda_arch'].value[0]) - ]) - elif self.compiler.name == 'cce': - flags['CFLAGS'].append('-g') - flags['ICON_CFLAGS'].append('-O3') - if self.spec.satisfies('%cce@13.0.0+coupling'): - # For externals/yac/tests/test_interpolation_method_conserv.c: - flags['ICON_YAC_CFLAGS'].append('-O2') - flags['FCFLAGS'].extend([ - '-hadd_paren', '-r am', '-Ktrap=divz,ovf,inv', - '-hflex_mp=intolerant', '-hfp0', '-O0' - ]) - if self.spec.variants['gpu'].value == 'openacc+cuda': - flags['FCFLAGS'].extend(['-hacc']) - elif self.compiler.name == 'aocc': - flags['CFLAGS'].extend(['-g', '-O2']) - flags['FCFLAGS'].extend(['-g', '-O2']) - if self.spec.satisfies('+yaxt'): - # Enable the PGI/Cray (NO_2D_PARAM) workaround for the test - # suite of the bundled YAXT (apply also when not self.run_tests - # to make sure we get identical installations): - flags['ICON_YAXT_FCFLAGS'].append('-DNO_2D_PARAM') - else: - flags['CFLAGS'].extend(['-g', '-O2']) - flags['FCFLAGS'].extend(['-g', '-O2']) - - if '+coupling' in self.spec or '+art' in self.spec: - xml2_spec = self.spec['libxml2'] - libs += xml2_spec.libs - # Account for the case when libxml2 is an external package installed - # to a system directory, which means that Spack will not inject the - # required -I flag with the compiler wrapper: - if is_system_path(xml2_spec.prefix): - xml2_headers = xml2_spec.headers - # We, however, should filter the pure system directories out: - xml2_headers.directories = [ - d for d in xml2_headers.directories - if not is_system_path(d) - ] - flags['CPPFLAGS'].append(xml2_headers.include_flags) - - if '+coupling' in self.spec: - libs += self.spec['libfyaml'].libs - - serialization = self.spec.variants['serialization'].value - if serialization == 'none': - args.append('--disable-serialization') - else: - args.extend([ - '--enable-serialization={0}'.format(serialization), - 'SB2PP={0}'.format(self.spec['serialbox'].pp_ser) - ]) - libs += self.spec['serialbox:fortran'].libs - if '+emvorado' in self.spec: libs += self.spec['eccodes:fortran'].libs - - if '+grib2' in self.spec: - libs += self.spec['eccodes:c'].libs - - libs += self.spec['lapack:fortran'].libs - libs += self.spec['blas:fortran'].libs - libs += self.spec['netcdf-fortran'].libs - libs += self.spec['netcdf-c'].libs - - if '+emvorado' in self.spec: libs += self.spec['hdf5:fortran,hl'].libs - elif '+sct' in self.spec: - libs += self.spec['hdf5'].libs - - if '+emvorado' in self.spec: libs += self.spec['zlib'].libs - if '+mpi' in self.spec: - args.extend([ - 'CC=' + self.spec['mpi'].mpicc, - 'FC=' + self.spec['mpi'].mpifc, - # We cannot provide a universal value for MPI_LAUNCH, therefore - # we have to disable the MPI checks: - '--disable-mpi-checks' - ]) - - if '+pytorch' in self.spec: - libs += self.spec['pytorch-fortran'].libs + if '+sct' in self.spec: + libs += self.spec['hdf5'].libs fcgroup = self.spec.variants['fcgroup'].value # ('none',) is the values spack assign if fcgroup is not set @@ -511,79 +196,6 @@ def configure_args(self): args.extend(self.fcgroup_to_config_arg()) flags.update(self.fcgroup_to_config_var()) - gpu = self.spec.variants['gpu'].value - if gpu == 'no': - args.append('--disable-gpu') - else: - args.extend([ - '--enable-gpu={0}'.format(gpu), - 'NVCC={0}'.format(self.spec['cuda'].prefix.bin.nvcc) - ]) - - # -cuda is an umbrella flag for NVHPC - # that pulls in all the needed CUDA libraries and wrappers - # It can't go into libs as spack prepends -l - flags['LDFLAGS'].append('-cuda') - - cuda_host_compiler = self.compiler.cxx - cuda_host_compiler_stdcxx_libs = self.compiler.stdcxx_libs - - flags['NVCFLAGS'].extend([ - '-g', '-O3', - '-arch=sm_{0}'.format(self.spec.variants['cuda_arch'].value[0]) - ]) - # cuda_host_compiler_stdcxx_libs might contain compiler-specific - # flags (i.e. not the linker -l flags), therefore we put - # the value to the config_flags directly. - flags['LIBS'].extend(cuda_host_compiler_stdcxx_libs) - - # Check for DSL variants and set corresponding Liskov options - dsl = self.spec.variants['dsl'].value - if dsl != ('none', ): - if 'substitute' in dsl: - args.append('--enable-liskov=substitute') - elif 'verify' in dsl: - args.append('--enable-liskov=verify') - elif 'serialize' in dsl: - raise error.UnsupportedPlatformError( - 'serialize mode is not supported yet by icon-liskov') - - if 'lam' in dsl: - args.append('--enable-dsl-local') - if 'nvtx' in dsl: - args.append('--enable-nvtx') - if 'fused' in dsl: - raise error.UnsupportedPlatformError( - 'liskov does not support fusing just yet') - - flags['LOC_GT4PY'].append(self.spec['py-gt4py'].prefix) - flags['LOC_ICON4PY_BIN'].append(self.spec['py-icon4py'].prefix) - - flags['LOC_ICON4PY_ATM_DYN_ICONAM'].append( - self.spec['py-icon4py:atm_dyn_iconam'].headers.directories[0]) - - if self.spec['py-icon4py'].version < Version("0.0.4"): - flags['LOC_ICON4PY_UTILS'].append( - os.path.dirname( - self.spec['py-icon4py:utils'].headers.directories[0])) - else: - flags['LOC_ICON4PY_TOOLS'].append( - self.spec['py-icon4py:tools'].headers.directories[0]) - if self.spec['py-icon4py'].version > Version("0.0.7"): - flags['LOC_ICON4PY_DIFFUSION'].append( - self.spec['py-icon4py:diffusion'].headers. - directories[0]) - flags['LOC_ICON4PY_INTERPOLATION'].append( - self.spec['py-icon4py:interpolation'].headers. - directories[0]) - if self.spec['py-icon4py'].version > Version("0.0.8"): - flags['LOC_ICON4PY_ADVECTION'].append( - self.spec['py-icon4py:advection'].headers. - directories[0]) - flags['LOC_GRIDTOOLS'].append( - self.spec['py-gridtools-cpp:data'].headers.directories[0]) - flags['GT4PYNVCFLAGS'] = flags['NVCFLAGS'] - # add configure arguments not yet available as variant extra_config_args = self.spec.variants['extra-config-args'].value if extra_config_args != ('none', ): @@ -611,9 +223,10 @@ def configure_args(self): ]) args.extend([ - '{0}={1}'.format(var, ' '.join(val)) for var, val in flags.items() + "{0}={1}".format(name, " ".join(value)) + for name, value in flags.items() ]) - + args.append(f"{super_libs} {libs.link_flags}") return args def fcgroup_to_config_arg(self): @@ -650,94 +263,6 @@ def validate_extra_config_args(self, arg): f'with the existing variant {variant_from_arg}. Set this variant instead.' ) - def check(self): - # By default "check" calls make with targets "check" and "test". - # This testing is beyond the scope of BuildBot test at CSCS. - # Therefore override this function, saves a lot of time too. - pass - - @run_after('install') - @on_package_attributes(run_tests=True) - def checksuite(self): - # script needs cdo to work, but not listed as dep of ICON - test_script = 'scripts/spack/test.py' - if os.path.exists(test_script): - test_py = Executable(test_script) - - # test.py fails if PYTHONHOME has any value, - # even '' or ' ' is failing, therefore delete - # it temporary from env - if 'PYTHONHOME' in os.environ: - PYTHONHOME = os.environ['PYTHONHOME'] - os.environ.pop('PYTHONHOME') - pythonhome_is_set = True - else: - pythonhome_is_set = False - - with open('spec.yaml', mode='w') as f: - f.write(self.spec.to_yaml()) - test_py('--spec', 'spec.yaml', fail_on_error=True) - - # restore PYTHONHOME after test.py - if pythonhome_is_set: - os.environ['PYTHONHOME'] = PYTHONHOME - else: - tty.warn('Cannot find test.py -> skipping tests') - - @property - def archive_files(self): - # Archive files that are normally archived for AutotoolsPackage: - archive = list(super(Icon, self).archive_files) - # Archive makefiles: - archive.extend( - [join_path(self.build_directory, f) for f in ['Makefile', '*.mk']]) - return archive - - @property - def build_directory(self): - """Overrides function from spack.build_system.autotools - - By default build_directory is identical as configure_directory - To enable out-of-source builds this is not the case anymore - """ - - return self.stage.source_path - - @property - def configure_directory(self): - """Returns the directory where 'configure' resides. - - Overides function from spack.build_systems.autotools - - """ - - source_path = self.build_directory - - # dev_path is indicator for dev-build or develop - # only case when out-of-source build are possible - if "dev_path" in self.spec.variants: - Git = which('git', required=True) - git_root = Git('rev-parse', - '--show-toplevel', - output=str, - fail_on_error=True).replace("\n", "") - if git_root != source_path: - # mark out-of-source build for function - # copy_runscript_related_input_files - self.out_of_source_build = True - self.out_of_source_configure_directory = git_root - return git_root - - return source_path - - @run_before('configure') - def report_out_of_source_directories(self): - if self.out_of_source_build: - tty.info(f'build-directory: {self.build_directory}') - tty.info( - f'configure-directory: {self.out_of_source_configure_directory}' - ) - def configure(self, spec, prefix): if os.path.exists( os.path.join(self.build_directory, @@ -758,7 +283,7 @@ def build_uses_same_spec(self): $ spack dev-build icon @develop ~dace $ spack dev-build icon @develop +dace - + configure is skipped for the latter. """ @@ -789,10 +314,12 @@ def build_uses_same_spec(self): @run_after('configure') def copy_runscript_related_input_files(self): - if self.out_of_source_build: - with working_dir(self.build_directory): + with working_dir(self.build_directory): + icon_dir = self.configure_directory + # only synchronize if out-of-source build + if os.path.abspath(icon_dir) != os.path.abspath( + self.build_directory): Rsync = which('rsync', required=True) - icon_dir = self.configure_directory Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", "--exclude=.*", "--exclude=standard_*") Rsync("-uavz", f"{icon_dir}/externals", ".", "--exclude=.git", diff --git a/repos/c2sm/packages/icontools/package.py b/repos/c2sm/packages/icontools/package.py index 9432ff2a31..dafea4b512 100644 --- a/repos/c2sm/packages/icontools/package.py +++ b/repos/c2sm/packages/icontools/package.py @@ -39,15 +39,17 @@ class Icontools(AutotoolsPackage): depends_on('libtool', type='build') depends_on('m4', type='build') - depends_on('netcdf-fortran', type=('build', 'link')) - depends_on('netcdf-c ~mpi', type=('build', 'link')) - depends_on('hdf5 ~mpi +hl', type=('build', 'link')) + depends_on( + 'netcdf-fortran %gcc' + ) # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on('netcdf-c ~mpi') + depends_on('hdf5 ~mpi +hl') depends_on( 'mpi', type=('build', 'link', 'run'), ) depends_on('eccodes +fortran ~aec', type=('build', 'link', 'run')) - depends_on('jasper@1.900.1', type=('build', 'link')) + depends_on('jasper@1.900.1') variant( 'slave', diff --git a/repos/c2sm/packages/libtorch/package.py b/repos/c2sm/packages/libtorch/package.py deleted file mode 100644 index 55e11ec2ec..0000000000 --- a/repos/c2sm/packages/libtorch/package.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * -import shutil - - -class Libtorch(Package): - ''' - PyTorch is a Python package that provides two high-level features: - - - Tensor computation (like NumPy) with strong GPU acceleration - - Deep neural networks built on a tape-based autograd system - ''' - - homepage = "https://pytorch.org/" - url = "https://download.pytorch.org/libtorch/cu117/libtorch-cxx11-abi-shared-with-deps-2.0.1%2Bcu117.zip" - - maintainers = ['juckerj'] - - version('2.0.1', - url=url, - sha256= - '262f723ee5a2caac977e089bc06e9d840ca33d70706fbd4a2fca04995bb94eb4') - - phases = ['install'] - - def install(self, spec, prefix): - # can't use Spack convenience-function 'install_tree' because it uses - # shutil.copy2 under the hood. For an unknown reason installing from - # the unzipped tarbal only works using shutil.copy. - shutil.copytree('lib', - prefix.lib, - symlinks=True, - copy_function=shutil.copy) - shutil.copytree('include', - prefix.include, - symlinks=True, - copy_function=shutil.copy) - shutil.copytree('share', - prefix.share, - symlinks=True, - copy_function=shutil.copy) - shutil.copytree('bin', - prefix.bin, - symlinks=True, - copy_function=shutil.copy) diff --git a/repos/c2sm/packages/metkit/package.py b/repos/c2sm/packages/metkit/package.py deleted file mode 100644 index 42733610db..0000000000 --- a/repos/c2sm/packages/metkit/package.py +++ /dev/null @@ -1,10 +0,0 @@ -from spack.package import * - -from spack.pkg.builtin.metkit import Metkit as SpackMetkit - - -class Metkit(SpackMetkit): - - # This file can be removed when this PR https://github.com/spack/spack/pull/42871 - # is included in a release on spack and this is used by spack-c2sm. - depends_on("eckit@:1.21", when="@:1.10") diff --git a/repos/c2sm/packages/py-frozendict/package.py b/repos/c2sm/packages/py-frozendict/package.py deleted file mode 100644 index 54f3a85f2b..0000000000 --- a/repos/c2sm/packages/py-frozendict/package.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * -from spack.pkg.builtin.py_frozendict import PyFrozendict as SpackPyFrozendict - - -class PyFrozendict(SpackPyFrozendict): - - version('2.4.0', - sha256= - 'c26758198e403337933a92b01f417a8240c954f553e1d4b5e0f8e39d9c8e3f0a') - - # TODO: remove this extension once we have a more recent - # version than v0.21.1 - def setup_build_environment(self, env): - # C extension is not supported for 3.11+. See also - # https://github.com/Marco-Sulla/python-frozendict/issues/68 - if self.spec.satisfies("^python@3.11:"): - env.set("FROZENDICT_PURE_PY", "1") diff --git a/repos/c2sm/packages/py-gt4py/package.py b/repos/c2sm/packages/py-gt4py/package.py index 8501ec9c59..5b2c95cedf 100644 --- a/repos/c2sm/packages/py-gt4py/package.py +++ b/repos/c2sm/packages/py-gt4py/package.py @@ -2,7 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - +import os from spack import * @@ -85,5 +85,13 @@ class PyGt4py(PythonPackage): depends_on('py-pytest-xdist', type=('build', 'run')) def test(self): + # workaround for not finding own python module + python_spec = self.spec['python'] + python_version = python_spec.version.up_to(2) + install_path = join_path(self.prefix, 'lib', f"python{python_version}", + 'site-packages') + os.environ[ + 'PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" + python('-m', 'pytest', '-v', '-s', '-n', 'auto', '-k', '.run_gtfn]', 'tests/next_tests', 'tests/eve_tests') diff --git a/repos/c2sm/packages/py-hatchling/package.py b/repos/c2sm/packages/py-hatchling/package.py index eca6c6e181..2afe34325d 100644 --- a/repos/c2sm/packages/py-hatchling/package.py +++ b/repos/c2sm/packages/py-hatchling/package.py @@ -1,38 +1,50 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * - class PyHatchling(PythonPackage): - """This is the extensible, standards compliant - build backend used by Hatch. - """ - - # FIXME: Add a proper url for your package's homepage here. + """Modern, extensible Python build backend.""" homepage = "https://hatch.pypa.io/latest/" + pypi = "hatchling/hatchling-1.4.1.tar.gz" + git = "https://github.com/pypa/hatch" - pypi = "hatchling/hatchling-1.11.1.tar.gz" - - maintainers = ['samkellerhals'] + license("MIT", checked_by="tgamblin") - version('1.11.1', + version("1.21.0", sha256= - '9f84361f70cf3a7ab9543b0c3ecc64211ed2ba8a606a71eb6a473c1c9b08e1d0') - - depends_on('py-setuptools', type='build') + "5c086772357a50723b825fd5da5278ac7e3697cdf7797d07541a6c90b6ff754c") + version("1.18.0", + sha256= + "50e99c3110ce0afc3f7bdbadff1c71c17758e476731c27607940cfa6686489ca") + version("1.17.0", + sha256= + "b1244db3f45b4ef5a00106a46612da107cdfaf85f1580b8e1c059fefc98b0930") + version("1.14.0", + sha256= + "462ea91df03ff5d52813b5613fec1313a1a2059d2e37343e572b3f979867c5da") + version("1.13.0", + sha256= + "f8d275a2cc720735286b7c2e2bc35da05761e6d3695c2fa416550395f10c53c7") + version("1.10.0", + sha256= + "5d31f43dffaf6265c808e1b5353662ffa5146d844278b55caa6c7f74f427ec50") + version("1.8.1", + sha256= + "448b04b23faed669b2b565b998ac955af4feea66c5deed3a1212ac9399d2e1cd") + version("1.4.1", + sha256= + "13461b42876ade4f75ee5d2a2c656b288ca0aab7f048ef66657ef166996b2118") - depends_on('python@3.7:', type=('build', 'run')) + depends_on("py-editables@0.3:", type=("build", "run")) + depends_on("py-packaging@21.3:", type=("build", "run")) + depends_on("py-pathspec@0.10.1:", when="@1.9:", type=("build", "run")) + depends_on("py-pathspec@0.9:", type=("build", "run")) - # Be less strict with py-pluggy, otherwise concretizer - # fails due to 0.12 and 1.0 required at the same time - #depends_on('py-pluggy@1.0.0:', type=('build', 'run')) + #WORKAROUND: py-pluggy@1: and py-tox' py-pluggy@0.12.0:0 are incompatible + #depends_on("py-pluggy@1:", type=("build", "run")) + depends_on("py-pluggy@0.12:", type=("build", "run")) - depends_on('py-pluggy@0.12:1', type=('build', 'run')) - depends_on('py-pathspec@0.10.1:', type=('build', 'run')) - depends_on('py-tomli@1.2.2:', type=('build', 'run')) - depends_on('py-packaging@21.3:', type=('build', 'run')) - depends_on('py-editables@0.3:', type=('build', 'run')) - depends_on('py-importlib-metadata', type=('build', 'run')) + depends_on("py-tomli@1.2.2:", when="^python@:3.10", type=("build", "run")) + depends_on("py-trove-classifiers", when="@1.14:", type=("build", "run")) diff --git a/repos/c2sm/packages/py-icon4py/package.py b/repos/c2sm/packages/py-icon4py/package.py index 4e5072f92d..c9dea8fb2e 100644 --- a/repos/c2sm/packages/py-icon4py/package.py +++ b/repos/c2sm/packages/py-icon4py/package.py @@ -50,6 +50,13 @@ def setup_build_environment(self, env): env.set("CMAKE_INCLUDE_PATH", self.spec['boost'].prefix.include) def test(self): + # workaround for not finding own python module + python_spec = self.spec['python'] + python_version = python_spec.version.up_to(2) + install_path = join_path(self.prefix, 'lib', f"python{python_version}", + 'site-packages') + os.environ[ + 'PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" # check if all installed module can be imported super().test() # unit tests diff --git a/repos/c2sm/packages/py-lark/package.py b/repos/c2sm/packages/py-lark/package.py deleted file mode 100644 index 53cb7fe0ee..0000000000 --- a/repos/c2sm/packages/py-lark/package.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.package import * - - -# TODO: This package is taken from a more recent version of Spack -# Remove this package once Spack in spack-c2sm has been updated -class PyLark(PythonPackage): - """Lark is a modern general-purpose parsing library for Python.""" - - homepage = "https://github.com/lark-parser/lark/" - pypi = "lark/lark-1.0.0.tar.gz" - - maintainers = ['samkellerhals'] - - version("1.1.2", - sha256= - "7a8d0c07d663da9391d7faee1bf1d7df4998c47ca43a593cbef5c7566acd057a") - version("1.1.1", - sha256= - "5115193433051f079374c4f81059fa4bf2afa78cc87dd87817ed4435e8647c82") - version("1.1.0", - sha256= - "669eab99a9627b2b9e0c6fb97f23113c64d673c93d804bca40b05b2a765f13c0") - version("1.0.0", - sha256= - "2269dee215e6c689d5ce9d34fdc6e749d0c1c763add3fc7935938ebd7da159cb") - version("0.12.0", - sha256= - "7da76fcfddadabbbbfd949bbae221efd33938451d90b1fefbbc423c3cccf48ef") - version("0.11.3", - sha256= - "3100d9749b5a85735ec428b83100876a5da664804579e729c23a36341f961e7e") - version("0.11.1", - sha256= - "f2c6ed79ae128a89714bbaa4a6ecb61b6eec84d1b5d63b9195ad461762f96298") - version("0.11.0", - sha256= - "29868417eb190fe7d6b1ff6bcd9446903e0c73a1ca69cec58c92a01cae0abc24") - version("0.10.1", - sha256= - "98f2c6f8e41fe601fd103476eb759ac1ad4d3dc8094633133a16cef5a32b0f65") - - depends_on("python@3.6:", when="@1.0.0:") - depends_on("py-setuptools", type="build") diff --git a/repos/c2sm/packages/py-tabulate/package.py b/repos/c2sm/packages/py-tabulate/package.py index f3b0593285..b6a8f08b4b 100644 --- a/repos/c2sm/packages/py-tabulate/package.py +++ b/repos/c2sm/packages/py-tabulate/package.py @@ -10,6 +10,10 @@ class PyTabulate(SpackPyTabulate): """Pretty-print tabular data""" + #BACKPORT: Add missing versions + version("0.9.0", + sha256= + "0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c") version("0.8.10", sha256= "6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519") diff --git a/repos/c2sm/packages/pytorch-fortran-proxy/package.py b/repos/c2sm/packages/pytorch-fortran-proxy/package.py deleted file mode 100644 index 15f0292f88..0000000000 --- a/repos/c2sm/packages/pytorch-fortran-proxy/package.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# -from spack import * - - -class PytorchFortranProxy(CMakePackage): - ''' - Pytorch Fortran bindings - C++ Backend - - The goal of this code is to provide Fortran HPC codes with a simple way to use - Pytorch deep learning framework. We want Fortran developers to take advantage - of rich and optimized Torch ecosystem from within their existing codes. - ''' - - homepage = "https://github.com/alexeedm/pytorch-fortran" - url = "https://github.com/alexeedm/pytorch-fortran.git" - - version('0.4', git=url, tag='v0.4') - - maintainers = ['juckerj'] - - depends_on('cuda') - depends_on('libtorch') - depends_on('py-pybind11') - - root_cmakelists_dir = 'src/proxy_lib' - - def cmake_args(self): - args = [ - self.define('OPENACC', 1), - self.define('CUDA_TOOLKIT_ROOT_DIR', self.spec['cuda'].prefix), - self.define('TORCH_CUDA_ARCH_LIST', "6.0") - ] - - return args diff --git a/repos/c2sm/packages/pytorch-fortran/package.py b/repos/c2sm/packages/pytorch-fortran/package.py deleted file mode 100644 index 8c19f3ee9d..0000000000 --- a/repos/c2sm/packages/pytorch-fortran/package.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# -from spack import * -import os - - -class PytorchFortran(CMakePackage): - ''' - Pytorch Fortran bindings - Fortran Frontend - - The goal of this code is to provide Fortran HPC codes with a simple way to use - Pytorch deep learning framework. We want Fortran developers to take advantage - of rich and optimized Torch ecosystem from within their existing codes. - ''' - - homepage = "https://github.com/alexeedm/pytorch-fortran" - url = "https://github.com/alexeedm/pytorch-fortran.git" - - version('0.4', git=url, tag='v0.4') - - maintainers = ['juckerj'] - - depends_on('pytorch-fortran-proxy') - - root_cmakelists_dir = 'src/f90_bindings' - - def cmake_args(self): - args = [self.define('OPENACC', 1)] - - return args - - @property - def libs(self): - libraries = ['libpytorch_fort_proxy'] - - libs = find_libraries(libraries, - root=self.prefix, - shared=True, - recursive=True) - - if libs and len(libs) == len(libraries): - return libs - - msg = 'Unable to recursively locate shared {0} libraries in {1}' - raise spack.error.NoLibrariesError( - msg.format(self.spec.name, self.spec.prefix)) - - @run_after('install') - def link_fmod_into_include(self): - mod = 'torch_ftn.mod' - src = os.path.join(self.prefix.include, 'mod_files', mod) - dest = os.path.join(self.prefix.include, mod) - os.symlink(src, dest) diff --git a/repos/c2sm/packages/xpmem/package.py b/repos/c2sm/packages/xpmem/package.py deleted file mode 100644 index d1c19da4f2..0000000000 --- a/repos/c2sm/packages/xpmem/package.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.package import * - - -# Identical to spack v0.21.1 except for the part labeled as WORKAROUND -class Xpmem(AutotoolsPackage): - """XPMEM is a Linux kernel module that enables a process to map the memory - of another process into its virtual address space.""" - - # The README file of the repository says that the development was - # transferred to a new repository on GitLab: https://gitlab.com/hjelmn/xpmem - # However, it looks like that the repository on GitHub has a more recent - # version of the codebase. - homepage = "https://github.com/hjelmn/xpmem" - url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz" - git = "https://github.com/hjelmn/xpmem.git" - - maintainers("skosukhin") - - version("master", branch="master") - - # Versions starting 2.6.4 are neither tagged nor released in the repo - # (the choice of commits is based on the commit history of - # 'kernel/xpmem_private.h'): - version("2.6.5-36", commit="0d0bad4e1d07b38d53ecc8f20786bb1328c446da") - version("2.6.5", commit="4efeed9cbaabe971f3766d67cb108e2c3316d4b8") - version("2.6.4", commit="522054850e4d1479d69f50f7190d1548bf9749fd") - - # Released versions: - version("2.6.3", - sha256= - "ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54") - version("2.6.2", - sha256= - "2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b") - - variant("kernel-module", - default=True, - description="Enable building the kernel module") - - # Added RHEL 8.3 kernel support - # Here 2.6.5-36 referes to 2.6.5 version and 36th commit id - patch("xpmem_v2.6.5-36.patch", when="@2.6.5-36", level=1) - patch( - "https://github.com/hjelmn/xpmem/commit/cbd6e5bd3d2a1d3823c335ddcd3c57b94474f578.patch?full_index=1", - sha256= - "75299398b6c15546479bfbb8aa972431f58637fe2f0328196a26738bd7148140", - when="@2.6.5-36", - level=1, - ) - patch( - "https://github.com/hjelmn/xpmem/commit/7d346aaf1fdfc24d38cebb4ad107b7f5c43769e9.patch?full_index=1", - sha256= - "6be8c5f33d55c611924d8412253740f6f4b738e6d98e32981fa300d2ccbe99cc", - when="@2.6.5-36", - level=1, - ) - - depends_on("autoconf", type="build") - depends_on("automake", type="build") - depends_on("libtool", type="build") - depends_on("m4", type="build") - - # It will become possible to disable the kernel module only starting 2.6.6: - # https://github.com/hjelmn/xpmem/pull/24 - conflicts("~kernel-module", when="@:2.6.5") - - # Ideally, we should list all non-Linux-based platforms here: - conflicts("+kernel-module", when="platform=darwin") - - # All compilers except for gcc are in conflict with +kernel-module: - #requires("%gcc", when="+kernel-module", msg="Linux kernel module must be compiled with gcc") - - #WORKAROUND: The above line is not working as expected. - # The below line tires to express the same thing. - # But it is only true in a context with nvhpc and gcc as the only compilers. - conflicts("%nvhpc", when="+kernel-module") - - def autoreconf(self, spec, prefix): - Executable("./autogen.sh")() - - @run_before("build") - def override_kernel_compiler(self): - # Override the compiler for kernel module source files. We need - # this additional argument for all installation phases. - if "+kernel-module" in self.spec: - make.add_default_arg("CC={0}".format(spack_cc)) - - def configure_args(self): - args = [] - - if "~kernel-module" in self.spec: - # The kernel module is enabled by default. An attempt of explicit - # enabling with '--enable-kernel-module' disables the module. - args.append("--disable-kernel-module") - - if self.spec.satisfies("@:2.6.5"): - fmt = self.spec.format - # The following arguments will not be needed starting 2.6.6: - # https://github.com/hjelmn/xpmem/pull/18 - args.extend([ - fmt("--with-default-prefix={prefix}"), - fmt("--with-module={prefix.share}/Modules/{name}/{version}"), - ]) - - return args - - @when("@:2.6.5") - def install(self, spec, prefix): - with working_dir(self.build_directory): - # Override the hardcoded prefix for 'cray-xpmem.conf' - make( - "ldsoconfdir={0}".format( - self.spec.prefix.etc.join("ld.so.conf.d")), - *self.install_targets, - ) diff --git a/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch b/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch deleted file mode 100644 index a17fba4ad1..0000000000 --- a/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- xpmem/kernel/xpmem_pfn.c 2021-02-08 22:49:05.321501753 -0800 -+++ xpmem.patch/kernel/xpmem_pfn.c 2021-02-08 23:30:30.875309634 -0800 -@@ -251,6 +251,12 @@ - cpu_to_node(task_cpu(current)) != cpu_to_node(task_cpu(src_task))) { - #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0) - saved_mask = current->cpus_mask; -+#elif LINUX_VERSION_CODE == KERNEL_VERSION(4,18, 0) -+ #ifdef RHEL_RELEASE_CODE -+ #if RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3) -+ saved_mask = current->cpus_mask; -+ #endif -+ #endif - #else - saved_mask = current->cpus_allowed; - #endif diff --git a/spack b/spack index e30fedab10..594a376c52 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit e30fedab102f9281a220fb4fae82e3f8c43a82ac +Subproject commit 594a376c521cc746978571b1181a47bbcff30a21 diff --git a/test/integration_test.py b/test/integration_test.py index 3dca6c709e..2ed2c2bdd0 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -1,5 +1,11 @@ import pytest -from spack_commands import ALL_PACKAGES, spack_info, spack_spec +from spack_commands import ( + ALL_PACKAGES, + spack_info, + spack_spec, + run_with_spack, + log_file, +) @pytest.mark.parametrize("package", ALL_PACKAGES) @@ -18,6 +24,10 @@ def test_icon_serialization(): spack_spec("icon serialization=create") +def test_icon_exclaim_dsl(): + spack_spec("icon-exclaim dsl=substitute") + + def test_icon_fcgroup(): spack_spec("icon fcgroup=DACE.externals/dace_icon.-O1") @@ -33,3 +43,11 @@ def test_int2lm_parallel(): def test_int2lm_no_parallel(): spack_spec("int2lm ~parallel") + + +def test_compilers(): + run_with_spack("spack compilers", log=log_file("compilers")) + + +def test_find(): + run_with_spack("spack find", log=log_file("find")) diff --git a/test/system_test.py b/test/system_test.py index 6782c9ba05..fb38b1f8ec 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -2,151 +2,131 @@ from spack_commands import spack_install -def test_install_libfyaml_default(): - spack_install('libfyaml', test_root=False) +def test_install_clang_format(): + spack_install('clang-format') -def test_install_libtorch_default(): - spack_install('libtorch', test_root=False) +def test_install_cosmo_eccodes_definitions(): + spack_install('cosmo-eccodes-definitions') -@pytest.mark.parametrize("version", ['2.25.0.1', '2.19.0.7']) -def test_install_cosmo_eccodes_definitions_version(version): - spack_install(f'cosmo-eccodes-definitions @{version}', test_root=False) - - -def test_install_cosmo_6_0(): - spack_install('cosmo @6.0 %nvhpc', test_root=False) - - -def test_install_eccodes_2_19_0(): - spack_install('eccodes @2.19.0', test_root=False) - - -def test_install_fdb_fortran(): - spack_install('fdb-fortran') - - -@pytest.mark.parametrize("version", ['10.4.4', 'fdb']) -def test_install_flexpart_ifs_version(version): - spack_install(f'flexpart-ifs @{version}', test_root=False) +def test_install_ecbuild(): + # Tests are disabled because they fail with: + # The following tests FAILED: + # 1 - ECBUILD-359 (Failed) + # 2 - ECBUILD-401 (Failed) + # 8 - ECBUILD-511 (Failed) + # 11 - bundle-subdir-std (Failed) + # 12 - bundle-subdir-ecbfind (Failed) + # 17 - test_ecbuild_find_package (Failed) + spack_install('ecbuild @3.7.2', test_root=False) def test_install_flexpart_cosmo(): - spack_install('flexpart-cosmo @V8C4.0') - - -def test_install_fdb_5_11_17_gcc(): - spack_install('fdb @5.11.17 %gcc') - + spack_install('flexpart-cosmo') -def test_install_fdb_5_11_17_nvhpc(): - # tests fail because compiler emitted warnings. - spack_install('fdb @5.11.17 %nvhpc', test_root=False) +def test_install_flexpart_ifs(): + spack_install('flexpart-ifs') -def test_install_icon_2024_1_gcc(): - spack_install('icon @2024.1-1 %gcc') +@pytest.mark.parametrize('version', + ['2024.01-1', '2.6.6-mch2a', '2.6.6-mch2b']) +def test_install_icon(version): + # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. + spack_install(f'icon @{version} %nvhpc ^cray-mpich%nvhpc') -def test_install_icon_2024_1_nvhpc(): - spack_install('icon @2024.1-1 %nvhpc') - -def test_install_conditional_dependencies(): +def test_install_icon_conditional_dependencies(): # +coupling triggers libfyaml, libxml2, netcdf-c # serialization=create triggers serialbox # +emvorado triggers eccodes, hdf5, zlib # +eccodes-definitions triggers cosmo-eccodes-definitions # +mpi triggers mpi - # gpu=openacc+cuda triggers cuda + # gpu=nvidia-80 triggers cuda + # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. spack_install( - 'icon @2024.1-1 %nvhpc +coupling serialization=create +emvorado +mpi gpu=openacc+cuda cuda_arch=80' + 'icon @2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' ) def test_install_icontools(): - spack_install('icontools @2.5.2') + spack_install('icontools') def test_install_int2lm_3_00_nvhpc(): + # Tests are disabled because they fail with: + # Error: cmake is a duplicate dependency, with conflicting dependency types + # which stems from the package's 'test_int2lm.py'. spack_install('int2lm @int2lm-3.00 %nvhpc', test_root=False) -def test_install_libgrib1_22_01_2020_nvhpc(): - spack_install('libgrib1 @22-01-2020 %nvhpc') +def test_install_libgrib1_nvhpc(): + spack_install('libgrib1 %nvhpc') def test_install_makedepf90(): - spack_install('makedepf90 @3.0.1', test_root=False) - - -def test_install_oasis_version_4_0_nvhpc(): - spack_install('oasis @4.0 %nvhpc') - - -def test_install_pytorch_fortran_version_0_4(): - spack_install( - 'pytorch-fortran@0.4%nvhpc ^pytorch-fortran-proxy@0.4%gcc ^cuda@12.3.0', - test_root=False) + # Tests are disabled because they fail with: + # test1.sh: No such file or directory + spack_install('makedepf90', test_root=False) -def test_install_pytorch_fortran_proxy_version_0_4(): - spack_install('pytorch-fortran-proxy@0.4%gcc ^cuda@12.3.0', - test_root=False) +def test_install_oasis_nvhpc(): + spack_install('oasis %nvhpc') -def test_install_py_cytoolz_install_default(): +def test_install_py_cytoolz(): spack_install('py-cytoolz') -def test_install_py_devtools_install_default(): +def test_install_py_devtools(): spack_install('py-devtools') -def test_install_py_factory_boy_install_default(): +def test_install_py_factory_boy(): spack_install('py-factory-boy') -def test_install_py_frozendict_install_default(): - spack_install('py-frozendict') - - -def test_install_py_gridtools_cpp_install_default(): +def test_install_py_gridtools_cpp(): spack_install('py-gridtools-cpp') -@pytest.mark.parametrize("version", ['1.0.3.7', '1.0.3.9', '1.0.3.10']) +@pytest.mark.parametrize("version", ['1.0.3.9']) def test_install_py_gt4py_for_version(version): spack_install(f'py-gt4py @{version}') +# fails due to sql error +def test_build_only_py_gt4py_for_1_0_3_10(): + spack_install('py-gt4py @1.0.3.10', test_root=False) + + @pytest.mark.parametrize("version, gt4py_version", [('0.0.13', '1.0.3.9'), ('0.0.14', '1.0.3.10')]) def test_install_py_icon4py(version, gt4py_version): spack_install(f'py-icon4py@{version} ^py-gt4py@{gt4py_version}') -def test_install_py_hatchling_default(): +def test_install_py_hatchling(): spack_install('py-hatchling') -def test_install_py_inflection_default(): +def test_install_py_inflection(): spack_install('py-inflection') -def test_install_py_pytest_factoryboy_default(): +def test_install_py_pytest_factoryboy(): spack_install('py-pytest-factoryboy') -def test_install_py_tabulate_default(): +def test_install_py_tabulate(): spack_install('py-tabulate') -def test_install_py_typing_extensions_default(): +def test_install_py_typing_extensions(): spack_install('py-typing-extensions') -def test_install_yaxt_default(): +def test_install_yaxt(): spack_install('yaxt') diff --git a/test/unit_test.py b/test/unit_test.py index e024555fe9..49eef857b7 100644 --- a/test/unit_test.py +++ b/test/unit_test.py @@ -1,10 +1,7 @@ -import unittest from spack_commands import time_format -class TimeFormatTest(unittest.TestCase): - - def test_example(self): - self.assertEqual(time_format(0.123), "0.12s") - self.assertEqual(time_format(123.456), "2m 3.46s") - self.assertEqual(time_format(123456.789), "34h 17m 36.79s") +def test_example(): + assert time_format(0.123) == "0.12s" + assert time_format(123.456) == "2m 3.46s" + assert time_format(123456.789) == "34h 17m 36.79s"