From db3efbc9c38ffab138c49df641d746864ab2dffd Mon Sep 17 00:00:00 2001 From: "Adrien M. Bernede" Date: Mon, 28 Sep 2020 12:19:52 -0700 Subject: [PATCH 01/63] Replacing spack_configs by submodule --- .gitmodules | 6 + scripts/radiuss-spack-config | 1 + scripts/radiuss-spack-configs | 1 + scripts/uberenv/spack_configs | 1 + .../blueos_3_ppc64le_ib/compilers.yaml | 144 ------------ .../blueos_3_ppc64le_ib/packages.yaml | 20 -- .../blueos_3_ppc64le_ib_p9/compilers.yaml | 144 ------------ .../blueos_3_ppc64le_ib_p9/packages.yaml | 20 -- scripts/uberenv/spack_configs/config.yaml | 80 ------- .../spack_configs/darwin/compilers.yaml | 65 ------ .../spack_configs/darwin/packages.yaml | 25 -- .../toss_3_x86_64_ib/compilers.yaml | 217 ------------------ .../toss_3_x86_64_ib/packages.yaml | 18 -- 13 files changed, 9 insertions(+), 733 deletions(-) create mode 160000 scripts/radiuss-spack-config create mode 160000 scripts/radiuss-spack-configs create mode 120000 scripts/uberenv/spack_configs delete mode 100644 scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/compilers.yaml delete mode 100644 scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/packages.yaml delete mode 100644 scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/compilers.yaml delete mode 100644 scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/packages.yaml delete mode 100644 scripts/uberenv/spack_configs/config.yaml delete mode 100644 scripts/uberenv/spack_configs/darwin/compilers.yaml delete mode 100644 scripts/uberenv/spack_configs/darwin/packages.yaml delete mode 100644 scripts/uberenv/spack_configs/toss_3_x86_64_ib/compilers.yaml delete mode 100644 scripts/uberenv/spack_configs/toss_3_x86_64_ib/packages.yaml diff --git a/.gitmodules b/.gitmodules index d77cfd4e..dc03214e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,9 @@ [submodule "src/tpl/raja"] path = src/tpl/raja url = https://github.com/LLNL/RAJA.git +[submodule "scripts/radiuss-spack-config"] + path = scripts/radiuss-spack-config + url = https://github.com/LLNL/radiuss-spack-config +[submodule "scripts/radiuss-spack-configs"] + path = scripts/radiuss-spack-configs + url = https://github.com/LLNL/radiuss-spack-configs diff --git a/scripts/radiuss-spack-config b/scripts/radiuss-spack-config new file mode 160000 index 00000000..a63393d6 --- /dev/null +++ b/scripts/radiuss-spack-config @@ -0,0 +1 @@ +Subproject commit a63393d6d4a9db29975cf866aa8e1a0e56bfe3b1 diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs new file mode 160000 index 00000000..a63393d6 --- /dev/null +++ b/scripts/radiuss-spack-configs @@ -0,0 +1 @@ +Subproject commit a63393d6d4a9db29975cf866aa8e1a0e56bfe3b1 diff --git a/scripts/uberenv/spack_configs b/scripts/uberenv/spack_configs new file mode 120000 index 00000000..15a22164 --- /dev/null +++ b/scripts/uberenv/spack_configs @@ -0,0 +1 @@ +../radiuss-spack-config/spack_configs \ No newline at end of file diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/compilers.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/compilers.yaml deleted file mode 100644 index 66bc9290..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/compilers.yaml +++ /dev/null @@ -1,144 +0,0 @@ -compilers: -- compiler: - spec: clang@3.9.1 - paths: - cc: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@4.0.0 - paths: - cc: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@9.0.0 - paths: - cc: /usr/tce/packages/clang/clang-9.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-9.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@coral2018.08.08 - paths: - cc: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang - cxx: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@default - paths: - cc: clang - cxx: clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@8.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@4.9.3 - paths: - cc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@default - paths: - cc: gcc - cxx: g++ - f77: gfortran - fc: gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@default - paths: - cc: xlc - cxx: xlc++ - f77: xlf2003 - fc: xlf2003 - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@beta2019.06.20 - paths: - cc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc - cxx: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@default - paths: - cc: pgcc - cxx: pgc++ - f77: pgfortran - fc: pgfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/packages.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/packages.yaml deleted file mode 100644 index 1297fe96..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/packages.yaml +++ /dev/null @@ -1,20 +0,0 @@ -packages: - all: - # This defaults us to machine specific flags of ivybridge which allows - # us to run on broadwell as well - target: [ppc64le] - compiler: [gcc, pgi, clang, xl] - - cmake: - version: [3.14.5] - paths: - cmake: /usr/tce/packages/cmake/cmake-3.14.5 - buildable: False - - cuda: - version: [10.1.168,9.2.148,8.0] - paths: - cuda@10.1.168: /usr/tce/packages/cuda/cuda-10.1.168 - cuda@9.2.148: /usr/tce/packages/cuda/cuda-9.2.148 - cuda@8.0: /usr/tce/packages/cuda/cuda-8.0 - buildable: False diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/compilers.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/compilers.yaml deleted file mode 100644 index 66bc9290..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/compilers.yaml +++ /dev/null @@ -1,144 +0,0 @@ -compilers: -- compiler: - spec: clang@3.9.1 - paths: - cc: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@4.0.0 - paths: - cc: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@9.0.0 - paths: - cc: /usr/tce/packages/clang/clang-9.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-9.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@coral2018.08.08 - paths: - cc: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang - cxx: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@default - paths: - cc: clang - cxx: clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@8.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@4.9.3 - paths: - cc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@default - paths: - cc: gcc - cxx: g++ - f77: gfortran - fc: gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@default - paths: - cc: xlc - cxx: xlc++ - f77: xlf2003 - fc: xlf2003 - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@beta2019.06.20 - paths: - cc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc - cxx: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@default - paths: - cc: pgcc - cxx: pgc++ - f77: pgfortran - fc: pgfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/packages.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/packages.yaml deleted file mode 100644 index 1297fe96..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/packages.yaml +++ /dev/null @@ -1,20 +0,0 @@ -packages: - all: - # This defaults us to machine specific flags of ivybridge which allows - # us to run on broadwell as well - target: [ppc64le] - compiler: [gcc, pgi, clang, xl] - - cmake: - version: [3.14.5] - paths: - cmake: /usr/tce/packages/cmake/cmake-3.14.5 - buildable: False - - cuda: - version: [10.1.168,9.2.148,8.0] - paths: - cuda@10.1.168: /usr/tce/packages/cuda/cuda-10.1.168 - cuda@9.2.148: /usr/tce/packages/cuda/cuda-9.2.148 - cuda@8.0: /usr/tce/packages/cuda/cuda-8.0 - buildable: False diff --git a/scripts/uberenv/spack_configs/config.yaml b/scripts/uberenv/spack_configs/config.yaml deleted file mode 100644 index 2095112f..00000000 --- a/scripts/uberenv/spack_configs/config.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# ------------------------------------------------------------------------- -# This is the default spack configuration file. -# -# Settings here are versioned with Spack and are intended to provide -# sensible defaults out of the box. Spack maintainers should edit this -# file to keep it current. -# -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/config.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/config.yaml -# ------------------------------------------------------------------------- -config: - # This is the path to the root of the Spack install tree. - # You can use $spack here to refer to the root of the spack instance. - install_tree: $spack/.. - - # install directory layout - install_path_scheme: "${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}" - -# Locations where templates should be found - template_dirs: - - $spack/templates - - # Locations where different types of modules should be installed. - module_roots: - tcl: $spack/share/spack/modules - lmod: $spack/share/spack/lmod - - - # Temporary locations Spack can try to use for builds. - # - # Spack will use the first one it finds that exists and is writable. - # You can use $tempdir to refer to the system default temp directory - # (as returned by tempfile.gettempdir()). - # - # A value of $spack/var/spack/stage indicates that Spack should run - # builds directly inside its install directory without staging them in - # temporary space. - # - # The build stage can be purged with `spack purge --stage`. - build_stage: - # skipping tempdir b/c running mpi tests fails with local fs - # - $tempdir - - $spack/../builds - - - # Cache directory already downloaded source tarballs and archived - # repositories. This can be purged with `spack purge --downloads`. - source_cache: $spack/var/spack/cache - - - # Cache directory for miscellaneous files, like the package index. - # This can be purged with `spack purge --misc-cache` - misc_cache: .spack/misccache - - - # If this is false, tools like curl that use SSL will not verify - # certifiates. (e.g., curl will use use the -k option) - verify_ssl: true - - - # If set to true, Spack will always check checksums after downloading - # archives. If false, Spack skips the checksum step. - checksum: true - - - # If set to true, `spack install` and friends will NOT clean - # potentially harmful variables from the build environment. Use wisely. - dirty: false - - - # The default number of jobs to use when running `make` in parallel. - # If set to 4, for example, `spack install` will run `make -j4`. - # If not set, all available cores are used by default. - # for uberenv, limit build_jobs to 8 - build_jobs: 8 diff --git a/scripts/uberenv/spack_configs/darwin/compilers.yaml b/scripts/uberenv/spack_configs/darwin/compilers.yaml deleted file mode 100644 index ed5cbf02..00000000 --- a/scripts/uberenv/spack_configs/darwin/compilers.yaml +++ /dev/null @@ -1,65 +0,0 @@ -compilers: -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: elcapitan - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@7.3.0-apple -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: sierra - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@8.0.0-apple - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: highsierra - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@9.0.0-apple - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: mojave - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@10.0.0-apple - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: mojave - paths: - cc: /usr/local/opt/llvm/bin/clang - cxx: /usr/local/opt/llvm/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@10.0.0 - target: x86_64 diff --git a/scripts/uberenv/spack_configs/darwin/packages.yaml b/scripts/uberenv/spack_configs/darwin/packages.yaml deleted file mode 100644 index 6e965957..00000000 --- a/scripts/uberenv/spack_configs/darwin/packages.yaml +++ /dev/null @@ -1,25 +0,0 @@ - -# ------------------------------------------------------------------------- -# This file controls default concretization preferences for Spack. -# -# Settings here are versioned with Spack and are intended to provide -# sensible defaults out of the box. Spack maintainers should edit this -# file to keep it current. -# -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml -# ------------------------------------------------------------------------- -packages: - all: - compiler: [clang] - -# cmake: -# version: [3.17.2] -# paths: -# cmake@3.17.2: /usr/local/Cellar/cmake/3.17.2 -# buildable: false diff --git a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/compilers.yaml b/scripts/uberenv/spack_configs/toss_3_x86_64_ib/compilers.yaml deleted file mode 100644 index 27b86198..00000000 --- a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/compilers.yaml +++ /dev/null @@ -1,217 +0,0 @@ -compilers: -- compiler: - spec: clang@3.9.1 - paths: - cc: /usr/tce/packages/clang/clang-3.9.1/bin/clang - cxx: /usr/tce/packages/clang/clang-3.9.1/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@4.0.0 - paths: - cc: /usr/tce/packages/clang/clang-4.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-4.0.0/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@6.0.0 - paths: - cc: /usr/tce/packages/clang/clang-6.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-6.0.0/bin/clang++ - f77: - fc: - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@9.0.0 - paths: - cc: /usr/tce/packages/clang/clang-9.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-9.0.0/bin/clang++ - f77: - fc: - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@4.9.3 - paths: - cc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@16.0.4 - paths: - cc: /usr/tce/packages/intel/intel-16.0.4/bin/icc - cxx: /usr/tce/packages/intel/intel-16.0.4/bin/icpc - f77: /usr/tce/packages/intel/intel-16.0.4/bin/ifort - fc: /usr/tce/packages/intel/intel-16.0.4/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@17.0.2 - paths: - cc: /usr/tce/packages/intel/intel-17.0.2/bin/icc - cxx: /usr/tce/packages/intel/intel-17.0.2/bin/icpc - f77: /usr/tce/packages/intel/intel-17.0.2/bin/ifort - fc: /usr/tce/packages/intel/intel-17.0.2/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@18.0.0 - paths: - cc: /usr/tce/packages/intel/intel-18.0.0/bin/icc - cxx: /usr/tce/packages/intel/intel-18.0.0/bin/icpc - f77: /usr/tce/packages/intel/intel-18.0.0/bin/ifort - fc: /usr/tce/packages/intel/intel-18.0.0/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@19.0.4 - paths: - cc: /usr/tce/packages/intel/intel-19.0.4/bin/icc - cxx: /usr/tce/packages/intel/intel-19.0.4/bin/icpc - f77: /usr/tce/packages/intel/intel-19.0.4/bin/ifort - fc: /usr/tce/packages/intel/intel-19.0.4/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@17.10 - paths: - cc: /usr/tce/packages/pgi/pgi-17.10/bin/pgcc - cxx: /usr/tce/packages/pgi/pgi-17.10/bin/pgc++ - f77: /usr/tce/packages/pgi/pgi-17.10/bin/pgf77 - fc: /usr/tce/packages/pgi/pgi-17.10/bin/pgf95 - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@18.5 - paths: - cc: /usr/tce/packages/pgi/pgi-18.5/bin/pgcc - cxx: /usr/tce/packages/pgi/pgi-18.5/bin/pgc++ - f77: /usr/tce/packages/pgi/pgi-18.5/bin/pgf77 - fc: /usr/tce/packages/pgi/pgi-18.5/bin/pgf95 - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@19.4 - paths: - cc: /usr/tce/packages/pgi/pgi-19.4/bin/pgcc - cxx: /usr/tce/packages/pgi/pgi-19.4/bin/pgc++ - f77: /usr/tce/packages/pgi/pgi-19.4/bin/pgfortran - fc: /usr/tce/packages/pgi/pgi-19.4/bin/pgfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@6.1.0 - paths: - cc: /usr/tce/packages/gcc/gcc-6.1.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-6.1.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-6.1.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-6.1.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@7.1.0 - paths: - cc: /usr/tce/packages/gcc/gcc-7.1.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-7.1.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-7.1.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-7.1.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@7.3.0 - paths: - cc: /usr/tce/packages/gcc/gcc-7.3.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-7.3.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-7.3.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-7.3.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@8.1.0 - paths: - cc: /usr/tce/packages/gcc/gcc-8.1.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.1.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.1.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.1.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/packages.yaml b/scripts/uberenv/spack_configs/toss_3_x86_64_ib/packages.yaml deleted file mode 100644 index c67d70f1..00000000 --- a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/packages.yaml +++ /dev/null @@ -1,18 +0,0 @@ -packages: - all: - # This defaults us to machine specific flags of ivybridge which allows - # us to run on broadwell as well - target: [ivybridge] - compiler: [gcc, intel, pgi, clang] - - cmake: - version: [3.14.5] - paths: - cmake: /usr/tce/packages/cmake/cmake-3.14.5 - buildable: False - - cuda: - version: [10.1.168] - paths: - cuda@10.1.168: /usr/tce/packages/cuda/cuda-10.1.168 - buildable: False From 63ab031eda1af787a9fd671fbf36b74297f22637 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 6 Oct 2020 13:45:03 -0700 Subject: [PATCH 02/63] Remove duplicate submodule --- .gitmodules | 3 --- scripts/radiuss-spack-config | 1 - 2 files changed, 4 deletions(-) delete mode 160000 scripts/radiuss-spack-config diff --git a/.gitmodules b/.gitmodules index dc03214e..3d736105 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,9 +7,6 @@ [submodule "src/tpl/raja"] path = src/tpl/raja url = https://github.com/LLNL/RAJA.git -[submodule "scripts/radiuss-spack-config"] - path = scripts/radiuss-spack-config - url = https://github.com/LLNL/radiuss-spack-config [submodule "scripts/radiuss-spack-configs"] path = scripts/radiuss-spack-configs url = https://github.com/LLNL/radiuss-spack-configs diff --git a/scripts/radiuss-spack-config b/scripts/radiuss-spack-config deleted file mode 160000 index a63393d6..00000000 --- a/scripts/radiuss-spack-config +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a63393d6d4a9db29975cf866aa8e1a0e56bfe3b1 From 8e1993c657e60e3ba7c75281fc803b10cffec82f Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 6 Oct 2020 13:45:36 -0700 Subject: [PATCH 03/63] Update spack-config submodule --- scripts/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs index a63393d6..1ed8697e 160000 --- a/scripts/radiuss-spack-configs +++ b/scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit a63393d6d4a9db29975cf866aa8e1a0e56bfe3b1 +Subproject commit 1ed8697e764798b6f38b2ee59389b24a0b66f10c From 3e957beaa29be5d5a44c35e92cae6378be1eef12 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 6 Oct 2020 13:46:39 -0700 Subject: [PATCH 04/63] Use same jobs as Umpire --- .gitlab/lassen-jobs.yml | 98 ++++++++++++++++++++++++++---------- .gitlab/quartz-jobs.yml | 94 ++++++++++------------------------ .gitlab/quartz-templates.yml | 5 +- 3 files changed, 99 insertions(+), 98 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index a068e03d..f06710b2 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -5,62 +5,106 @@ # SPDX-License-Identifier: (MIT) ############################################################################## -#### -# All lassen build jobs +# CPU ONLY +ibm_clang_9: + variables: + SPEC: "+fortran %clang@9.0.0ibm" + extends: .build_and_test_on_lassen -clang_default (build and test on lassen): +ibm_clang_9_gcc_8: variables: - SPEC: "%clang@default" + SPEC: "+fortran %clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen -gcc_default (build and test on lassen): +gcc_8_3_1: variables: - SPEC: "%gcc@default" + SPEC: "+fortran%gcc@8.3.1" extends: .build_and_test_on_lassen -xl_default (build and test on lassen): +pgi_19_10: variables: - SPEC: "%xl@default" + SPEC: "+fortran%pgi@19.10" extends: .build_and_test_on_lassen -pgi_default (build and test on lassen): +pgi_19_10_gcc_8_3_1: variables: - SPEC: "%pgi@default" + SPEC: "+fortran%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + extends: .build_and_test_on_lassen + +pgi_20_4_gcc_8_3_1: + variables: + SPEC: "+fortran%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + extends: .build_and_test_on_lassen + +xl_16_1_1_7: + variables: + SPEC: "+fortran%xl@16.1.1.7" + extends: .build_and_test_on_lassen + +xl_16_1_1_7_gcc_8_3_1: + variables: + SPEC: "+fortran%xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + extends: .build_and_test_on_lassen + +# CUDA + +ibm_clang_9_cuda: + variables: + SPEC: "+fortran+cuda+allow-untested-versions %clang@9.0.0ibm ^cuda@10.1.168" extends: .build_and_test_on_lassen - allow_failure: true -gcc_8_3_1 (build and test on lassen): +ibm_clang_9_gcc_8_cuda: variables: - SPEC: "%gcc@8.3.1" + SPEC: "+fortran+cuda %clang@9.0.0ibm +allow-untested-versions cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" extends: .build_and_test_on_lassen -clang_3_9_1 (build and test on lassen): +gcc_8_3_1_cuda: variables: - SPEC: "%clang@3.9.1" - extends: .build_and_test_on_lassen_advanced + SPEC: "+fortran+cuda %gcc@8.3.1 ^cuda@10.1.168" + extends: .build_and_test_on_lassen -clang_4_0_0 (build and test on lassen): +pgi_19_10_cuda: variables: - SPEC: "%clang@4.0.0" - extends: .build_and_test_on_lassen_advanced + SPEC: "+fortran+cuda %pgi@19.10 ^cuda@10.1.168" + extends: .build_and_test_on_lassen -clang_9_0_0 (build and test on lassen): +pgi_19_10_gcc_8_3_1_cuda: variables: - SPEC: "%clang@9.0.0" + SPEC: "+fortran+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" + allow_failure: true extends: .build_and_test_on_lassen -nvcc_gcc_4_9_3 (build and test on lassen): +pgi_20_4_gcc_8_3_1_cuda_11: variables: - SPEC: "%gcc@4.9.3+cuda" + SPEC: "+fortran+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" + allow_failure: true extends: .build_and_test_on_lassen -nvcc_gcc_4_9_3_raja (build and test on lassen): +xl_16_1_1_7_cuda: variables: - SPEC: "%gcc@4.9.3+cuda+raja" + SPEC: "+fortran+cuda %xl@16.1.1.7 ^cuda@10.1.168" extends: .build_and_test_on_lassen + +xl_16_1_1_7_gcc_8_3_1_cuda_11: + variables: + SPEC: "+fortran+cuda %xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" allow_failure: true + extends: .build_and_test_on_lassen + +# EXTRAS + +clang_9_0_0_libcpp (build and test on lassen): + variables: + SPEC: "%clang@9.0.0+libcpp" + extends: .build_and_test_on_lassen + +clang_9_0_0_memleak (build and test on lassen): + variables: + SPEC: "%clang@9.0.0 cxxflags=-fsanitize=address" + ASAN_OPTIONS: "detect_leaks=1" + extends: .build_and_test_on_lassen -nvcc_xl-beta-2019.06.20 (build and test on lassen): +xl_default_omp_target (build and test on lassen): variables: - SPEC: "%xl@beta2019.06.20+cuda" + SPEC: "%xl@default+openmp+openmp_target" extends: .build_and_test_on_lassen diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 31edccb8..467c8cdd 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -4,93 +4,51 @@ # # SPDX-License-Identifier: (MIT) ############################################################################## - -#### -# Quartz build jobs -clang_3_9_1 (build and test on quartz): - variables: - SPEC: "%clang@3.9.1" - extends: .build_and_test_on_quartz_advanced - -clang_4_0_0 (build and test on quartz): - variables: - SPEC: "%clang@4.0.0" - extends: .build_and_test_on_quartz_advanced - -clang_6_0_0 (build and test on quartz): - variables: - SPEC: "%clang@6.0.0" - extends: .build_and_test_on_quartz_advanced - -clang_9_0_0 (build and test on quartz): +clang_10: variables: - SPEC: "%clang@9.0.0" + SPEC: "+fortran %clang@10.0.1" extends: .build_and_test_on_quartz - -gcc_4_9_3 (build and test on quartz): - variables: - SPEC: "%gcc@4.9.3" - extends: .build_and_test_on_quartz_advanced - -gcc_6_1_0 (build and test on quartz): - variables: - SPEC: "%gcc@6.1.0" - extends: .build_and_test_on_quartz_advanced - -gcc_7_1_0 (build and test on quartz): - variables: - SPEC: "%gcc@7.1.0" - extends: .build_and_test_on_quartz_advanced - -gcc_7_3_0 (build and test on quartz): +clang_9: variables: - SPEC: "%gcc@7.3.0" - extends: .build_and_test_on_quartz_advanced + SPEC: "+fortran %clang@9.0.0" + extends: .build_and_test_on_quartz -gcc_8_1_0 (build and test on quartz): +gcc_8_1_0: variables: - SPEC: "%gcc@8.1.0" + SPEC: "+fortran %gcc@8.1.0" extends: .build_and_test_on_quartz - -icpc_16_0_4 (build and test on quartz): +icpc_17_0_2: variables: - SPEC: "%intel@16.0.4" - extends: .build_and_test_on_quartz_advanced - allow_failure: true + SPEC: "+fortran %intel@17.0.2" + extends: .build_and_test_on_quartz -icpc_17_0_2 (build and test on quartz): +icpc_18_0_2: variables: - SPEC: "%intel@17.0.2" - extends: .build_and_test_on_quartz_advanced + SPEC: "+fortran tests=none %intel@18.0.2" + extends: .build_and_test_on_quartz -icpc_18_0_0 (build and test on quartz): +icpc_19_1_0: variables: - SPEC: "%intel@18.0.0" - extends: .build_and_test_on_quartz_advanced - allow_failure: true + SPEC: "+fortran %intel@19.1.0" + extends: .build_and_test_on_quartz -icpc_19_0_4 (build and test on quartz): +pgi_19_7: variables: - SPEC: "%intel@19.0.4" + SPEC: "+fortran %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" + PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz - -pgi_17_10 (build and test on quartz): +pgi_20_1: variables: - SPEC: "%pgi@17.10" - extends: .build_and_test_on_quartz_advanced - allow_failure: true + SPEC: "+fortran %pgi@20.1" + PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 + extends: .build_and_test_on_quartz -pgi_18_5 (build and test on quartz): - variables: - SPEC: "%pgi@18.5" - extends: .build_and_test_on_quartz_advanced - allow_failure: true +# EXTRAS -pgi_19_4 (build and test on quartz): +gcc_4_9_3: variables: - SPEC: "%pgi@19.4" + SPEC: "%gcc@4.9.3" extends: .build_and_test_on_quartz - allow_failure: true diff --git a/.gitlab/quartz-templates.yml b/.gitlab/quartz-templates.yml index 744fe246..c9b4ed67 100644 --- a/.gitlab/quartz-templates.yml +++ b/.gitlab/quartz-templates.yml @@ -29,8 +29,7 @@ allocate_resources (on quartz): extends: .on_quartz stage: q_allocate_resources script: - - salloc -N 1 -c 36 -p pdebug -t 10 --no-shell --job-name=${ALLOC_NAME} - needs: [] + - salloc -N 1 -c 36 -p pdebug -t 20 --no-shell --job-name=${ALLOC_NAME} #### # In post-build phase, deallocate resources @@ -47,8 +46,8 @@ release_resources (on quartz): #### # Generic quartz build job, extending build script .build_and_test_on_quartz: + extends: [.build_toss_3_x86_64_ib_script, .on_quartz, .venv] stage: q_build_and_test - extends: [.build_toss_3_x86_64_ib_script, .on_quartz] .build_and_test_on_quartz_advanced: extends: [.build_and_test_on_quartz, .advanced_pipeline] From 052898aa2e83f9a25ee1357cd7cc0fa8c5aeeefb Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 6 Oct 2020 13:47:29 -0700 Subject: [PATCH 05/63] Update Umpire package --- scripts/uberenv/packages/umpire/package.py | 43 ++++++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/uberenv/packages/umpire/package.py index edeb7f86..07a7873c 100644 --- a/scripts/uberenv/packages/umpire/package.py +++ b/scripts/uberenv/packages/umpire/package.py @@ -12,6 +12,8 @@ from os import environ as env from os.path import join as pjoin +import re + def cmake_cache_entry(name, value, comment=""): """Generate a string for a cmake cache variable""" @@ -48,7 +50,6 @@ def get_spec_path(spec, package_name, path_replacements = {}, use_bin = False) : return path - class Umpire(CMakePackage, CudaPackage): """An application-focused API for memory management on NUMA & GPU architectures""" @@ -85,6 +86,7 @@ class Umpire(CMakePackage, CudaPackage): variant('numa', default=False, description='Enable NUMA support') variant('shared', default=True, description='Enable Shared libs') variant('openmp', default=False, description='Build with OpenMP support') + variant('openmp_target', default=False, description='Build with OpenMP 4.5 support') variant('deviceconst', default=False, description='Enables support for constant device memory') variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), @@ -97,6 +99,7 @@ class Umpire(CMakePackage, CudaPackage): conflicts('+numa', when='@:0.3.2') conflicts('~c', when='+fortran', msg='Fortran API requires C API') + conflicts('~openmp', when='+openmp_target', msg='OpenMP target requires OpenMP') phases = ['hostconfig', 'cmake', 'build', 'install'] @@ -157,6 +160,7 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if os.path.isfile(env["SPACK_FC"]): f_compiler = env["SPACK_FC"] + ####################################################################### # By directly fetching the names of the actual compilers we appear # to doing something evil here, but this is necessary to create a @@ -201,6 +205,8 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#------------------\n\n".format("-" * 60)) cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + if '+fortran' in spec: + cfg.write(cmake_cache_entry("CMAKE_Fortran_COMPILER", f_compiler)) # use global spack compiler flags cflags = ' '.join(spec.compiler_flags['cflags']) @@ -215,7 +221,14 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if cxxflags: cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) - if ("gfortran" in f_compiler) and ("clang" in cpp_compiler): + fflags = ' '.join(spec.compiler_flags['fflags']) + cfg.write(cmake_cache_entry("CMAKE_Fortran_FLAGS", fflags)) + + fortran_compilers = ["gfortran", "xlf"] + if any(compiler in f_compiler for compiler in fortran_compilers) and ("clang" in cpp_compiler): + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) + libdir = pjoin(os.path.dirname( os.path.dirname(f_compiler)), "lib") flags = "" @@ -227,6 +240,18 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("BLT_EXE_LINKER_FLAGS", flags, description)) + + gcc_toolchain_regex = re.compile(".*gcc-toolchain.*") + gcc_name_regex = re.compile(".*gcc-name.*") + + using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) + using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) + compilers_using_toolchain = ["pgi", "xl", "icpc"] + if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): + if using_toolchain or using_gcc_name: + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) + if "toss_3_x86_64_ib" in sys_type: release_flags = "-O3" reldebinf_flags = "-O3 -g" @@ -254,14 +279,20 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", cudacompiler)) + cuda_flags = [] + if not spec.satisfies('cuda_arch=none'): cuda_arch = spec.variants['cuda_arch'].value - flag = '-arch sm_{0}'.format(cuda_arch[0]) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", flag)) + cuda_flags.append('-arch sm_{0}'.format(cuda_arch[0])) if '+deviceconst' in spec: cfg.write(cmake_cache_option("ENABLE_DEVICE_CONST", True)) + if using_toolchain: + cuda_flags.append("-Xcompiler {}".format(using_toolchain[0])) + + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", ' '.join(cuda_flags))) + else: cfg.write(cmake_cache_option("ENABLE_CUDA", False)) @@ -269,6 +300,10 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option("ENABLE_FORTRAN", '+fortran' in spec)) cfg.write(cmake_cache_option("ENABLE_NUMA", '+numa' in spec)) cfg.write(cmake_cache_option("ENABLE_OPENMP", '+openmp' in spec)) + if "+openmp_target" in spec: + cfg.write(cmake_cache_option("ENABLE_OPENMP_TARGET", True)) + if ('%xl' in spec): + cfg.write(cmake_cache_entry("OpenMP_CXX_FLAGS", "-qsmp;-qoffload")) cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec)) From f3972969133df31c56f551dbd241aa51aa709d01 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 6 Oct 2020 14:10:08 -0700 Subject: [PATCH 06/63] Remove unecessary .venv mention --- .gitlab/quartz-templates.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/quartz-templates.yml b/.gitlab/quartz-templates.yml index c9b4ed67..459bf121 100644 --- a/.gitlab/quartz-templates.yml +++ b/.gitlab/quartz-templates.yml @@ -46,7 +46,7 @@ release_resources (on quartz): #### # Generic quartz build job, extending build script .build_and_test_on_quartz: - extends: [.build_toss_3_x86_64_ib_script, .on_quartz, .venv] + extends: [.build_toss_3_x86_64_ib_script, .on_quartz] stage: q_build_and_test .build_and_test_on_quartz_advanced: From 0167232b95559762efe1f68ea0062bcc67330df7 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 6 Oct 2020 14:52:37 -0700 Subject: [PATCH 07/63] Remove fortran variant (doesn't exist for CHAI) --- .gitlab/lassen-jobs.yml | 32 ++++++++++++++++---------------- .gitlab/quartz-jobs.yml | 16 ++++++++-------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index f06710b2..69e017b3 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -8,86 +8,86 @@ # CPU ONLY ibm_clang_9: variables: - SPEC: "+fortran %clang@9.0.0ibm" + SPEC: " %clang@9.0.0ibm" extends: .build_and_test_on_lassen ibm_clang_9_gcc_8: variables: - SPEC: "+fortran %clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: " %clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen gcc_8_3_1: variables: - SPEC: "+fortran%gcc@8.3.1" + SPEC: "%gcc@8.3.1" extends: .build_and_test_on_lassen pgi_19_10: variables: - SPEC: "+fortran%pgi@19.10" + SPEC: "%pgi@19.10" extends: .build_and_test_on_lassen pgi_19_10_gcc_8_3_1: variables: - SPEC: "+fortran%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: "%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen pgi_20_4_gcc_8_3_1: variables: - SPEC: "+fortran%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: "%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen xl_16_1_1_7: variables: - SPEC: "+fortran%xl@16.1.1.7" + SPEC: "%xl@16.1.1.7" extends: .build_and_test_on_lassen xl_16_1_1_7_gcc_8_3_1: variables: - SPEC: "+fortran%xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: "%xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen # CUDA ibm_clang_9_cuda: variables: - SPEC: "+fortran+cuda+allow-untested-versions %clang@9.0.0ibm ^cuda@10.1.168" + SPEC: "+cuda+allow-untested-versions %clang@9.0.0ibm ^cuda@10.1.168" extends: .build_and_test_on_lassen ibm_clang_9_gcc_8_cuda: variables: - SPEC: "+fortran+cuda %clang@9.0.0ibm +allow-untested-versions cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" + SPEC: "+cuda %clang@9.0.0ibm +allow-untested-versions cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" extends: .build_and_test_on_lassen gcc_8_3_1_cuda: variables: - SPEC: "+fortran+cuda %gcc@8.3.1 ^cuda@10.1.168" + SPEC: "+cuda %gcc@8.3.1 ^cuda@10.1.168" extends: .build_and_test_on_lassen pgi_19_10_cuda: variables: - SPEC: "+fortran+cuda %pgi@19.10 ^cuda@10.1.168" + SPEC: "+cuda %pgi@19.10 ^cuda@10.1.168" extends: .build_and_test_on_lassen pgi_19_10_gcc_8_3_1_cuda: variables: - SPEC: "+fortran+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" + SPEC: "+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" allow_failure: true extends: .build_and_test_on_lassen pgi_20_4_gcc_8_3_1_cuda_11: variables: - SPEC: "+fortran+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" + SPEC: "+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" allow_failure: true extends: .build_and_test_on_lassen xl_16_1_1_7_cuda: variables: - SPEC: "+fortran+cuda %xl@16.1.1.7 ^cuda@10.1.168" + SPEC: "+cuda %xl@16.1.1.7 ^cuda@10.1.168" extends: .build_and_test_on_lassen xl_16_1_1_7_gcc_8_3_1_cuda_11: variables: - SPEC: "+fortran+cuda %xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" + SPEC: "+cuda %xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" allow_failure: true extends: .build_and_test_on_lassen diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 467c8cdd..f02304f9 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -6,43 +6,43 @@ ############################################################################## clang_10: variables: - SPEC: "+fortran %clang@10.0.1" + SPEC: " %clang@10.0.1" extends: .build_and_test_on_quartz clang_9: variables: - SPEC: "+fortran %clang@9.0.0" + SPEC: " %clang@9.0.0" extends: .build_and_test_on_quartz gcc_8_1_0: variables: - SPEC: "+fortran %gcc@8.1.0" + SPEC: " %gcc@8.1.0" extends: .build_and_test_on_quartz icpc_17_0_2: variables: - SPEC: "+fortran %intel@17.0.2" + SPEC: " %intel@17.0.2" extends: .build_and_test_on_quartz icpc_18_0_2: variables: - SPEC: "+fortran tests=none %intel@18.0.2" + SPEC: " tests=none %intel@18.0.2" extends: .build_and_test_on_quartz icpc_19_1_0: variables: - SPEC: "+fortran %intel@19.1.0" + SPEC: " %intel@19.1.0" extends: .build_and_test_on_quartz pgi_19_7: variables: - SPEC: "+fortran %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" + SPEC: " %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz pgi_20_1: variables: - SPEC: "+fortran %pgi@20.1" + SPEC: " %pgi@20.1" PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz From 6d84d092e863450dbde5200faf3eb69cf0c398eb Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 6 Oct 2020 15:59:02 -0700 Subject: [PATCH 08/63] Fixup symlink --- scripts/uberenv/spack_configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uberenv/spack_configs b/scripts/uberenv/spack_configs index 15a22164..2c6464c5 120000 --- a/scripts/uberenv/spack_configs +++ b/scripts/uberenv/spack_configs @@ -1 +1 @@ -../radiuss-spack-config/spack_configs \ No newline at end of file +../radiuss-spack-configs/spack_configs \ No newline at end of file From 3641d6e1c9bbd5921a78d2bdd68ecca8c1547b40 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 7 Oct 2020 08:30:08 -0700 Subject: [PATCH 09/63] Update Umpire package version --- scripts/uberenv/packages/umpire/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/uberenv/packages/umpire/package.py index 07a7873c..feff285d 100644 --- a/scripts/uberenv/packages/umpire/package.py +++ b/scripts/uberenv/packages/umpire/package.py @@ -59,6 +59,7 @@ class Umpire(CMakePackage, CudaPackage): version('develop', branch='develop', submodules='True') version('master', branch='main', submodules='True') + version('4.1.2', tag='v4.1.2', submodules='True') version('3.0.0', tag='v3.0.0', submodules='True') version('2.1.0', tag='v2.1.0', submodules='True') version('2.0.0', tag='v2.0.0', submodules='True') From f1104cd6f2871b891aee2b93dcb9dea562b82fb5 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 7 Oct 2020 10:53:35 -0700 Subject: [PATCH 10/63] Point to custom Spack for untested (CUDA,host) combos --- scripts/uberenv/project.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/uberenv/project.json b/scripts/uberenv/project.json index 4cbf5809..1335d596 100644 --- a/scripts/uberenv/project.json +++ b/scripts/uberenv/project.json @@ -3,7 +3,8 @@ "package_version" : "develop", "package_final_phase" : "hostconfig", "package_source_dir" : "../..", -"spack_url": "https://github.com/spack/spack", -"spack_commit": "bc53bb9b7cbf250c98cfe77d334ed30d6b958c21", +"spack_url": "https://github.com/davidbeckingsale/spack", +"spack_branch": "feature/allow-untested-cuda-versions", +"spack_commit": "f96e256bee1948aa030916aae0c1b2645230fb9f", "spack_activate" : {} } From de4275368ed7fb6c315388eeac32885bd50b2b06 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 7 Oct 2020 12:45:59 -0700 Subject: [PATCH 11/63] Bump BLT --- blt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blt b/blt index 4a73f821..a1f6138d 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit 4a73f821a335cfdab68505c58dc5f16eec1521b6 +Subproject commit a1f6138d9680d3781514d3705a39c5620a1b49d4 From 50bc59d5ab82f435bc3aa6ebdc44c90ed5d55aee Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 7 Oct 2020 16:27:13 -0700 Subject: [PATCH 12/63] Fixup directory name --- scripts/gitlab/build_and_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index f3dc8e48..e75096c4 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -53,7 +53,7 @@ then if [[ -d /dev/shm ]] then - prefix="/dev/shm/${hostname}/${spec// /_}" + prefix="/dev/shm/${hostname}/${spec//[ \/\-\=]/_}" mkdir -p ${prefix} prefix_opt="--prefix=${prefix}" fi From 22e9bc9607e2c0d663c85006770e272e4dc00486 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Mon, 12 Oct 2020 10:56:44 -0700 Subject: [PATCH 13/63] Propagate allow-untested-versions variant --- scripts/uberenv/packages/chai/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 0fa189f3..49925e40 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -78,6 +78,8 @@ class Chai(CMakePackage, CudaPackage): depends_on('cmake@3.9:', type='build', when="+cuda") depends_on('umpire+cuda', when="+cuda") depends_on('raja+cuda', when="+raja+cuda") + depends_on('umpire+cuda+allow-untested-versions', when="+cuda+allow-untested-versions") + depends_on('raja+cuda+allow-untested-versions', when="+raja+cuda+allow-untested-versions") phases = ['hostconfig', 'cmake', 'build', 'install'] From 810f4d78cfd2ec42aec7cb8b8c192b98b77e5c86 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 13 Oct 2020 11:54:22 -0700 Subject: [PATCH 14/63] Use umpire@develop for PGI --- .gitlab/lassen-jobs.yml | 12 ++++++------ .gitlab/quartz-jobs.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 69e017b3..e070b63c 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -23,17 +23,17 @@ gcc_8_3_1: pgi_19_10: variables: - SPEC: "%pgi@19.10" + SPEC: "%pgi@19.10 ^umpire@develop" extends: .build_and_test_on_lassen pgi_19_10_gcc_8_3_1: variables: - SPEC: "%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: "%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^umpire@develop" extends: .build_and_test_on_lassen pgi_20_4_gcc_8_3_1: variables: - SPEC: "%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: "%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^umpire@develop" extends: .build_and_test_on_lassen xl_16_1_1_7: @@ -65,18 +65,18 @@ gcc_8_3_1_cuda: pgi_19_10_cuda: variables: - SPEC: "+cuda %pgi@19.10 ^cuda@10.1.168" + SPEC: "+cuda %pgi@19.10 ^cuda@10.1.168 ^umpire@develop" extends: .build_and_test_on_lassen pgi_19_10_gcc_8_3_1_cuda: variables: - SPEC: "+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" + SPEC: "+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168 ^umpire@develop" allow_failure: true extends: .build_and_test_on_lassen pgi_20_4_gcc_8_3_1_cuda_11: variables: - SPEC: "+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" + SPEC: "+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2 ^umpire@develop" allow_failure: true extends: .build_and_test_on_lassen diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index f02304f9..84844799 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -36,13 +36,13 @@ icpc_19_1_0: pgi_19_7: variables: - SPEC: " %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" + SPEC: " %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3 ^umpire@develop" PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz pgi_20_1: variables: - SPEC: " %pgi@20.1" + SPEC: " %pgi@20.1 ^umpire@develop" PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz From 195445dec77f00be25600192b941b5bb091526df Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 13 Oct 2020 12:53:33 -0700 Subject: [PATCH 15/63] Updates for PGI --- CMakeLists.txt | 6 ++++++ blt | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 221ee451..eac73140 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,12 @@ endif() set(ENABLE_COPY_HEADERS Off CACHE BOOL "") set(BLT_CXX_STD c++11 CACHE STRING "") +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") + message(STATUS "Setting CMAKE_CXX_EXTENSIONS to ON for PGI Compiler") + set( CMAKE_CXX_EXTENSIONS ON ) + endif() +endif() + if (ENABLE_CUDA) cmake_minimum_required(VERSION 3.9) else () diff --git a/blt b/blt index a1f6138d..c48fcb58 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit a1f6138d9680d3781514d3705a39c5620a1b49d4 +Subproject commit c48fcb58b22bb94fdc46b6e9b242d95e2de5c9c8 From 2b06de78cf1b4a0569e9b407bd67f2edc3529a76 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 13:31:51 -0700 Subject: [PATCH 16/63] Update BLT --- blt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blt b/blt index c48fcb58..a1f6138d 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit c48fcb58b22bb94fdc46b6e9b242d95e2de5c9c8 +Subproject commit a1f6138d9680d3781514d3705a39c5620a1b49d4 From 67d3066aea694c9141965c4cf3d98b4b80318489 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 13:33:10 -0700 Subject: [PATCH 17/63] Don't try and run tests if they are disabled --- scripts/gitlab/build_and_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index e75096c4..c801abd9 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -120,7 +120,7 @@ then fi # Test -if [[ "${option}" != "--build-only" ]] +if [[ "${option}" != "--build-only" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} then echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~ Testing CHAI" From 335ecae9ed54b66c7d445ca716ca13c32f87922e Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 13:35:24 -0700 Subject: [PATCH 18/63] Remove omp_target job --- .gitlab/lassen-jobs.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index e070b63c..f7738dc7 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -103,8 +103,3 @@ clang_9_0_0_memleak (build and test on lassen): SPEC: "%clang@9.0.0 cxxflags=-fsanitize=address" ASAN_OPTIONS: "detect_leaks=1" extends: .build_and_test_on_lassen - -xl_default_omp_target (build and test on lassen): - variables: - SPEC: "%xl@default+openmp+openmp_target" - extends: .build_and_test_on_lassen From 2f4fae1db5b6a946c223678ce7579a784a4d1461 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 13:37:41 -0700 Subject: [PATCH 19/63] Propagate tests=none --- scripts/uberenv/packages/chai/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 49925e40..57fa54d5 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -75,6 +75,9 @@ class Chai(CMakePackage, CudaPackage): depends_on('umpire') depends_on('raja', when="+raja") + depends_on('raja tests=none', when='tests=none') + depends_on('umpire tests=none', when='tests=none') + depends_on('cmake@3.9:', type='build', when="+cuda") depends_on('umpire+cuda', when="+cuda") depends_on('raja+cuda', when="+raja+cuda") From 2cb36623c92bcb8ae15d8e66024891e86c396a62 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 14:58:00 -0700 Subject: [PATCH 20/63] Always depend on umpire@develop --- .gitlab/lassen-jobs.yml | 16 ++++++---------- .gitlab/quartz-jobs.yml | 4 ++-- scripts/uberenv/packages/chai/package.py | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index f7738dc7..162bbbcd 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -28,12 +28,12 @@ pgi_19_10: pgi_19_10_gcc_8_3_1: variables: - SPEC: "%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^umpire@develop" + SPEC: "%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen pgi_20_4_gcc_8_3_1: variables: - SPEC: "%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^umpire@develop" + SPEC: "%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen xl_16_1_1_7: @@ -65,18 +65,18 @@ gcc_8_3_1_cuda: pgi_19_10_cuda: variables: - SPEC: "+cuda %pgi@19.10 ^cuda@10.1.168 ^umpire@develop" + SPEC: "+cuda %pgi@19.10 ^cuda@10.1.168" extends: .build_and_test_on_lassen pgi_19_10_gcc_8_3_1_cuda: variables: - SPEC: "+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168 ^umpire@develop" + SPEC: "+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" allow_failure: true extends: .build_and_test_on_lassen pgi_20_4_gcc_8_3_1_cuda_11: variables: - SPEC: "+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2 ^umpire@develop" + SPEC: "+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" allow_failure: true extends: .build_and_test_on_lassen @@ -93,13 +93,9 @@ xl_16_1_1_7_gcc_8_3_1_cuda_11: # EXTRAS -clang_9_0_0_libcpp (build and test on lassen): - variables: - SPEC: "%clang@9.0.0+libcpp" - extends: .build_and_test_on_lassen - clang_9_0_0_memleak (build and test on lassen): variables: SPEC: "%clang@9.0.0 cxxflags=-fsanitize=address" ASAN_OPTIONS: "detect_leaks=1" + allow_failure: true extends: .build_and_test_on_lassen diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 84844799..f02304f9 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -36,13 +36,13 @@ icpc_19_1_0: pgi_19_7: variables: - SPEC: " %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3 ^umpire@develop" + SPEC: " %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz pgi_20_1: variables: - SPEC: " %pgi@20.1 ^umpire@develop" + SPEC: " %pgi@20.1" PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 57fa54d5..f3ffb91a 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -72,7 +72,7 @@ class Chai(CMakePackage, CudaPackage): multi=False, description='Tests to run') depends_on('cmake@3.8:', type='build') - depends_on('umpire') + depends_on('umpire@develop') depends_on('raja', when="+raja") depends_on('raja tests=none', when='tests=none') From c6b44f23fa3625a811daae8e57e78a98bc288525 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 15:12:22 -0700 Subject: [PATCH 21/63] Make sure CHAI excludes lib dirs when mixing GCC toolchains --- scripts/uberenv/packages/chai/package.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index f3ffb91a..0fa259f0 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -205,6 +205,17 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("BLT_EXE_LINKER_FLAGS", flags, description)) + gcc_toolchain_regex = re.compile(".*gcc-toolchain.*") + gcc_name_regex = re.compile(".*gcc-name.*") + + using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) + using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) + compilers_using_toolchain = ["pgi", "xl", "icpc"] + if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): + if using_toolchain or using_gcc_name: + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) + if "+cuda" in spec: cfg.write("#------------------{0}\n".format("-" * 60)) cfg.write("# Cuda\n") From 0c9e33055329f2d46f9497b29c4e41d705272d0d Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 15:13:49 -0700 Subject: [PATCH 22/63] Don't always build Umpire C interface --- scripts/uberenv/packages/umpire/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/uberenv/packages/umpire/package.py index feff285d..2ae7a5d0 100644 --- a/scripts/uberenv/packages/umpire/package.py +++ b/scripts/uberenv/packages/umpire/package.py @@ -83,7 +83,7 @@ class Umpire(CMakePackage, CudaPackage): patch('camp_target_umpire_3.0.0.patch', when='@3.0.0') variant('fortran', default=False, description='Build C/Fortran API') - variant('c', default=True, description='Build C API') + variant('c', default=False, description='Build C API') variant('numa', default=False, description='Enable NUMA support') variant('shared', default=True, description='Enable Shared libs') variant('openmp', default=False, description='Build with OpenMP support') From 11f56acd454a5398acd04d994108bddc61fe2ac8 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Oct 2020 15:16:08 -0700 Subject: [PATCH 23/63] Add missing re import --- scripts/uberenv/packages/chai/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 0fa259f0..0ca965c8 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -12,6 +12,8 @@ from os import environ as env from os.path import join as pjoin +import re + def cmake_cache_entry(name, value, comment=""): """Generate a string for a cmake cache variable""" From 8e76ded5288443e47b435a3cb336b4867a4ece92 Mon Sep 17 00:00:00 2001 From: Daniel Taller Date: Wed, 2 Dec 2020 16:48:59 -0800 Subject: [PATCH 24/63] some initial changes, not ready --- benchmarks/chai_managed_ptr_benchmarks.cpp | 116 ++++++++++----------- tests/integration/managed_ptr_tests.cpp | 26 ++--- tests/integration/raja-chai-nested.cpp | 1 + tests/integration/raja-chai-tests.cpp | 1 + tests/unit/managed_ptr_unit_tests.cpp | 14 +-- 5 files changed, 80 insertions(+), 78 deletions(-) diff --git a/benchmarks/chai_managed_ptr_benchmarks.cpp b/benchmarks/chai_managed_ptr_benchmarks.cpp index 62eb6564..e2abb2cc 100644 --- a/benchmarks/chai_managed_ptr_benchmarks.cpp +++ b/benchmarks/chai_managed_ptr_benchmarks.cpp @@ -134,8 +134,8 @@ static void benchmark_use_managed_ptr_cpu(benchmark::State& state) values[i] = i * i; } -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + gpuDeviceSynchronize(); #endif while (state.KeepRunning()) { @@ -145,8 +145,8 @@ static void benchmark_use_managed_ptr_cpu(benchmark::State& state) free(values); object.free(); -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + gpuDeviceSynchronize(); #endif } @@ -209,7 +209,7 @@ static void benchmark_pass_copy_to_gpu(benchmark::State& state) while (state.KeepRunning()) { copy_kernel<<<1, 1>>>(helper); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } } @@ -225,10 +225,10 @@ static void benchmark_copy_to_gpu(benchmark::State& state) while (state.KeepRunning()) { ClassWithSize* gpuPointer; - cudaMalloc(&gpuPointer, sizeof(ClassWithSize)); - cudaMemcpy(gpuPointer, cpuPointer, sizeof(ClassWithSize), cudaMemcpyHostToDevice); - cudaFree(gpuPointer); - cudaDeviceSynchronize(); + gpuMalloc(&gpuPointer, sizeof(ClassWithSize)); + gpuMemcpy(gpuPointer, cpuPointer, sizeof(ClassWithSize), gpuMemcpyHostToDevice); + gpuFree(gpuPointer); + gpuDeviceSynchronize(); } delete cpuPointer; @@ -258,11 +258,11 @@ static void benchmark_placement_new_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize* address; - cudaMalloc(&address, sizeof(ClassWithSize)); + gpuMalloc(&address, sizeof(ClassWithSize)); placement_new_kernel<<<1, 1>>>(address); placement_delete_kernel<<<1, 1>>>(address); - cudaFree(address); - cudaDeviceSynchronize(); + gpuFree(address); + gpuDeviceSynchronize(); } } @@ -290,11 +290,11 @@ static void benchmark_new_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize** buffer; - cudaMalloc(&buffer, sizeof(ClassWithSize*)); + gpuMalloc(&buffer, sizeof(ClassWithSize*)); create_kernel<<<1, 1>>>(buffer); delete_kernel<<<1, 1>>>(buffer); - cudaFree(buffer); - cudaDeviceSynchronize(); + gpuFree(buffer); + gpuDeviceSynchronize(); } } @@ -317,15 +317,15 @@ static void benchmark_new_on_gpu_and_copy_to_host(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize** gpuBuffer; - cudaMalloc(&gpuBuffer, sizeof(ClassWithSize*)); + gpuMalloc(&gpuBuffer, sizeof(ClassWithSize*)); create_kernel<<<1, 1>>>(gpuBuffer); ClassWithSize** cpuBuffer = (ClassWithSize**) malloc(sizeof(ClassWithSize*)); - cudaMemcpy(cpuBuffer, gpuBuffer, sizeof(ClassWithSize*), cudaMemcpyDeviceToHost); - cudaFree(gpuBuffer); + gpuMemcpy(cpuBuffer, gpuBuffer, sizeof(ClassWithSize*), gpuMemcpyDeviceToHost); + gpuFree(gpuBuffer); ClassWithSize* gpuPointer = cpuBuffer[0]; free(cpuBuffer); delete_kernel_2<<<1, 1>>>(gpuPointer); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } } @@ -348,7 +348,7 @@ static void benchmark_create_on_stack_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { create_on_stack_kernel<<<1, 1>>>(); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } } @@ -379,19 +379,19 @@ void benchmark_use_managed_ptr_gpu(benchmark::State& state) int numValues = 100; int* values; - cudaMalloc(&values, numValues * sizeof(int)); + gpuMalloc(&values, numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } - cudaFree(values); + gpuFree(values); object.free(); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } BENCHMARK(benchmark_use_managed_ptr_gpu); @@ -409,19 +409,19 @@ void benchmark_curiously_recurring_template_pattern_gpu(benchmark::State& state) int numValues = 100; int* values; - cudaMalloc(&values, numValues * sizeof(int)); + gpuMalloc(&values, numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } - cudaFree(values); + gpuFree(values); delete derivedCRTP; - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } BENCHMARK(benchmark_curiously_recurring_template_pattern_gpu); @@ -438,19 +438,19 @@ void benchmark_no_inheritance_gpu(benchmark::State& state) int numValues = 100; int* values; - cudaMalloc(&values, numValues * sizeof(int)); + gpuMalloc(&values, numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } - cudaFree(values); + gpuFree(values); delete noInheritance; - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } BENCHMARK(benchmark_no_inheritance_gpu); @@ -471,19 +471,19 @@ void benchmark_bulk_use_managed_ptr_gpu(benchmark::State& state) chai::managed_ptr object = chai::make_managed(2); int* values; - cudaMalloc(&values, N * sizeof(int)); + gpuMalloc(&values, N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } - cudaFree(values); + gpuFree(values); object.free(); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_use_managed_ptr_gpu, 1); @@ -519,19 +519,19 @@ void benchmark_bulk_curiously_recurring_template_pattern_gpu(benchmark::State& s auto object = *derivedCRTP; int* values; - cudaMalloc(&values, N * sizeof(int)); + gpuMalloc(&values, N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } - cudaFree(values); + gpuFree(values); delete derivedCRTP; - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_curiously_recurring_template_pattern_gpu, 1); @@ -567,19 +567,19 @@ void benchmark_bulk_no_inheritance_gpu(benchmark::State& state) auto object = *noInheritance; int* values; - cudaMalloc(&values, N * sizeof(int)); + gpuMalloc(&values, N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } - cudaFree(values); + gpuFree(values); delete noInheritance; - cudaDeviceSynchronize(); + gpuDeviceSynchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_no_inheritance_gpu, 1); @@ -612,8 +612,8 @@ static void benchmark_bulk_polymorphism_cpu(benchmark::State& state) values[i] = i * i; } -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + gpuDeviceSynchronize(); #endif while (state.KeepRunning()) { @@ -626,8 +626,8 @@ static void benchmark_bulk_polymorphism_cpu(benchmark::State& state) free(values); delete object; -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + gpuDeviceSynchronize(); #endif } @@ -659,8 +659,8 @@ static void benchmark_bulk_use_managed_ptr_cpu(benchmark::State& state) values[i] = i * i; } -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + gpuDeviceSynchronize(); #endif while (state.KeepRunning()) { @@ -673,8 +673,8 @@ static void benchmark_bulk_use_managed_ptr_cpu(benchmark::State& state) free(values); object.free(); -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + gpuDeviceSynchronize(); #endif } diff --git a/tests/integration/managed_ptr_tests.cpp b/tests/integration/managed_ptr_tests.cpp index 441bc638..f6e69b80 100644 --- a/tests/integration/managed_ptr_tests.cpp +++ b/tests/integration/managed_ptr_tests.cpp @@ -326,7 +326,7 @@ TEST(managed_ptr, managed_array_of_managed_ptr) delete[] expectedValues; } -#ifdef __CUDACC__ +#ifdef CHAI_GPUCC template __global__ void deviceNew(T** arr) { @@ -348,21 +348,21 @@ GPU_TEST(managed_ptr, make_on_device) hostArray[0] = nullptr; int** deviceArray = nullptr; - cudaMalloc(&deviceArray, sizeof(int*)); + gpuMalloc(&deviceArray, sizeof(int*)); int** deviceArray2 = nullptr; - cudaMalloc(&deviceArray2, sizeof(int*)); + gpuMalloc(&deviceArray2, sizeof(int*)); deviceNew<<<1, 1>>>(deviceArray); - cudaMemcpy(hostArray, deviceArray, sizeof(int*), cudaMemcpyDeviceToHost); - cudaMemcpy(deviceArray2, hostArray, sizeof(int*), cudaMemcpyHostToDevice); + gpuMemcpy(hostArray, deviceArray, sizeof(int*), gpuMemcpyDeviceToHost); + gpuMemcpy(deviceArray2, hostArray, sizeof(int*), gpuMemcpyHostToDevice); ASSERT_NE(hostArray[0], nullptr); deviceDelete<<<1, 1>>>(deviceArray2); free(hostArray); - cudaFree(deviceArray); - cudaFree(deviceArray2); + gpuFree(deviceArray); + gpuFree(deviceArray2); } GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) @@ -373,16 +373,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) // Initialize device side memory to hold a pointer RawArrayClass** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); + gpuMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), cudaMemcpyDeviceToHost); + gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); @@ -402,16 +402,16 @@ GPU_TEST(managed_ptr, gpu_build_managed_ptr) // Initialize device side memory to hold a pointer RawArrayClass** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); + gpuMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), cudaMemcpyDeviceToHost); + gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); diff --git a/tests/integration/raja-chai-nested.cpp b/tests/integration/raja-chai-nested.cpp index 68f26cc6..9f721911 100644 --- a/tests/integration/raja-chai-nested.cpp +++ b/tests/integration/raja-chai-nested.cpp @@ -27,6 +27,7 @@ using namespace std; #include "gtest/gtest.h" +// TODO: add hip policy for these tests. #if defined(RAJA_ENABLE_CUDA) #define PARALLEL_RAJA_DEVICE __device__ #elif defined(RAJA_ENABLE_OPENMP) diff --git a/tests/integration/raja-chai-tests.cpp b/tests/integration/raja-chai-tests.cpp index b40a436c..f20af4a9 100644 --- a/tests/integration/raja-chai-tests.cpp +++ b/tests/integration/raja-chai-tests.cpp @@ -16,6 +16,7 @@ #include "gtest/gtest.h" +// TODO: add hip policy for these tests. #define CUDA_TEST(X, Y) \ static void cuda_test_##X##_##Y(); \ TEST(X, Y) { cuda_test_##X##_##Y(); } \ diff --git a/tests/unit/managed_ptr_unit_tests.cpp b/tests/unit/managed_ptr_unit_tests.cpp index f6743251..cd38d8ea 100644 --- a/tests/unit/managed_ptr_unit_tests.cpp +++ b/tests/unit/managed_ptr_unit_tests.cpp @@ -464,7 +464,7 @@ TEST(managed_ptr, reinterpret_pointer_cast) derived.free(); } -#ifdef __CUDACC__ +#ifdef CHAI_GPUCC GPU_TEST(managed_ptr, gpu_default_constructor) { @@ -603,16 +603,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) // Initialize device side memory to hold a pointer Simple** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(Simple*)); + gpuMalloc(&gpuPointerHolder, sizeof(Simple*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), cudaMemcpyDeviceToHost); + gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); @@ -632,16 +632,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device_2) // Initialize device side memory to hold a pointer Simple** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(Simple*)); + gpuMalloc(&gpuPointerHolder, sizeof(Simple*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), cudaMemcpyDeviceToHost); + gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); From 13cae1270f346390c716e9dfd7882f74a0e23d3f Mon Sep 17 00:00:00 2001 From: Daniel Taller Date: Wed, 2 Dec 2020 18:21:15 -0800 Subject: [PATCH 25/63] fixes --- benchmarks/chai_managed_ptr_benchmarks.cpp | 105 +++++++++++---------- tests/integration/managed_ptr_tests.cpp | 25 ++--- tests/unit/managed_ptr_unit_tests.cpp | 13 +-- 3 files changed, 73 insertions(+), 70 deletions(-) diff --git a/benchmarks/chai_managed_ptr_benchmarks.cpp b/benchmarks/chai_managed_ptr_benchmarks.cpp index e2abb2cc..7352f2c5 100644 --- a/benchmarks/chai_managed_ptr_benchmarks.cpp +++ b/benchmarks/chai_managed_ptr_benchmarks.cpp @@ -9,6 +9,7 @@ #include "benchmark/benchmark.h" #include "chai/config.hpp" +#include "chai/ArrayManager.hpp" #include "chai/managed_ptr.hpp" #include "../src/util/forall.hpp" @@ -135,7 +136,7 @@ static void benchmark_use_managed_ptr_cpu(benchmark::State& state) } #ifdef CHAI_GPUCC - gpuDeviceSynchronize(); + chai::synchronize(); #endif while (state.KeepRunning()) { @@ -146,7 +147,7 @@ static void benchmark_use_managed_ptr_cpu(benchmark::State& state) object.free(); #ifdef CHAI_GPUCC - gpuDeviceSynchronize(); + chai::synchronize(); #endif } @@ -209,7 +210,7 @@ static void benchmark_pass_copy_to_gpu(benchmark::State& state) while (state.KeepRunning()) { copy_kernel<<<1, 1>>>(helper); - gpuDeviceSynchronize(); + chai::synchronize(); } } @@ -225,10 +226,10 @@ static void benchmark_copy_to_gpu(benchmark::State& state) while (state.KeepRunning()) { ClassWithSize* gpuPointer; - gpuMalloc(&gpuPointer, sizeof(ClassWithSize)); - gpuMemcpy(gpuPointer, cpuPointer, sizeof(ClassWithSize), gpuMemcpyHostToDevice); - gpuFree(gpuPointer); - gpuDeviceSynchronize(); + chai::gpuMalloc((void**)(&gpuPointer), sizeof(ClassWithSize)); + chai::gpuMemcpy(gpuPointer, cpuPointer, sizeof(ClassWithSize), gpuMemcpyHostToDevice); + chai::gpuFree(gpuPointer); + chai::synchronize(); } delete cpuPointer; @@ -258,11 +259,11 @@ static void benchmark_placement_new_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize* address; - gpuMalloc(&address, sizeof(ClassWithSize)); + chai::gpuMalloc((void**)(&address), sizeof(ClassWithSize)); placement_new_kernel<<<1, 1>>>(address); placement_delete_kernel<<<1, 1>>>(address); - gpuFree(address); - gpuDeviceSynchronize(); + chai::gpuFree(address); + chai::synchronize(); } } @@ -290,11 +291,11 @@ static void benchmark_new_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize** buffer; - gpuMalloc(&buffer, sizeof(ClassWithSize*)); + chai::gpuMalloc((void**)(&buffer), sizeof(ClassWithSize*)); create_kernel<<<1, 1>>>(buffer); delete_kernel<<<1, 1>>>(buffer); - gpuFree(buffer); - gpuDeviceSynchronize(); + chai::gpuFree(buffer); + chai::synchronize(); } } @@ -317,15 +318,15 @@ static void benchmark_new_on_gpu_and_copy_to_host(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize** gpuBuffer; - gpuMalloc(&gpuBuffer, sizeof(ClassWithSize*)); + chai::gpuMalloc((void**)(&gpuBuffer), sizeof(ClassWithSize*)); create_kernel<<<1, 1>>>(gpuBuffer); ClassWithSize** cpuBuffer = (ClassWithSize**) malloc(sizeof(ClassWithSize*)); - gpuMemcpy(cpuBuffer, gpuBuffer, sizeof(ClassWithSize*), gpuMemcpyDeviceToHost); - gpuFree(gpuBuffer); + chai::gpuMemcpy(cpuBuffer, gpuBuffer, sizeof(ClassWithSize*), gpuMemcpyDeviceToHost); + chai::gpuFree(gpuBuffer); ClassWithSize* gpuPointer = cpuBuffer[0]; free(cpuBuffer); delete_kernel_2<<<1, 1>>>(gpuPointer); - gpuDeviceSynchronize(); + chai::synchronize(); } } @@ -348,7 +349,7 @@ static void benchmark_create_on_stack_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { create_on_stack_kernel<<<1, 1>>>(); - gpuDeviceSynchronize(); + chai::synchronize(); } } @@ -379,19 +380,19 @@ void benchmark_use_managed_ptr_gpu(benchmark::State& state) int numValues = 100; int* values; - gpuMalloc(&values, numValues * sizeof(int)); + chai::gpuMalloc((void**)(&values), numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - gpuDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - gpuDeviceSynchronize(); + chai::synchronize(); } - gpuFree(values); + chai::gpuFree(values); object.free(); - gpuDeviceSynchronize(); + chai::synchronize(); } BENCHMARK(benchmark_use_managed_ptr_gpu); @@ -409,19 +410,19 @@ void benchmark_curiously_recurring_template_pattern_gpu(benchmark::State& state) int numValues = 100; int* values; - gpuMalloc(&values, numValues * sizeof(int)); + chai::gpuMalloc((void**)(&values), numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - gpuDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - gpuDeviceSynchronize(); + chai::synchronize(); } - gpuFree(values); + chai::gpuFree(values); delete derivedCRTP; - gpuDeviceSynchronize(); + chai::synchronize(); } BENCHMARK(benchmark_curiously_recurring_template_pattern_gpu); @@ -438,19 +439,19 @@ void benchmark_no_inheritance_gpu(benchmark::State& state) int numValues = 100; int* values; - gpuMalloc(&values, numValues * sizeof(int)); + chai::gpuMalloc((void**)(&values), numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - gpuDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - gpuDeviceSynchronize(); + chai::synchronize(); } - gpuFree(values); + chai::gpuFree(values); delete noInheritance; - gpuDeviceSynchronize(); + chai::synchronize(); } BENCHMARK(benchmark_no_inheritance_gpu); @@ -471,19 +472,19 @@ void benchmark_bulk_use_managed_ptr_gpu(benchmark::State& state) chai::managed_ptr object = chai::make_managed(2); int* values; - gpuMalloc(&values, N * sizeof(int)); + chai::gpuMalloc((void**)(&values), N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - gpuDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - gpuDeviceSynchronize(); + chai::synchronize(); } - gpuFree(values); + chai::gpuFree(values); object.free(); - gpuDeviceSynchronize(); + chai::synchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_use_managed_ptr_gpu, 1); @@ -519,19 +520,19 @@ void benchmark_bulk_curiously_recurring_template_pattern_gpu(benchmark::State& s auto object = *derivedCRTP; int* values; - gpuMalloc(&values, N * sizeof(int)); + chai::gpuMalloc((void**)(&values), N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - gpuDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - gpuDeviceSynchronize(); + chai::synchronize(); } - gpuFree(values); + chai::gpuFree(values); delete derivedCRTP; - gpuDeviceSynchronize(); + chai::synchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_curiously_recurring_template_pattern_gpu, 1); @@ -567,19 +568,19 @@ void benchmark_bulk_no_inheritance_gpu(benchmark::State& state) auto object = *noInheritance; int* values; - gpuMalloc(&values, N * sizeof(int)); + chai::gpuMalloc((void**)(&values), N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - gpuDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - gpuDeviceSynchronize(); + chai::synchronize(); } - gpuFree(values); + chai::gpuFree(values); delete noInheritance; - gpuDeviceSynchronize(); + chai::synchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_no_inheritance_gpu, 1); @@ -613,7 +614,7 @@ static void benchmark_bulk_polymorphism_cpu(benchmark::State& state) } #ifdef CHAI_GPUCC - gpuDeviceSynchronize(); + chai::synchronize(); #endif while (state.KeepRunning()) { @@ -627,7 +628,7 @@ static void benchmark_bulk_polymorphism_cpu(benchmark::State& state) delete object; #ifdef CHAI_GPUCC - gpuDeviceSynchronize(); + chai::synchronize(); #endif } @@ -660,7 +661,7 @@ static void benchmark_bulk_use_managed_ptr_cpu(benchmark::State& state) } #ifdef CHAI_GPUCC - gpuDeviceSynchronize(); + chai::synchronize(); #endif while (state.KeepRunning()) { @@ -674,7 +675,7 @@ static void benchmark_bulk_use_managed_ptr_cpu(benchmark::State& state) object.free(); #ifdef CHAI_GPUCC - gpuDeviceSynchronize(); + chai::synchronize(); #endif } diff --git a/tests/integration/managed_ptr_tests.cpp b/tests/integration/managed_ptr_tests.cpp index f6e69b80..55a51e9d 100644 --- a/tests/integration/managed_ptr_tests.cpp +++ b/tests/integration/managed_ptr_tests.cpp @@ -12,6 +12,7 @@ static void gpu_test_##X##Y() #include "chai/config.hpp" +#include "chai/ArrayManager.hpp" #include "chai/ManagedArray.hpp" #include "chai/managed_ptr.hpp" @@ -348,21 +349,21 @@ GPU_TEST(managed_ptr, make_on_device) hostArray[0] = nullptr; int** deviceArray = nullptr; - gpuMalloc(&deviceArray, sizeof(int*)); + chai::gpuMalloc((void**)(&deviceArray), sizeof(int*)); int** deviceArray2 = nullptr; - gpuMalloc(&deviceArray2, sizeof(int*)); + chai::gpuMalloc((void**)(&deviceArray2), sizeof(int*)); deviceNew<<<1, 1>>>(deviceArray); - gpuMemcpy(hostArray, deviceArray, sizeof(int*), gpuMemcpyDeviceToHost); - gpuMemcpy(deviceArray2, hostArray, sizeof(int*), gpuMemcpyHostToDevice); + chai::gpuMemcpy(hostArray, deviceArray, sizeof(int*), gpuMemcpyDeviceToHost); + chai::gpuMemcpy(deviceArray2, hostArray, sizeof(int*), gpuMemcpyHostToDevice); ASSERT_NE(hostArray[0], nullptr); deviceDelete<<<1, 1>>>(deviceArray2); free(hostArray); - gpuFree(deviceArray); - gpuFree(deviceArray2); + chai::gpuFree(deviceArray); + chai::gpuFree(deviceArray2); } GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) @@ -373,16 +374,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) // Initialize device side memory to hold a pointer RawArrayClass** gpuPointerHolder = nullptr; - gpuMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(RawArrayClass*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); // Free device side memory - gpuFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); @@ -402,16 +403,16 @@ GPU_TEST(managed_ptr, gpu_build_managed_ptr) // Initialize device side memory to hold a pointer RawArrayClass** gpuPointerHolder = nullptr; - gpuMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(RawArrayClass*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); // Free device side memory - gpuFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); diff --git a/tests/unit/managed_ptr_unit_tests.cpp b/tests/unit/managed_ptr_unit_tests.cpp index cd38d8ea..f3cfe4ae 100644 --- a/tests/unit/managed_ptr_unit_tests.cpp +++ b/tests/unit/managed_ptr_unit_tests.cpp @@ -12,6 +12,7 @@ static void gpu_test_##X_##Y() #include "chai/config.hpp" +#include "chai/ArrayManager.hpp" #include "chai/ManagedArray.hpp" #include "chai/managed_ptr.hpp" @@ -603,16 +604,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) // Initialize device side memory to hold a pointer Simple** gpuPointerHolder = nullptr; - gpuMalloc(&gpuPointerHolder, sizeof(Simple*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(Simple*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); // Free device side memory - gpuFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); @@ -632,16 +633,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device_2) // Initialize device side memory to hold a pointer Simple** gpuPointerHolder = nullptr; - gpuMalloc(&gpuPointerHolder, sizeof(Simple*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(Simple*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); // Free device side memory - gpuFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); From c59aa3786c034a8aca1090be9b6f904878fc9fb4 Mon Sep 17 00:00:00 2001 From: Alan Dayton Date: Tue, 15 Dec 2020 15:06:57 -0800 Subject: [PATCH 26/63] Add the ability to enable or disable managed_ptr --- CMakeLists.txt | 1 + benchmarks/CMakeLists.txt | 18 ++++++++++-------- src/chai/CMakeLists.txt | 7 ++++++- src/chai/config.hpp.in | 1 + tests/integration/CMakeLists.txt | 22 ++++++++++++---------- tests/unit/CMakeLists.txt | 26 ++++++++++++++------------ 6 files changed, 44 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6110b9d3..8116788b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ option(ENABLE_UM "Use CUDA unified (managed) memory" Off) option(ENABLE_PINNED "Use pinned host memory" Off) option(ENABLE_RAJA_PLUGIN "Build plugin to set RAJA execution spaces" Off) option(CHAI_ENABLE_GPU_ERROR_CHECKING "Enable GPU error checking" On) +option(CHAI_ENABLE_MANAGED_PTR "Enable managed_ptr" On) option(CHAI_DEBUG "Enable Debug Logging.") set(ENABLE_RAJA_NESTED_TEST ON CACHE BOOL "Enable raja-chai-nested-tests, which fails to build on Debug CUDA builds.") diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 14bbc1be..2cb6ec63 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -13,6 +13,7 @@ if (ENABLE_CUDA) ${chai_benchmark_depends} cuda) endif () + if (ENABLE_HIP) set (chai_benchmark_depends ${chai_benchmark_depends} @@ -37,12 +38,13 @@ blt_add_benchmark( NAME managedarray_benchmarks COMMAND managedarray_benchmarks) -blt_add_executable( - NAME managed_ptr_benchmarks - SOURCES chai_managed_ptr_benchmarks.cpp - DEPENDS_ON ${chai_benchmark_depends}) - -blt_add_benchmark( - NAME managed_ptr_benchmarks - COMMAND managed_ptr_benchmarks) +if (CHAI_ENABLE_MANAGED_PTR) + blt_add_executable( + NAME managed_ptr_benchmarks + SOURCES chai_managed_ptr_benchmarks.cpp + DEPENDS_ON ${chai_benchmark_depends}) + blt_add_benchmark( + NAME managed_ptr_benchmarks + COMMAND managed_ptr_benchmarks) +endif () diff --git a/src/chai/CMakeLists.txt b/src/chai/CMakeLists.txt index 4ce3c41f..7f557507 100644 --- a/src/chai/CMakeLists.txt +++ b/src/chai/CMakeLists.txt @@ -25,10 +25,15 @@ set (chai_headers ExecutionSpaces.hpp ManagedArray.hpp ManagedArray.inl - managed_ptr.hpp PointerRecord.hpp Types.hpp) +if (CHAI_ENABLE_MANAGED_PTR) + set (chai_headers + ${chai_headers} + managed_ptr.hpp) +endif () + if(DISABLE_RM) set(chai_headers ${chai_headers} diff --git a/src/chai/config.hpp.in b/src/chai/config.hpp.in index 26961f34..8325184c 100644 --- a/src/chai/config.hpp.in +++ b/src/chai/config.hpp.in @@ -15,6 +15,7 @@ #cmakedefine CHAI_ENABLE_UM #cmakedefine CHAI_DEBUG #cmakedefine CHAI_ENABLE_GPU_ERROR_CHECKING +#cmakedefine CHAI_ENABLE_MANAGED_PTR #cmakedefine CHAI_ENABLE_RAJA_PLUGIN #cmakedefine CHAI_ENABLE_GPU_SIMULATION_MODE #cmakedefine CHAI_ENABLE_PINNED diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 6825dfeb..b422be2b 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -25,18 +25,20 @@ blt_add_test( NAME managed_array_test COMMAND managed_array_tests) -blt_add_executable( - NAME managed_ptr_tests - SOURCES managed_ptr_tests.cpp - DEPENDS_ON ${chai_integration_test_depends}) +if (CHAI_ENABLE_MANAGED_PTR) + blt_add_executable( + NAME managed_ptr_tests + SOURCES managed_ptr_tests.cpp + DEPENDS_ON ${chai_integration_test_depends}) -target_include_directories( - managed_ptr_tests - PUBLIC ${PROJECT_BINARY_DIR}/include) + target_include_directories( + managed_ptr_tests + PUBLIC ${PROJECT_BINARY_DIR}/include) -blt_add_test( - NAME managed_ptr_test - COMMAND managed_ptr_tests) + blt_add_test( + NAME managed_ptr_test + COMMAND managed_ptr_tests) +endif () if (ENABLE_RAJA_PLUGIN) set(raja_test_depends diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 2438289e..d2236989 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -36,15 +36,17 @@ blt_add_test( NAME array_manager_unit_test COMMAND array_manager_unit_tests) -blt_add_executable( - NAME managed_ptr_unit_tests - SOURCES managed_ptr_unit_tests.cpp - DEPENDS_ON ${chai_unit_test_depends}) - -target_include_directories( - managed_ptr_unit_tests - PUBLIC ${PROJECT_BINARY_DIR}/include) - -blt_add_test( - NAME managed_ptr_unit_test - COMMAND managed_ptr_unit_tests) +if (CHAI_ENABLE_MANAGED_PTR) + blt_add_executable( + NAME managed_ptr_unit_tests + SOURCES managed_ptr_unit_tests.cpp + DEPENDS_ON ${chai_unit_test_depends}) + + target_include_directories( + managed_ptr_unit_tests + PUBLIC ${PROJECT_BINARY_DIR}/include) + + blt_add_test( + NAME managed_ptr_unit_test + COMMAND managed_ptr_unit_tests) +endif () From 86f11c5e8291eaa2741c08e1ffe289c66d281a1c Mon Sep 17 00:00:00 2001 From: Alan Dayton Date: Tue, 15 Dec 2020 17:09:06 -0800 Subject: [PATCH 27/63] Add error message for trying to use managed_ptr without it being configured --- src/chai/CMakeLists.txt | 7 +------ src/chai/managed_ptr.hpp | 9 +++++++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/chai/CMakeLists.txt b/src/chai/CMakeLists.txt index 7f557507..4ce3c41f 100644 --- a/src/chai/CMakeLists.txt +++ b/src/chai/CMakeLists.txt @@ -25,15 +25,10 @@ set (chai_headers ExecutionSpaces.hpp ManagedArray.hpp ManagedArray.inl + managed_ptr.hpp PointerRecord.hpp Types.hpp) -if (CHAI_ENABLE_MANAGED_PTR) - set (chai_headers - ${chai_headers} - managed_ptr.hpp) -endif () - if(DISABLE_RM) set(chai_headers ${chai_headers} diff --git a/src/chai/managed_ptr.hpp b/src/chai/managed_ptr.hpp index 1afcf9c2..55a0f122 100644 --- a/src/chai/managed_ptr.hpp +++ b/src/chai/managed_ptr.hpp @@ -9,6 +9,8 @@ #include "chai/config.hpp" +#if defined(CHAI_ENABLE_MANAGED_PTR) + #ifndef CHAI_DISABLE_RM #include "chai/ArrayManager.hpp" #endif @@ -1318,5 +1320,12 @@ namespace chai { } } // namespace chai +#else // defined(CHAI_ENABLE_MANAGED_PTR) + +#error CHAI must be configured with -DCHAI_ENABLE_MANAGED_PTR=ON to use managed_ptr! \ + If CHAI_ENABLE_MANAGED_PTR is defined as a macro, it is safe to include managed_ptr.hpp. + +#endif // defined(CHAI_ENABLE_MANAGED_PTR) + #endif // MANAGED_PTR From d035030a6d8d679848c249a1ce0b33c1fd2dcd94 Mon Sep 17 00:00:00 2001 From: "Benjamin T. Liu" Date: Tue, 9 Mar 2021 17:15:58 -0800 Subject: [PATCH 28/63] Fix to allow UM and PINNED memory to be active simultaneously --- src/chai/ArrayManager.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/chai/ArrayManager.cpp b/src/chai/ArrayManager.cpp index 5f364840..c190b3bc 100644 --- a/src/chai/ArrayManager.cpp +++ b/src/chai/ArrayManager.cpp @@ -305,8 +305,9 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre if (space_ptr == pointer_record->m_pointers[space_t]) pointer_record->m_pointers[space_t] = nullptr; } - } else { -#elif defined(CHAI_ENABLE_PINNED) + } else +#endif +#if defined(CHAI_ENABLE_PINNED) if (space_ptr == pointer_record->m_pointers[PINNED]) { callback(pointer_record, ACTION_FREE, @@ -320,7 +321,7 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre if (space_ptr == pointer_record->m_pointers[space_t]) pointer_record->m_pointers[space_t] = nullptr; } - } else { + } else #endif callback(pointer_record, ACTION_FREE, @@ -331,9 +332,6 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre alloc.deallocate(space_ptr); pointer_record->m_pointers[space] = nullptr; -#if defined(CHAI_ENABLE_UM) || defined(CHAI_ENABLE_PINNED) - } -#endif } else { From 8c2c45efb083c5e19733cad7d1a76ea645d20ef7 Mon Sep 17 00:00:00 2001 From: "Benjamin T. Liu" Date: Thu, 11 Mar 2021 16:10:59 -0800 Subject: [PATCH 29/63] Add missing brace --- src/chai/ArrayManager.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/chai/ArrayManager.cpp b/src/chai/ArrayManager.cpp index c190b3bc..a96e0d21 100644 --- a/src/chai/ArrayManager.cpp +++ b/src/chai/ArrayManager.cpp @@ -323,6 +323,7 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre } } else #endif + { callback(pointer_record, ACTION_FREE, ExecutionSpace(space)); @@ -332,6 +333,7 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre alloc.deallocate(space_ptr); pointer_record->m_pointers[space] = nullptr; + } } else { From 74fb1abc2908190c77302afdf6055544ea85b58e Mon Sep 17 00:00:00 2001 From: "Benjamin T. Liu" Date: Thu, 11 Mar 2021 16:28:24 -0800 Subject: [PATCH 30/63] Add braces --- src/chai/ArrayManager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/chai/ArrayManager.cpp b/src/chai/ArrayManager.cpp index ad801ee9..8a07c7cc 100644 --- a/src/chai/ArrayManager.cpp +++ b/src/chai/ArrayManager.cpp @@ -308,8 +308,9 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre alloc.deallocate(space_ptr); for (int space_t = CPU; space_t < NUM_EXECUTION_SPACES; ++space_t) { - if (space_ptr == pointer_record->m_pointers[space_t]) + if (space_ptr == pointer_record->m_pointers[space_t]) { pointer_record->m_pointers[space_t] = nullptr; + } } } else #endif @@ -324,8 +325,9 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre alloc.deallocate(space_ptr); for (int space_t = CPU; space_t < NUM_EXECUTION_SPACES; ++space_t) { - if (space_ptr == pointer_record->m_pointers[space_t]) + if (space_ptr == pointer_record->m_pointers[space_t]) { pointer_record->m_pointers[space_t] = nullptr; + } } } else #endif From dfd991bdec3998df09a8705c44fbbd6a3b72eaa8 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 16 Mar 2021 09:50:49 -0700 Subject: [PATCH 31/63] prevent 0 byte allocations in ManagedArray_thin.incl --- src/chai/ManagedArray_thin.inl | 35 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/chai/ManagedArray_thin.inl b/src/chai/ManagedArray_thin.inl index 5e3afffc..3f7c3ab7 100644 --- a/src/chai/ManagedArray_thin.inl +++ b/src/chai/ManagedArray_thin.inl @@ -142,20 +142,27 @@ CHAI_HOST void ManagedArray::allocate(size_t elems, ExecutionSpace space, UserCallback const &) { if (!m_is_slice) { - (void) space; // Quiet compiler warning when CHAI_LOG does nothing - CHAI_LOG(Debug, "Allocating array of size " << elems - << " in space " - << space); - - m_elems = elems; - - #if defined(CHAI_ENABLE_UM) - gpuMallocManaged(&m_active_pointer, sizeof(T) * elems); - #else // not CHAI_ENABLE_UM - m_active_pointer = static_cast(malloc(sizeof(T) * elems)); - #endif - - CHAI_LOG(Debug, "m_active_ptr allocated at address: " << m_active_pointer); + if (elems > 0) { + (void) space; // Quiet compiler warning when CHAI_LOG does nothing + CHAI_LOG(Debug, "Allocating array of size " << elems + << " in space " + << space); + + m_elems = elems; + + #if defined(CHAI_ENABLE_UM) + gpuMallocManaged(&m_active_pointer, sizeof(T) * elems); + #else // not CHAI_ENABLE_UM + m_active_pointer = static_cast(malloc(sizeof(T) * elems)); + #endif + + CHAI_LOG(Debug, "m_active_ptr allocated at address: " << m_active_pointer); + + } + else { + m_active_pointer = nullptr; + m_elems = 0; + } } else { CHAI_LOG(Debug, "Attempted to allocate slice!"); From b27c5475de3cb0551ebe004e7bcf395287d9a61c Mon Sep 17 00:00:00 2001 From: Alan Dayton Date: Wed, 17 Mar 2021 11:55:33 -0700 Subject: [PATCH 32/63] Give hip builds a chance of working --- src/chai/RajaExecutionSpacePlugin.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/chai/RajaExecutionSpacePlugin.cpp b/src/chai/RajaExecutionSpacePlugin.cpp index e5f4027f..e914d7f3 100644 --- a/src/chai/RajaExecutionSpacePlugin.cpp +++ b/src/chai/RajaExecutionSpacePlugin.cpp @@ -29,6 +29,10 @@ RajaExecutionSpacePlugin::preCapture(const RAJA::util::PluginContext& p) #if defined(CHAI_ENABLE_CUDA) case RAJA::Platform::cuda: m_arraymanager->setExecutionSpace(chai::GPU); break; +#endif +#if defined(CHAI_ENABLE_HIP) + case RAJA::Platform::hip: + m_arraymanager->setExecutionSpace(chai::GPU); break; #endif default: m_arraymanager->setExecutionSpace(chai::NONE); From 4bf9596220f1023442ae97b8895ab626ceab89d5 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 13:45:59 -0700 Subject: [PATCH 33/63] Update CI --- .gitlab/lassen-jobs.yml | 44 +++++++++++++++++++++++++++------ .gitlab/quartz-jobs.yml | 55 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 162bbbcd..049d26e2 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -8,22 +8,27 @@ # CPU ONLY ibm_clang_9: variables: - SPEC: " %clang@9.0.0ibm" + SPEC: "%clang@9.0.0ibm" extends: .build_and_test_on_lassen ibm_clang_9_gcc_8: variables: - SPEC: " %clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + SPEC: "%clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" + extends: .build_and_test_on_lassen + +clang_10_0_1_mpi (build and test on lassen): + variables: + SPEC: "%clang@10.0.1ibm +mpi ^spectrum-mpi" extends: .build_and_test_on_lassen gcc_8_3_1: variables: - SPEC: "%gcc@8.3.1" + SPEC: "%gcc@8.3.1 ^cmake@3.14.5" extends: .build_and_test_on_lassen pgi_19_10: variables: - SPEC: "%pgi@19.10 ^umpire@develop" + SPEC: "%pgi@19.10" extends: .build_and_test_on_lassen pgi_19_10_gcc_8_3_1: @@ -82,20 +87,45 @@ pgi_20_4_gcc_8_3_1_cuda_11: xl_16_1_1_7_cuda: variables: - SPEC: "+cuda %xl@16.1.1.7 ^cuda@10.1.168" + SPEC: "+cuda %xl@16.1.1.7 ^cuda@10.1.168 ^cmake@3.14.5" extends: .build_and_test_on_lassen xl_16_1_1_7_gcc_8_3_1_cuda_11: variables: - SPEC: "+cuda %xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" + SPEC: "+cuda %xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2 ^cmake@3.14.5" allow_failure: true extends: .build_and_test_on_lassen # EXTRAS +clang_9_0_0_libcpp (build and test on lassen): + variables: + SPEC: "%clang@9.0.0+libcpp" + extends: .build_and_test_on_lassen + clang_9_0_0_memleak (build and test on lassen): variables: - SPEC: "%clang@9.0.0 cxxflags=-fsanitize=address" + SPEC: "+asan +sanitizer_tests %clang@9.0.0 cxxflags=-fsanitize=address" ASAN_OPTIONS: "detect_leaks=1" + extends: .build_and_test_on_lassen + +clang_9_0_0_datarace (build and test on lassen): + variables: + SPEC: "%clang@9.0.0 cxxflags=-fsanitize=thread" allow_failure: true extends: .build_and_test_on_lassen + +gcc_8_3_1_dev_benchmarks: + variables: + SPEC: "+dev_benchmarks %gcc@8.3.1 build_type=Release ^cmake@3.14.5" + extends: .build_and_test_on_lassen + +xl_default_omp_target (build and test on lassen): + variables: + SPEC: "%xl@default+openmp+openmp_target ^cmake@3.14.5" + extends: .build_and_test_on_lassen + +gcc_8_3_1_numa: + variables: + SPEC: "+numa%gcc@8.3.1" + extends: .build_and_test_on_lassen diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index f02304f9..80d5a6cb 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -5,6 +5,7 @@ # SPDX-License-Identifier: (MIT) ############################################################################## clang_10: +<<<<<<< Updated upstream variables: SPEC: " %clang@10.0.1" extends: .build_and_test_on_quartz @@ -52,3 +53,57 @@ gcc_4_9_3: variables: SPEC: "%gcc@4.9.3" extends: .build_and_test_on_quartz +======= + variables: + SPEC: "+fortran %clang@10.0.1" + extends: .build_and_test_on_ruby + +clang_9: + variables: + SPEC: "+fortran %clang@9.0.0" + extends: .build_and_test_on_ruby + +gcc_8_1_0: + variables: + SPEC: "+fortran %gcc@8.1.0" + extends: .build_and_test_on_ruby + +icpc_17_0_2: + variables: + SPEC: "+fortran %intel@17.0.2" + extends: .build_and_test_on_ruby + +icpc_18_0_2: + variables: + SPEC: "+fortran tests=none %intel@18.0.2" + extends: .build_and_test_on_ruby + +icpc_19_1_0: + variables: + SPEC: "+fortran %intel@19.1.0" + extends: .build_and_test_on_ruby + +pgi_19_7: + variables: + SPEC: "+fortran %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" + PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 + extends: .build_and_test_on_ruby + +pgi_20_1: + variables: + SPEC: "+fortran %pgi@20.1" + PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 + extends: .build_and_test_on_ruby + +# EXTRAS + +gcc_4_9_3: + variables: + SPEC: "%gcc@4.9.3" + extends: .build_and_test_on_ruby + +gcc_8_1_0_numa: + variables: + SPEC: "+fortran+numa %gcc@8.1.0" + extends: .build_and_test_on_ruby +>>>>>>> Stashed changes From ee6db27c7e1dfbea1d0928feda7bc8655b433b62 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 13:47:11 -0700 Subject: [PATCH 34/63] Update radiuss-spack-configs --- scripts/radiuss-spack-configs | 2 +- scripts/uberenv/spack_configs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs index 1ed8697e..aa21de2d 160000 --- a/scripts/radiuss-spack-configs +++ b/scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit 1ed8697e764798b6f38b2ee59389b24a0b66f10c +Subproject commit aa21de2ddf3576dd7ceac4303508d30632c0c083 diff --git a/scripts/uberenv/spack_configs b/scripts/uberenv/spack_configs index 2c6464c5..17d3bf7a 120000 --- a/scripts/uberenv/spack_configs +++ b/scripts/uberenv/spack_configs @@ -1 +1 @@ -../radiuss-spack-configs/spack_configs \ No newline at end of file +../radiuss-spack-configs \ No newline at end of file From 7089f60e8d7442bc3605cd3ea0313936add1c6f7 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 14:03:19 -0700 Subject: [PATCH 35/63] Fixup quartz-jobs --- .gitlab/quartz-jobs.yml | 65 ++++------------------------------------- 1 file changed, 5 insertions(+), 60 deletions(-) diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 80d5a6cb..43e964cb 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -5,7 +5,6 @@ # SPDX-License-Identifier: (MIT) ############################################################################## clang_10: -<<<<<<< Updated upstream variables: SPEC: " %clang@10.0.1" extends: .build_and_test_on_quartz @@ -17,27 +16,27 @@ clang_9: gcc_8_1_0: variables: - SPEC: " %gcc@8.1.0" + SPEC: "%gcc@8.1.0" extends: .build_and_test_on_quartz icpc_17_0_2: variables: - SPEC: " %intel@17.0.2" + SPEC: "%intel@17.0.2" extends: .build_and_test_on_quartz icpc_18_0_2: variables: - SPEC: " tests=none %intel@18.0.2" + SPEC: "tests=none %intel@18.0.2" extends: .build_and_test_on_quartz icpc_19_1_0: variables: - SPEC: " %intel@19.1.0" + SPEC: "%intel@19.1.0" extends: .build_and_test_on_quartz pgi_19_7: variables: - SPEC: " %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" + SPEC: "%pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 extends: .build_and_test_on_quartz @@ -53,57 +52,3 @@ gcc_4_9_3: variables: SPEC: "%gcc@4.9.3" extends: .build_and_test_on_quartz -======= - variables: - SPEC: "+fortran %clang@10.0.1" - extends: .build_and_test_on_ruby - -clang_9: - variables: - SPEC: "+fortran %clang@9.0.0" - extends: .build_and_test_on_ruby - -gcc_8_1_0: - variables: - SPEC: "+fortran %gcc@8.1.0" - extends: .build_and_test_on_ruby - -icpc_17_0_2: - variables: - SPEC: "+fortran %intel@17.0.2" - extends: .build_and_test_on_ruby - -icpc_18_0_2: - variables: - SPEC: "+fortran tests=none %intel@18.0.2" - extends: .build_and_test_on_ruby - -icpc_19_1_0: - variables: - SPEC: "+fortran %intel@19.1.0" - extends: .build_and_test_on_ruby - -pgi_19_7: - variables: - SPEC: "+fortran %pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" - PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 - extends: .build_and_test_on_ruby - -pgi_20_1: - variables: - SPEC: "+fortran %pgi@20.1" - PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 - extends: .build_and_test_on_ruby - -# EXTRAS - -gcc_4_9_3: - variables: - SPEC: "%gcc@4.9.3" - extends: .build_and_test_on_ruby - -gcc_8_1_0_numa: - variables: - SPEC: "+fortran+numa %gcc@8.1.0" - extends: .build_and_test_on_ruby ->>>>>>> Stashed changes From bf6d284dcca133ed0278b2eb337def4118a3790d Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 14:45:26 -0700 Subject: [PATCH 36/63] Update jobs to remove ones that don't apply --- .gitlab-ci.yml | 2 +- .gitlab/lassen-jobs.yml | 24 ++---------------------- .gitlab/quartz-jobs.yml | 2 +- scripts/uberenv/packages/chai/package.py | 2 +- 4 files changed, 5 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2414f786..5a9da66e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,7 +65,7 @@ stages: # allow pre-allocation the same way slurm does. .build_blueos_3_ppc64le_ib_script: script: - - lalloc 1 -W 10 scripts/gitlab/build_and_test.sh + - lalloc 1 -W 20 scripts/gitlab/build_and_test.sh .build_blueos_3_ppc64le_ib_p9_script: extends: .build_blueos_3_ppc64le_ib_script diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 049d26e2..3d2fe1e5 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -16,11 +16,6 @@ ibm_clang_9_gcc_8: SPEC: "%clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen -clang_10_0_1_mpi (build and test on lassen): - variables: - SPEC: "%clang@10.0.1ibm +mpi ^spectrum-mpi" - extends: .build_and_test_on_lassen - gcc_8_3_1: variables: SPEC: "%gcc@8.3.1 ^cmake@3.14.5" @@ -105,7 +100,7 @@ clang_9_0_0_libcpp (build and test on lassen): clang_9_0_0_memleak (build and test on lassen): variables: - SPEC: "+asan +sanitizer_tests %clang@9.0.0 cxxflags=-fsanitize=address" + SPEC: "%clang@9.0.0 cxxflags=-fsanitize=address" ASAN_OPTIONS: "detect_leaks=1" extends: .build_and_test_on_lassen @@ -113,19 +108,4 @@ clang_9_0_0_datarace (build and test on lassen): variables: SPEC: "%clang@9.0.0 cxxflags=-fsanitize=thread" allow_failure: true - extends: .build_and_test_on_lassen - -gcc_8_3_1_dev_benchmarks: - variables: - SPEC: "+dev_benchmarks %gcc@8.3.1 build_type=Release ^cmake@3.14.5" - extends: .build_and_test_on_lassen - -xl_default_omp_target (build and test on lassen): - variables: - SPEC: "%xl@default+openmp+openmp_target ^cmake@3.14.5" - extends: .build_and_test_on_lassen - -gcc_8_3_1_numa: - variables: - SPEC: "+numa%gcc@8.3.1" - extends: .build_and_test_on_lassen + extends: .build_and_test_on_lassen \ No newline at end of file diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 43e964cb..52932b48 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -26,7 +26,7 @@ icpc_17_0_2: icpc_18_0_2: variables: - SPEC: "tests=none %intel@18.0.2" + SPEC: " tests=none %intel@18.0.2" extends: .build_and_test_on_quartz icpc_19_1_0: diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 0ca965c8..fe7b2442 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -75,7 +75,7 @@ class Chai(CMakePackage, CudaPackage): depends_on('cmake@3.8:', type='build') depends_on('umpire@develop') - depends_on('raja', when="+raja") + depends_on('raja@main', when="+raja") depends_on('raja tests=none', when='tests=none') depends_on('umpire tests=none', when='tests=none') From 039a47a7b2f484bcaf568e66505ef5ce9866e957 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 15:17:31 -0700 Subject: [PATCH 37/63] Make RAJA enabled by default --- scripts/uberenv/packages/chai/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index fe7b2442..290da95e 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -68,8 +68,8 @@ class Chai(CMakePackage, CudaPackage): version('1.1.0', tag='v1.1.0', submodules='True') version('1.0', tag='v1.0', submodules='True') - variant('shared', default=True, description='Build Shared Libs') - variant('raja', default=False, description='Build plugin for RAJA') + variant('shared', default=False, description='Build Shared Libs') + variant('raja', default=True, description='Build plugin for RAJA') variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') From f83c62e37a654cc83cb7730eb7b8a38d349b4e39 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 15:18:24 -0700 Subject: [PATCH 38/63] Don't build tests of dependencies --- scripts/uberenv/packages/chai/package.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 290da95e..70c2b50e 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -76,9 +76,8 @@ class Chai(CMakePackage, CudaPackage): depends_on('cmake@3.8:', type='build') depends_on('umpire@develop') depends_on('raja@main', when="+raja") - - depends_on('raja tests=none', when='tests=none') - depends_on('umpire tests=none', when='tests=none') + depends_on('raja tests=none') + depends_on('umpire tests=none') depends_on('cmake@3.9:', type='build', when="+cuda") depends_on('umpire+cuda', when="+cuda") From 22523ed164186615ed1cfb7ca76ef5b00c1a108f Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 15:31:02 -0700 Subject: [PATCH 39/63] Update packages --- scripts/uberenv/packages/chai/package.py | 2 +- scripts/uberenv/packages/raja/package.py | 6 +++--- scripts/uberenv/packages/umpire/package.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 70c2b50e..d9ed2fc8 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -75,7 +75,7 @@ class Chai(CMakePackage, CudaPackage): depends_on('cmake@3.8:', type='build') depends_on('umpire@develop') - depends_on('raja@main', when="+raja") + depends_on('raja@develop', when="+raja") depends_on('raja tests=none') depends_on('umpire tests=none') diff --git a/scripts/uberenv/packages/raja/package.py b/scripts/uberenv/packages/raja/package.py index 605cc046..9cf4ac47 100644 --- a/scripts/uberenv/packages/raja/package.py +++ b/scripts/uberenv/packages/raja/package.py @@ -71,9 +71,9 @@ class Raja(CMakePackage, CudaPackage): version('0.4.1', tag='v0.4.1', submodules="True") version('0.4.0', tag='v0.4.0', submodules="True") - variant('openmp', default=True, description='Build OpenMP backend') - variant('shared', default=True, description='Build Shared Libs') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), + variant('openmp', default=False, description='Build OpenMP backend') + variant('shared', default=False, description='Build Shared Libs') + variant('tests', default='none', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') depends_on('cmake@3.8:', type='build') diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/uberenv/packages/umpire/package.py index 2ae7a5d0..6acf7ba0 100644 --- a/scripts/uberenv/packages/umpire/package.py +++ b/scripts/uberenv/packages/umpire/package.py @@ -85,12 +85,12 @@ class Umpire(CMakePackage, CudaPackage): variant('fortran', default=False, description='Build C/Fortran API') variant('c', default=False, description='Build C API') variant('numa', default=False, description='Enable NUMA support') - variant('shared', default=True, description='Enable Shared libs') + variant('shared', default=False, description='Enable Shared libs') variant('openmp', default=False, description='Build with OpenMP support') variant('openmp_target', default=False, description='Build with OpenMP 4.5 support') variant('deviceconst', default=False, description='Enables support for constant device memory') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), + variant('tests', default='none', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') From 38e4daefca7926ddc108bc5dee8822995f2521ee Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 15:35:42 -0700 Subject: [PATCH 40/63] Update RAJA/Umpire packages from their repos --- scripts/uberenv/packages/raja/package.py | 96 +++++++++++++++++----- scripts/uberenv/packages/umpire/package.py | 69 ++++++++++++++++ 2 files changed, 143 insertions(+), 22 deletions(-) diff --git a/scripts/uberenv/packages/raja/package.py b/scripts/uberenv/packages/raja/package.py index 9cf4ac47..c7ea7117 100644 --- a/scripts/uberenv/packages/raja/package.py +++ b/scripts/uberenv/packages/raja/package.py @@ -12,6 +12,8 @@ from os import environ as env from os.path import join as pjoin +import re + def cmake_cache_entry(name, value, comment=""): """Generate a string for a cmake cache variable""" @@ -56,7 +58,9 @@ class Raja(CMakePackage, CudaPackage): git = "https://github.com/LLNL/RAJA.git" version('develop', branch='develop', submodules='True') - version('master', branch='main', submodules='True') + version('main', branch='main', submodules='True') + version('0.12.1', tag='v0.12.1', submodules="True") + version('0.12.0', tag='v0.12.0', submodules="True") version('0.11.0', tag='v0.11.0', submodules="True") version('0.10.1', tag='v0.10.1', submodules="True") version('0.10.0', tag='v0.10.0', submodules="True") @@ -73,11 +77,16 @@ class Raja(CMakePackage, CudaPackage): variant('openmp', default=False, description='Build OpenMP backend') variant('shared', default=False, description='Build Shared Libs') + variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') + variant('hip', default=False, description='Build with HIP support') variant('tests', default='none', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') depends_on('cmake@3.8:', type='build') depends_on('cmake@3.9:', when='+cuda', type='build') + depends_on('hip', when='+hip') + + conflicts('+openmp', when='+hip') phases = ['hostconfig', 'cmake', 'build', 'install'] @@ -92,6 +101,8 @@ def _get_host_config_path(self, spec): var='' if '+cuda' in spec: var= '-'.join([var,'cuda']) + if '+libcpp' in spec: + var='-'.join([var,'libcpp']) host_config_path = "hc-%s-%s-%s%s-%s.cmake" % (socket.gethostname().rstrip('1234567890'), self._get_sys_type(spec), @@ -183,10 +194,14 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): # use global spack compiler flags cflags = ' '.join(spec.compiler_flags['cflags']) + if "+libcpp" in spec: + cflags += ' '.join([cflags,"-DGTEST_HAS_CXXABI_H_=0"]) if cflags: cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) + if "+libcpp" in spec: + cxxflags += ' '.join([cxxflags,"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0"]) if cxxflags: cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) @@ -203,21 +218,18 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, description)) - if "toss_3_x86_64_ib" in sys_type: - release_flags = "-O3 -msse4.2 -funroll-loops -finline-functions" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) - reldebinf_flags = "-O3 -g -msse4.2 -funroll-loops -finline-functions" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELWITHDEBINFO", reldebinf_flags)) - debug_flags = "-O0 -g" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - - if "blueos_3_ppc64le_ib" in sys_type: - release_flags = "-O3" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) - reldebinf_flags = "-O3 -g" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELWITHDEBINFO", reldebinf_flags)) - debug_flags = "-O0 -g" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) + gcc_toolchain_regex = re.compile("--gcc-toolchain=(.*)") + gcc_name_regex = re.compile(".*gcc-name.*") + + using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) + if(using_toolchain): + gcc_toolchain_path = gcc_toolchain_regex.match(using_toolchain[0]) + using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) + compilers_using_toolchain = ["pgi", "xl", "icpc"] + if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): + if using_toolchain or using_gcc_name: + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) if "+cuda" in spec: cfg.write("#------------------{0}\n".format("-" * 60)) @@ -233,14 +245,52 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", cudacompiler)) + cuda_release_flags = "-O3 -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" + cuda_reldebinf_flags = "-O3 -g -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" + cuda_debug_flags = "-O0 -g -Xcompiler -O0 -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" + + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELEASE", cuda_release_flags)) + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELWITHDEBINFO", cuda_reldebinf_flags)) + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_DEBUG", cuda_debug_flags)) + if not spec.satisfies('cuda_arch=none'): cuda_arch = spec.variants['cuda_arch'].value - flag = '-arch sm_{0}'.format(cuda_arch[0]) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", flag)) + cfg.write(cmake_cache_string("CUDA_ARCH", 'sm_{0}'.format(cuda_arch[0]))) else: cfg.write(cmake_cache_option("ENABLE_CUDA", False)) + if "+hip" in spec: + cfg.write("#------------------{0}\n".format("-" * 60)) + cfg.write("# HIP\n") + cfg.write("#------------------{0}\n\n".format("-" * 60)) + + cfg.write(cmake_cache_option("ENABLE_HIP", True)) + + hip_root = spec['hip'].prefix + rocm_root = hip_root + "/.." + cfg.write(cmake_cache_entry("HIP_ROOT_DIR", + hip_root)) + cfg.write(cmake_cache_entry("HIP_CLANG_PATH", + rocm_root + '/llvm/bin')) + cfg.write(cmake_cache_entry("HIP_HIPCC_FLAGS", + '--amdgpu-target=gfx906')) + cfg.write(cmake_cache_entry("HIP_RUNTIME_INCLUDE_DIRS", + "{0}/include;{0}/../hsa/include".format(hip_root))) + if ('%gcc' in spec) or (using_toolchain): + if ('%gcc' in spec): + gcc_bin = os.path.dirname(self.compiler.cxx) + gcc_prefix = join_path(gcc_bin, '..') + else: + gcc_prefix = gcc_toolchain_path.group(1) + cfg.write(cmake_cache_entry("HIP_CLANG_FLAGS", + "--gcc-toolchain={0}".format(gcc_prefix))) + cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", + "-Wl,-rpath {}/lib64".format(gcc_prefix))) + + else: + cfg.write(cmake_cache_option("ENABLE_HIP", False)) + cfg.write("#------------------{0}\n".format("-" * 60)) cfg.write("# Other\n") cfg.write("#------------------{0}\n\n".format("-" * 60)) @@ -255,16 +305,18 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option("BUILD_SHARED_LIBS","+shared" in spec)) cfg.write(cmake_cache_option("ENABLE_OPENMP","+openmp" in spec)) - # Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS which - # is used by the spack compiler wrapper. This can go away when BLT - # removes -Werror from GTest flags + # Note 1: Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS + # which is used by the spack compiler wrapper. This can go away when + # BLT removes -Werror from GTest flags + # Note 2: Tests are either built if variant is set, or if run-tests + # option is passed. if self.spec.satisfies('%clang target=ppc64le:'): cfg.write(cmake_cache_option("ENABLE_TESTS",False)) if 'tests=benchmarks' in spec or not 'tests=none' in spec: print("MSG: no testing supported on %clang target=ppc64le:") else: cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) - cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec)) + cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec or self.run_tests)) ####################### # Close and save diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/uberenv/packages/umpire/package.py index 6acf7ba0..0282d560 100644 --- a/scripts/uberenv/packages/umpire/package.py +++ b/scripts/uberenv/packages/umpire/package.py @@ -84,6 +84,7 @@ class Umpire(CMakePackage, CudaPackage): variant('fortran', default=False, description='Build C/Fortran API') variant('c', default=False, description='Build C API') + variant('mpi', default=False, description='Enable MPI support') variant('numa', default=False, description='Enable NUMA support') variant('shared', default=False, description='Enable Shared libs') variant('openmp', default=False, description='Build with OpenMP support') @@ -94,13 +95,26 @@ class Umpire(CMakePackage, CudaPackage): multi=False, description='Tests to run') variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') + variant('hip', default=False, description='Build with HIP support') + variant('tools', default=False, description='Enable tools') + variant('dev_benchmarks', default=False, description='Enable Developer Benchmarks') + variant('werror', default=False, description='Enable warnings as errors') + variant('asan', default=False, description='Enable ASAN') + variant('sanitizer_tests', default=False, description='Enable address sanitizer tests') depends_on('cmake@3.8:', type='build') depends_on('cmake@3.9:', when='+cuda', type='build') + depends_on('mpi', when='+mpi') + depends_on('hip', when='+hip') conflicts('+numa', when='@:0.3.2') conflicts('~c', when='+fortran', msg='Fortran API requires C API') conflicts('~openmp', when='+openmp_target', msg='OpenMP target requires OpenMP') + conflicts('+cuda', when='+hip') + conflicts('+openmp', when='+hip') + conflicts('+openmp_target', when='+hip') + conflicts('+deviceconst', when='~hip~cuda') + conflicts('+sanitizer_tests', when='~asan') phases = ['hostconfig', 'cmake', 'build', 'install'] @@ -266,6 +280,19 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS_RELWITHDEBINFO", reldebinf_flags)) cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) + #Configuration to enable developer benchmarks (i.e. No-Op) + if "+dev_benchmarks" in spec: + cfg.write("#------------------{0}\n".format("-" * 60)) + cfg.write("# Developer Benchmarks\n") + cfg.write("#------------------{0}\n\n".format("-" * 60)) + + cfg.write(cmake_cache_option("ENABLE_DEVELOPER_BENCHMARKS", True)) + cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", True)) #Enable BLT GoogleBenchmark support + else: + cfg.write(cmake_cache_option("ENABLE_DEVELOPER_BENCHMARKS", False)) + cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", False)) + + if "+cuda" in spec: cfg.write("#------------------{0}\n".format("-" * 60)) cfg.write("# Cuda\n") @@ -297,8 +324,46 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): else: cfg.write(cmake_cache_option("ENABLE_CUDA", False)) + if "+hip" in spec: + cfg.write("#------------------{0}\n".format("-" * 60)) + cfg.write("# HIP\n") + cfg.write("#------------------{0}\n\n".format("-" * 60)) + + cfg.write(cmake_cache_option("ENABLE_HIP", True)) + +# -DHIP_ROOT_DIR=/opt/rocm-3.6.0/hip -DHIP_CLANG_PATH=/opt/rocm-3.6.0/llvm/bin + + hip_root = spec['hip'].prefix + rocm_root = hip_root + "/.." + cfg.write(cmake_cache_entry("HIP_ROOT_DIR", + hip_root)) + cfg.write(cmake_cache_entry("HIP_CLANG_PATH", + rocm_root + '/llvm/bin')) + cfg.write(cmake_cache_entry("HIP_HIPCC_FLAGS", + '--amdgpu-target=gfx906')) + cfg.write(cmake_cache_entry("HIP_RUNTIME_INCLUDE_DIRS", + "{0}/include;{0}/../hsa/include".format(hip_root))) + if '%gcc' in spec: + gcc_bin = os.path.dirname(self.compiler.cxx) + gcc_prefix = join_path(gcc_bin, '..') + cfg.write(cmake_cache_entry("HIP_CLANG_FLAGS", + "--gcc-toolchain={0}".format(gcc_prefix))) + cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", + "-Wl,-rpath {}/lib64".format(gcc_prefix))) + + if '+deviceconst' in spec: + cfg.write(cmake_cache_option("ENABLE_DEVICE_CONST", True)) + + else: + cfg.write(cmake_cache_option("ENABLE_HIP", False)) + cfg.write(cmake_cache_option("ENABLE_C", '+c' in spec)) cfg.write(cmake_cache_option("ENABLE_FORTRAN", '+fortran' in spec)) + + if "+mpi" in spec: + cfg.write(cmake_cache_option("ENABLE_MPI", '+mpi' in spec)) + cfg.write(cmake_cache_entry("MPI_CXX_COMPILER", spec['mpi'].mpicxx)) + cfg.write(cmake_cache_option("ENABLE_NUMA", '+numa' in spec)) cfg.write(cmake_cache_option("ENABLE_OPENMP", '+openmp' in spec)) if "+openmp_target" in spec: @@ -308,6 +373,10 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec)) + cfg.write(cmake_cache_option("ENABLE_TOOLS", '+tools' in spec)) + cfg.write(cmake_cache_option("ENABLE_WARNINGS_AS_ERRORS", '+werror' in spec)) + cfg.write(cmake_cache_option("ENABLE_ASAN", '+asan' in spec)) + cfg.write(cmake_cache_option("ENABLE_SANITIZER_TESTS", '+sanitizer_tests' in spec)) ####################### # Close and save From d214120533d03e442676d10d1a21ebcba25d7e6c Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Fri, 30 Apr 2021 16:05:04 -0700 Subject: [PATCH 41/63] Try removing conflicting depends --- scripts/uberenv/packages/chai/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index d9ed2fc8..8ca0949b 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -76,8 +76,6 @@ class Chai(CMakePackage, CudaPackage): depends_on('cmake@3.8:', type='build') depends_on('umpire@develop') depends_on('raja@develop', when="+raja") - depends_on('raja tests=none') - depends_on('umpire tests=none') depends_on('cmake@3.9:', type='build', when="+cuda") depends_on('umpire+cuda', when="+cuda") From e96adf7022bb533ab8246fd288ae9bb3a9e64c55 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Mon, 3 May 2021 11:46:00 -0700 Subject: [PATCH 42/63] Build with RAJA/Umpire @main --- scripts/uberenv/packages/chai/package.py | 4 ++-- scripts/uberenv/packages/umpire/package.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 8ca0949b..fb88d55c 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -74,8 +74,8 @@ class Chai(CMakePackage, CudaPackage): multi=False, description='Tests to run') depends_on('cmake@3.8:', type='build') - depends_on('umpire@develop') - depends_on('raja@develop', when="+raja") + depends_on('umpire@main') + depends_on('raja@main', when="+raja") depends_on('cmake@3.9:', type='build', when="+cuda") depends_on('umpire+cuda', when="+cuda") diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/uberenv/packages/umpire/package.py index 0282d560..8bc83634 100644 --- a/scripts/uberenv/packages/umpire/package.py +++ b/scripts/uberenv/packages/umpire/package.py @@ -58,7 +58,7 @@ class Umpire(CMakePackage, CudaPackage): git = 'https://github.com/LLNL/Umpire.git' version('develop', branch='develop', submodules='True') - version('master', branch='main', submodules='True') + version('main', branch='main', submodules='True') version('4.1.2', tag='v4.1.2', submodules='True') version('3.0.0', tag='v3.0.0', submodules='True') version('2.1.0', tag='v2.1.0', submodules='True') From ebf41382cebb1fc94171243d89f61c8b2e446f79 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 4 May 2021 15:51:07 -0700 Subject: [PATCH 43/63] Remove some PGI jobs --- .gitlab/lassen-jobs.yml | 32 -------------------------------- .gitlab/quartz-jobs.yml | 6 ------ 2 files changed, 38 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 3d2fe1e5..564efbb4 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -21,21 +21,6 @@ gcc_8_3_1: SPEC: "%gcc@8.3.1 ^cmake@3.14.5" extends: .build_and_test_on_lassen -pgi_19_10: - variables: - SPEC: "%pgi@19.10" - extends: .build_and_test_on_lassen - -pgi_19_10_gcc_8_3_1: - variables: - SPEC: "%pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" - extends: .build_and_test_on_lassen - -pgi_20_4_gcc_8_3_1: - variables: - SPEC: "%pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" - extends: .build_and_test_on_lassen - xl_16_1_1_7: variables: SPEC: "%xl@16.1.1.7" @@ -63,23 +48,6 @@ gcc_8_3_1_cuda: SPEC: "+cuda %gcc@8.3.1 ^cuda@10.1.168" extends: .build_and_test_on_lassen -pgi_19_10_cuda: - variables: - SPEC: "+cuda %pgi@19.10 ^cuda@10.1.168" - extends: .build_and_test_on_lassen - -pgi_19_10_gcc_8_3_1_cuda: - variables: - SPEC: "+cuda %pgi@19.10 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" - allow_failure: true - extends: .build_and_test_on_lassen - -pgi_20_4_gcc_8_3_1_cuda_11: - variables: - SPEC: "+cuda %pgi@20.4 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2" - allow_failure: true - extends: .build_and_test_on_lassen - xl_16_1_1_7_cuda: variables: SPEC: "+cuda %xl@16.1.1.7 ^cuda@10.1.168 ^cmake@3.14.5" diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 52932b48..cc7b409a 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -34,12 +34,6 @@ icpc_19_1_0: SPEC: "%intel@19.1.0" extends: .build_and_test_on_quartz -pgi_19_7: - variables: - SPEC: "%pgi@19.7 cxxflags=-I/usr/tce/packages/gcc/gcc-4.9.3/include/c++/4.9.3" - PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 - extends: .build_and_test_on_quartz - pgi_20_1: variables: SPEC: " %pgi@20.1" From 31fd4a6d86df274a30b4b8f0ae75f9b83aa14a6f Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 5 May 2021 08:06:23 -0700 Subject: [PATCH 44/63] Add libc++ variant to CHAI --- scripts/uberenv/packages/chai/package.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index fb88d55c..13535986 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -72,6 +72,7 @@ class Chai(CMakePackage, CudaPackage): variant('raja', default=True, description='Build plugin for RAJA') variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') + variant('libcpp', default=False, description='Use libc++') depends_on('cmake@3.8:', type='build') depends_on('umpire@main') @@ -189,6 +190,8 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) + if "+libcpp" in spec: + cxxflags += ' '.join([cxxflags,"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0"]) if cxxflags: cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) From 1bab650bb13e631baec239ae40b5bc4c2f8a2603 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 5 May 2021 08:10:17 -0700 Subject: [PATCH 45/63] Update BLT to 0.4.0 --- blt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blt b/blt index cbe99c93..7ec2cb80 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit cbe99c93d0d83ca04f13018cdfed9a3663ea248c +Subproject commit 7ec2cb80525b55d06da683a876b382472ef70661 From 64cc0549e05f71559314e6f0c1ba2456f4eec06b Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 5 May 2021 12:03:15 -0700 Subject: [PATCH 46/63] Propagate libcpp dependency --- scripts/uberenv/packages/chai/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/uberenv/packages/chai/package.py index 13535986..28442947 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/uberenv/packages/chai/package.py @@ -83,6 +83,7 @@ class Chai(CMakePackage, CudaPackage): depends_on('raja+cuda', when="+raja+cuda") depends_on('umpire+cuda+allow-untested-versions', when="+cuda+allow-untested-versions") depends_on('raja+cuda+allow-untested-versions', when="+raja+cuda+allow-untested-versions") + depends_on('umpire+libcpp', when='+libcpp') phases = ['hostconfig', 'cmake', 'build', 'install'] From 2ad27734942eccca7a06ab2c1dc950e2ad04ce0d Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 12 May 2021 07:45:43 -0700 Subject: [PATCH 47/63] Delete pointer record in free --- src/chai/ManagedArray.inl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/chai/ManagedArray.inl b/src/chai/ManagedArray.inl index 33db39d0..1796f614 100644 --- a/src/chai/ManagedArray.inl +++ b/src/chai/ManagedArray.inl @@ -268,6 +268,7 @@ CHAI_HOST void ManagedArray::free(ExecutionSpace space) m_offset = 0; // The call to m_resource_manager::free, above, has deallocated m_pointer_record if space == NONE. if (space == NONE) { + delete m_pointer_record; m_pointer_record = &ArrayManager::s_null_record; } } else { From 4385219cdf4cf224d38b8e36e643283bfe2b7085 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 12 May 2021 08:32:30 -0700 Subject: [PATCH 48/63] Fix leaked pointer record --- src/chai/ManagedArray.inl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/chai/ManagedArray.inl b/src/chai/ManagedArray.inl index 1796f614..64426d65 100644 --- a/src/chai/ManagedArray.inl +++ b/src/chai/ManagedArray.inl @@ -268,11 +268,16 @@ CHAI_HOST void ManagedArray::free(ExecutionSpace space) m_offset = 0; // The call to m_resource_manager::free, above, has deallocated m_pointer_record if space == NONE. if (space == NONE) { - delete m_pointer_record; m_pointer_record = &ArrayManager::s_null_record; } } else { - CHAI_LOG(Debug, "Cannot free a slice!"); + // Clean up if pointer record exists but this array has length 0 + if (m_pointer_record != &ArrayManager::s_null_record && (*this == nullptr)) { + delete m_pointer_record; + m_pointer_record = &ArrayManager::s_null_record; + } else { + CHAI_LOG(Debug, "Cannot free a slice!"); + } } } From 24bd025fd561d76b7fc19c04eff73e836e99920b Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 12 May 2021 11:38:22 -0700 Subject: [PATCH 49/63] Fixup leaks --- src/chai/ManagedArray.inl | 10 ++-------- tests/integration/managed_array_tests.cpp | 8 ++++++++ tests/integration/raja-chai-nested.cpp | 12 +++++++++++- tests/integration/raja-chai-tests.cpp | 9 +++++++++ tests/unit/array_manager_unit_tests.cpp | 3 +++ 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/chai/ManagedArray.inl b/src/chai/ManagedArray.inl index 64426d65..a8048c0b 100644 --- a/src/chai/ManagedArray.inl +++ b/src/chai/ManagedArray.inl @@ -253,7 +253,7 @@ template CHAI_INLINE CHAI_HOST void ManagedArray::free(ExecutionSpace space) { - if(!m_is_slice && *this != nullptr) { + if(!m_is_slice) { if (m_resource_manager == nullptr) { m_resource_manager = ArrayManager::getInstance(); } @@ -271,13 +271,7 @@ CHAI_HOST void ManagedArray::free(ExecutionSpace space) m_pointer_record = &ArrayManager::s_null_record; } } else { - // Clean up if pointer record exists but this array has length 0 - if (m_pointer_record != &ArrayManager::s_null_record && (*this == nullptr)) { - delete m_pointer_record; - m_pointer_record = &ArrayManager::s_null_record; - } else { - CHAI_LOG(Debug, "Cannot free a slice!"); - } + CHAI_LOG(Debug, "Cannot free a slice!"); } } diff --git a/tests/integration/managed_array_tests.cpp b/tests/integration/managed_array_tests.cpp index dc0fdb23..510267e6 100644 --- a/tests/integration/managed_array_tests.cpp +++ b/tests/integration/managed_array_tests.cpp @@ -1941,6 +1941,8 @@ TEST(ManagedArray, NoAllocation) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } TEST(ManagedArray, NoAllocationNull) @@ -1953,6 +1955,8 @@ TEST(ManagedArray, NoAllocationNull) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } #if defined(CHAI_ENABLE_CUDA) || defined(CHAI_ENABLE_HIP) @@ -1965,6 +1969,8 @@ GPU_TEST(ManagedArray, NoAllocationGPU) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } GPU_TEST(ManagedArray, NoAllocationNullGPU) @@ -1977,5 +1983,7 @@ GPU_TEST(ManagedArray, NoAllocationNullGPU) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } #endif diff --git a/tests/integration/raja-chai-nested.cpp b/tests/integration/raja-chai-nested.cpp index 9f721911..152a18a2 100644 --- a/tests/integration/raja-chai-nested.cpp +++ b/tests/integration/raja-chai-nested.cpp @@ -109,6 +109,9 @@ CUDA_TEST(Chai, NestedSimple) ASSERT_FLOAT_EQ(v1[index], index * 1.0f); ASSERT_FLOAT_EQ(v2[index], index * 2.0f); }); + + v1.free(); + v2.free(); } CUDA_TEST(Chai, NestedView) @@ -169,6 +172,9 @@ CUDA_TEST(Chai, NestedView) [=](int i, int j) { ASSERT_FLOAT_EQ(v2(i, j), v1(i, j) * 2.0f); }); + + v1_array.free(); + v2_array.free(); } CUDA_TEST(Chai, NestedMultiView) @@ -237,6 +243,8 @@ CUDA_TEST(Chai, NestedMultiView) [=](int i, int j) { ASSERT_FLOAT_EQ(mview(1, i, j), mview(0, i, j) * 2.0f); }); + v1_array.free(); + v2_array.free(); } /////////////////////////////////////////////////////////////////////////// @@ -368,7 +376,9 @@ void runLTimesTests(Index_type num_moments, } }); - //rm->setExecutionSpace(chai::NONE); + L_data.free(); + psi_data.free(); + phi_data.free(); } TEST(Chai, LTimes) diff --git a/tests/integration/raja-chai-tests.cpp b/tests/integration/raja-chai-tests.cpp index f20af4a9..6e7d6dd3 100644 --- a/tests/integration/raja-chai-tests.cpp +++ b/tests/integration/raja-chai-tests.cpp @@ -63,6 +63,9 @@ CUDA_TEST(ChaiTest, Simple) ASSERT_FLOAT_EQ(raw_v2[i], i * 2.0f * 2.0f); ; } + + v1.free(); + v2.free(); } CUDA_TEST(ChaiTest, Views) @@ -96,6 +99,9 @@ CUDA_TEST(ChaiTest, Views) ASSERT_FLOAT_EQ(raw_v2[i], i * 1.0f * 2.0f * 2.0f); ; } + + v1_array.free(); + v2_array.free(); } CUDA_TEST(ChaiTest, MultiView) @@ -138,4 +144,7 @@ CUDA_TEST(ChaiTest, MultiView) ASSERT_FLOAT_EQ(raw_v2[i], i * 1.0f * 2.0f * 2.0f); ; } + + v1_array.free(); + v2_array.free(); } diff --git a/tests/unit/array_manager_unit_tests.cpp b/tests/unit/array_manager_unit_tests.cpp index 5e1e68ea..3ea44ddc 100644 --- a/tests/unit/array_manager_unit_tests.cpp +++ b/tests/unit/array_manager_unit_tests.cpp @@ -72,6 +72,9 @@ TEST(ArrayManager, getPointerMap) ASSERT_EQ(rm->getTotalNumArrays(), 2); ASSERT_EQ(rm->getTotalSize(), (sizeOfArray1 * sizeof(int)) + (sizeOfArray2 * sizeof(double))); + + array1.free(); + array2.free(); } /*! From cfde35c8168d4b9d4dfe6986d9dfc20399bcf481 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 13 May 2021 08:35:48 -0700 Subject: [PATCH 50/63] Allow IBM Clang 9 GCC 8 to fail --- .gitlab/lassen-jobs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index 564efbb4..b511be1c 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -41,6 +41,7 @@ ibm_clang_9_cuda: ibm_clang_9_gcc_8_cuda: variables: SPEC: "+cuda %clang@9.0.0ibm +allow-untested-versions cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" + allow_failure: true extends: .build_and_test_on_lassen gcc_8_3_1_cuda: From c80f8575f435d86838cab5ba621822701fb42c2d Mon Sep 17 00:00:00 2001 From: Sergey Klevtsov Date: Tue, 13 Jul 2021 23:34:21 -0700 Subject: [PATCH 51/63] Add CMP0025 policy to CMakeLists.txt --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 698a864d..902c3199 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ ############################################################################## cmake_policy(SET CMP0057 NEW) cmake_policy(SET CMP0048 NEW) +cmake_policy(SET CMP0025 NEW) project(Chai LANGUAGES CXX VERSION 2.3.0) From df3e8e0f1b4350622bb9a68cfc7a23edc16e4bb9 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Wed, 14 Jul 2021 16:19:09 -0700 Subject: [PATCH 52/63] Fix bug when using partial space list --- src/chai/ManagedArray.inl | 4 ++++ tests/integration/managed_array_tests.cpp | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/chai/ManagedArray.inl b/src/chai/ManagedArray.inl index a8048c0b..55b934fd 100644 --- a/src/chai/ManagedArray.inl +++ b/src/chai/ManagedArray.inl @@ -39,6 +39,10 @@ CHAI_HOST_DEVICE ManagedArray::ManagedArray( #if !defined(CHAI_DEVICE_COMPILE) m_pointer_record = new PointerRecord(); int i = 0; + for (int s = CPU; s < NUM_EXECUTION_SPACES; ++s) { + m_pointer_record->m_allocators[s] = m_resource_manager->getAllocatorId(ExecutionSpace(s)); + } + for (const auto& space : spaces) { m_pointer_record->m_allocators[space] = allocators.begin()[i++].getId(); } diff --git a/tests/integration/managed_array_tests.cpp b/tests/integration/managed_array_tests.cpp index 510267e6..53de5795 100644 --- a/tests/integration/managed_array_tests.cpp +++ b/tests/integration/managed_array_tests.cpp @@ -29,6 +29,8 @@ #include "chai/ManagedArray.hpp" +#include "umpire/ResourceManager.hpp" + struct my_point { double x; @@ -1973,6 +1975,23 @@ GPU_TEST(ManagedArray, NoAllocationGPU) array.free(); } +GPU_TEST(ManagedArray, NoAllocationGPUList) +{ + auto& rm = umpire::ResourceManager::getInstance(); + chai::ManagedArray array(10, + std::initializer_list{chai::CPU}, + std::initializer_list{rm.getAllocator("HOST")} + ); + + forall(gpu(), 0, 10, [=] __device__ (int i) { + array[i] = i; + }); + + forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); +} + GPU_TEST(ManagedArray, NoAllocationNullGPU) { chai::ManagedArray array; From 108bd228a3fdf666a26d69128c021ecfb2cb27e2 Mon Sep 17 00:00:00 2001 From: Josh Essman Date: Mon, 19 Jul 2021 10:18:41 -0500 Subject: [PATCH 53/63] feat: add begin/end iterators to ManagedArray --- src/chai/ManagedArray.hpp | 20 +++++++++++++ src/chai/ManagedArray.inl | 12 ++++++++ src/chai/ManagedArray_thin.inl | 12 ++++++++ tests/integration/managed_array_tests.cpp | 34 +++++++++++++++++++++++ 4 files changed, 78 insertions(+) diff --git a/src/chai/ManagedArray.hpp b/src/chai/ManagedArray.hpp index 2a9057fe..e8c10ca3 100644 --- a/src/chai/ManagedArray.hpp +++ b/src/chai/ManagedArray.hpp @@ -240,6 +240,26 @@ class ManagedArray : public CHAICopyable */ CHAI_HOST T* getPointer(ExecutionSpace space, bool do_move = true) const; + /*! + * \brief Move data to the current execution space (actually determined + * by where the code is executing) and return an iterator to the + * beginning of the array. + * + * \return Iterator (as raw pointer) to the start of the array in the + * current execution space + */ + CHAI_HOST_DEVICE T* begin() const; + + /*! + * \brief Move data to the current execution space (actually determined + * by where the code is executing) and return an iterator to + * one past the end of the array. + * + * \return Iterator (as raw pointer) to the element after the last element + * of the array in the current execution space + */ + CHAI_HOST_DEVICE T* end() const; + /*! * \brief * diff --git a/src/chai/ManagedArray.inl b/src/chai/ManagedArray.inl index a8048c0b..4cb10a09 100644 --- a/src/chai/ManagedArray.inl +++ b/src/chai/ManagedArray.inl @@ -548,6 +548,18 @@ T* ManagedArray::getPointer(ExecutionSpace space, bool do_move) const { return data(space, do_move); } +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::begin() const { + return data(); +} + +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::end() const { + return data() + size(); +} + //template //ManagedArray::operator ManagedArray< // typename std::conditional::value, diff --git a/src/chai/ManagedArray_thin.inl b/src/chai/ManagedArray_thin.inl index 3f7c3ab7..586c7245 100644 --- a/src/chai/ManagedArray_thin.inl +++ b/src/chai/ManagedArray_thin.inl @@ -136,6 +136,18 @@ T* ManagedArray::getPointer(ExecutionSpace space, bool do_move) const return data(space, do_move); } +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::begin() const { + return data(); +} + +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::end() const { + return data() + size(); +} + template CHAI_INLINE CHAI_HOST void ManagedArray::allocate(size_t elems, diff --git a/tests/integration/managed_array_tests.cpp b/tests/integration/managed_array_tests.cpp index 510267e6..a0f64457 100644 --- a/tests/integration/managed_array_tests.cpp +++ b/tests/integration/managed_array_tests.cpp @@ -1130,6 +1130,40 @@ GPU_TEST(ManagedArray, cdataGPU) #endif #endif +TEST(ManagedArray, Iterators) +{ + int length = 10; + chai::ManagedArray array(length); + + forall(sequential(), 0, length, [=] (int i) { + array[i] = i; + }); + + // Make sure the iterator distance is the size of the array + EXPECT_EQ(std::distance(array.begin(), array.end()), length); + + // Double each element with a range-based for loop + for (int& val : array) + { + val *= 2; + } + + // Double each element again with an + std::for_each(array.begin(), array.end(), [](int& val) { val *= 2; }); + + // Make sure a reference to a const array can be iterated over + const chai::ManagedArray& const_array = array; + int i = 0; + for (const int val : const_array) + { + EXPECT_EQ(val, i * 4); + i++; + } + + array.free(); + assert_empty_map(true); +} + TEST(ManagedArray, Reset) { chai::ManagedArray array(20); From ac641884c875d3d4f4bf6f0d245606ec43e4defc Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 3 Aug 2021 14:18:07 -0700 Subject: [PATCH 54/63] Switch to uberenv submodule --- .gitmodules | 3 + .../project.json => .uberenv_config.json | 5 +- .../chai/package.py | 0 .../raja/package.py | 0 .../umpire/camp_target_umpire_3.0.0.patch | 0 .../umpire/package.py | 0 scripts/uberenv | 1 + scripts/uberenv/LICENSE | 64 -- scripts/uberenv/Makefile | 6 - scripts/uberenv/README.md | 19 - scripts/uberenv/docs/sphinx/conf.py | 324 ------- scripts/uberenv/docs/sphinx/index.rst | 194 ----- scripts/uberenv/gen_spack_env_script.py | 128 --- scripts/uberenv/spack_configs | 1 - scripts/uberenv/uberenv.py | 800 ------------------ 15 files changed, 7 insertions(+), 1538 deletions(-) rename scripts/uberenv/project.json => .uberenv_config.json (70%) rename scripts/{uberenv/packages => spack_packages}/chai/package.py (100%) rename scripts/{uberenv/packages => spack_packages}/raja/package.py (100%) rename scripts/{uberenv/packages => spack_packages}/umpire/camp_target_umpire_3.0.0.patch (100%) rename scripts/{uberenv/packages => spack_packages}/umpire/package.py (100%) create mode 160000 scripts/uberenv delete mode 100644 scripts/uberenv/LICENSE delete mode 100644 scripts/uberenv/Makefile delete mode 100644 scripts/uberenv/README.md delete mode 100644 scripts/uberenv/docs/sphinx/conf.py delete mode 100644 scripts/uberenv/docs/sphinx/index.rst delete mode 100644 scripts/uberenv/gen_spack_env_script.py delete mode 120000 scripts/uberenv/spack_configs delete mode 100755 scripts/uberenv/uberenv.py diff --git a/.gitmodules b/.gitmodules index 3d736105..6d879449 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "scripts/radiuss-spack-configs"] path = scripts/radiuss-spack-configs url = https://github.com/LLNL/radiuss-spack-configs +[submodule "scripts/uberenv"] + path = scripts/uberenv + url = https://github.com/LLNL/uberenv.git diff --git a/scripts/uberenv/project.json b/.uberenv_config.json similarity index 70% rename from scripts/uberenv/project.json rename to .uberenv_config.json index 1335d596..1a0136d9 100644 --- a/scripts/uberenv/project.json +++ b/.uberenv_config.json @@ -6,5 +6,6 @@ "spack_url": "https://github.com/davidbeckingsale/spack", "spack_branch": "feature/allow-untested-cuda-versions", "spack_commit": "f96e256bee1948aa030916aae0c1b2645230fb9f", -"spack_activate" : {} -} +"spack_activate" : {}, +"spack_configs_path": "scripts/radiuss-spack-configs", +"spack_packages_path": "scripts/spack_packages"} diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/spack_packages/chai/package.py similarity index 100% rename from scripts/uberenv/packages/chai/package.py rename to scripts/spack_packages/chai/package.py diff --git a/scripts/uberenv/packages/raja/package.py b/scripts/spack_packages/raja/package.py similarity index 100% rename from scripts/uberenv/packages/raja/package.py rename to scripts/spack_packages/raja/package.py diff --git a/scripts/uberenv/packages/umpire/camp_target_umpire_3.0.0.patch b/scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch similarity index 100% rename from scripts/uberenv/packages/umpire/camp_target_umpire_3.0.0.patch rename to scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/spack_packages/umpire/package.py similarity index 100% rename from scripts/uberenv/packages/umpire/package.py rename to scripts/spack_packages/umpire/package.py diff --git a/scripts/uberenv b/scripts/uberenv new file mode 160000 index 00000000..105e384f --- /dev/null +++ b/scripts/uberenv @@ -0,0 +1 @@ +Subproject commit 105e384f585e2391c42b2def93124a6580319c1c diff --git a/scripts/uberenv/LICENSE b/scripts/uberenv/LICENSE deleted file mode 100644 index fcd00312..00000000 --- a/scripts/uberenv/LICENSE +++ /dev/null @@ -1,64 +0,0 @@ -Copyright (c) 2014-2018, Lawrence Livermore National Security, LLC. - -Produced at the Lawrence Livermore National Laboratory - -LLNL-CODE-666778 - -All rights reserved. - -This file is part of Conduit. - -For details, see: http://software.llnl.gov/conduit/. - -Please also read conduit/LICENSE - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the disclaimer below. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the disclaimer (as noted below) in the - documentation and/or other materials provided with the distribution. - -* Neither the name of the LLNS/LLNL nor the names of its contributors may - be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -Additional BSD Notice - - 1. This notice is required to be provided under our contract with the U.S. - Department of Energy (DOE). This work was produced at Lawrence - Livermore National Laboratory under Contract No. DE-AC52-07NA27344 with - the DOE. - - 2. Neither the United States Government nor Lawrence Livermore National - Security, LLC nor any of their employees, makes any warranty, express - or implied, or assumes any liability or responsibility for the - accuracy, completeness, or usefulness of any information, apparatus, - product, or process disclosed, or represents that its use would not - infringe privately-owned rights. - - 3. Also, reference herein to any specific commercial products, process, - or services by trade name, trademark, manufacturer or otherwise does - not necessarily constitute or imply its endorsement, recommendation, - or favoring by the United States Government or Lawrence Livermore - National Security, LLC. The views and opinions of authors expressed - herein do not necessarily state or reflect those of the United - States Government or Lawrence Livermore National Security, LLC, and - shall not be used for advertising or product endorsement purposes. - diff --git a/scripts/uberenv/Makefile b/scripts/uberenv/Makefile deleted file mode 100644 index 2760762d..00000000 --- a/scripts/uberenv/Makefile +++ /dev/null @@ -1,6 +0,0 @@ - -default: - sphinx-build -E -a -b html docs/sphinx/ _docs_html - -clean: - rm -rf _docs_html \ No newline at end of file diff --git a/scripts/uberenv/README.md b/scripts/uberenv/README.md deleted file mode 100644 index 82d68201..00000000 --- a/scripts/uberenv/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# uberenv -Automates using Spack (https://www.spack.io/) to build and deploy software. - -Uberenv is a short python script that helps automate using Spack to build -third-party dependencies for development and to deploy Spack packages. - -Uberenv was released as part of the Conduit (https://github.com/LLNL/conduit/). It is included in-source in several projects, this repo is used to hold the latest reference version. - -For more details, see Uberenv's documention: - -https://uberenv.readthedocs.io - -You can also find details about how it is used in Conduit's documentation: - -https://llnl-conduit.readthedocs.io/en/latest/building.html#building-conduit-and-third-party-dependencies - -Conduit's source repo also serves as an example for uberenv and spack configuration files, etc: - -https://github.com/LLNL/conduit/tree/master/scripts/uberenv diff --git a/scripts/uberenv/docs/sphinx/conf.py b/scripts/uberenv/docs/sphinx/conf.py deleted file mode 100644 index a8475c7b..00000000 --- a/scripts/uberenv/docs/sphinx/conf.py +++ /dev/null @@ -1,324 +0,0 @@ -# -*- coding: utf-8 -*- -# -############################################################################### -# Copyright (c) 2015-2019, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-666778 -# -# All rights reserved. -# -# This file is part of Conduit. -# -# For details, see: http://software.llnl.gov/conduit/. -# -# Please also read conduit/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### -# -# Uberenv documentation build configuration file, created by -# sphinx-quickstart on Thu Oct 16 11:23:46 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax' -] - -# try to add the breathe extension -try: - import breathe - extensions.append('breathe') -except: - pass - -# Add any paths that contain templates here, relative to this directory. -# templates_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Uberenv' -copyright = u'Copyright (c) 2015-2019, LLNS' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = 'current' -# The full version, including alpha/beta/rc tags. -release = 'current' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { 'logo_only' : True } - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'Uberenvdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'Uberenv.tex', u'Uberenv Documentation', - u'LLNS', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'uberenv', u'Uberenv Documentation', - [u'LLNS'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'Uberenv', u'Uberenv Documentation', - u'LLNS', 'Uberenv', 'Automates using spack to build and deploy software.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -# try to use the read the docs theme -try: - import sphinx_rtd_theme - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -except: - pass diff --git a/scripts/uberenv/docs/sphinx/index.rst b/scripts/uberenv/docs/sphinx/index.rst deleted file mode 100644 index 457ec596..00000000 --- a/scripts/uberenv/docs/sphinx/index.rst +++ /dev/null @@ -1,194 +0,0 @@ -.. ############################################################################ -.. # Copyright (c) 2014-2018, Lawrence Livermore National Security, LLC. -.. # -.. # Produced at the Lawrence Livermore National Laboratory -.. # -.. # LLNL-CODE-666778 -.. # -.. # All rights reserved. -.. # -.. # This file is part of Conduit. -.. # -.. # For details, see: http://software.llnl.gov/conduit/. -.. # -.. # Please also read conduit/LICENSE -.. # -.. # Redistribution and use in source and binary forms, with or without -.. # modification, are permitted provided that the following conditions are met: -.. # -.. # * Redistributions of source code must retain the above copyright notice, -.. # this list of conditions and the disclaimer below. -.. # -.. # * Redistributions in binary form must reproduce the above copyright notice, -.. # this list of conditions and the disclaimer (as noted below) in the -.. # documentation and/or other materials provided with the distribution. -.. # -.. # * Neither the name of the LLNS/LLNL nor the names of its contributors may -.. # be used to endorse or promote products derived from this software without -.. # specific prior written permission. -.. # -.. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -.. # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.. # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -.. # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -.. # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.. # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.. # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.. # POSSIBILITY OF SUCH DAMAGE. -.. # -.. ############################################################################ - -.. _building_with_uberenv: - -Uberenv -~~~~~~~~~~~~~~~ - -**Uberenv** automates using `Spack `_ to build and deploy software. - -Many projects leverage `Spack `_ to help build the software dependencies needed to develop and deploy their projects on HPC systems. Uberenv is a python script that helps automate using Spack to build -third-party dependencies for development and to deploy Spack packages. - -Uberenv was released as part of Conduit (https://github.com/LLNL/conduit/). It is included in-source in several projects. The -https://github.com/llnl/uberenv/ repo is used to hold the latest reference version of Uberenv. - - -uberenv.py -~~~~~~~~~~~~~~~~~~~~~ - -``uberenv.py`` is a single file python script that automates fetching Spack, building and installing third party dependencies, and can optionally install packages as well. To automate the full install process, ``uberenv.py`` uses a target Spack package along with extra settings such as Spack compiler and external third party package details for common HPC platforms. - -``uberenv.py`` is included directly in a project's source code repo in the folder: ``scripts/uberenv/`` -This folder is also used to store extra Spack and Uberenv configuration files unique to the target project. ``uberenv.py`` uses a ``project.json`` file to specify project details, including the target Spack package name and which Spack repo is used. Conduit's source repo serves as an example for Uberenv and Spack configuration files, etc: - -https://github.com/LLNL/conduit/tree/master/scripts/uberenv - - -``uberenv.py`` is developed by LLNL in support of the `Ascent `_, Axom, and `Conduit `_ projects. - - -Command Line Options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Build configuration -------------------- - -``uberenv.py`` has a few options that allow you to control how dependencies are built: - - ======================= ============================================== ================================================ - Option Description Default - ======================= ============================================== ================================================ - ``--prefix`` Destination directory ``uberenv_libs`` - ``--spec`` Spack spec linux: **%gcc** - osx: **%clang** - ``--spack-config-dir`` Folder with Spack settings files linux: (empty) - osx: ``scripts/uberenv/spack_configs/darwin/`` - ``-k`` Ignore SSL Errors **False** - ``--install`` Fully install target, not just dependencies **False** - ``--run_tests`` Invoke tests during build and against install **False** - ``--project-json`` File for project specific settings ``project.json`` - ======================= ============================================== ================================================ - -The ``-k`` option exists for sites where SSL certificate interception undermines fetching -from github and https hosted source tarballs. When enabled, ``uberenv.py`` clones Spack using: - -.. code:: bash - - git -c http.sslVerify=false clone https://github.com/llnl/spack.git - -And passes ``-k`` to any Spack commands that may fetch via https. - - -Default invocation on Linux: - -.. code:: bash - - python scripts/uberenv/uberenv.py --prefix uberenv_libs \ - --spec %gcc - -Default invocation on OSX: - -.. code:: bash - - python scripts/uberenv/uberenv.py --prefix uberenv_libs \ - --spec %clang \ - --spack-config-dir scripts/uberenv/spack_configs/darwin/ - - -Use the ``--install`` option to install the target package (not just its development dependencies): - -.. code:: bash - - python scripts/uberenv/uberenv.py --install - - -If the target Spack package supports Spack's testing hooks, you can run tests during the build process to validate the build and install, using the ``--run_tests`` option: - -.. code:: bash - - python scripts/uberenv/uberenv.py --install \ - --run_tests - -For details on Spack's spec syntax, see the `Spack Specs & dependencies `_ documentation. - - -Uberenv looks for configuration yaml files under ``scripts/uberenv/spack_config/{platform}`` or you can use the **--spack-config-dir** option to specify a directory with compiler and packages yaml files to use with Spack. See the `Spack Compiler Configuration `_ -and `Spack System Packages -`_ -documentation for details. - -.. note:: - The bootstrapping process ignores ``~/.spack/compilers.yaml`` to avoid conflicts - and surprises from a user's specific Spack settings on HPC platforms. - -When run, ``uberenv.py`` checkouts a specific version of Spack from github as ``spack`` in the -destination directory. It then uses Spack to build and install the target packages' dependencies into -``spack/opt/spack/``. Finally, the target package generates a host-config file ``{hostname}.cmake``, which is -copied to destination directory. This file specifies the compiler settings and paths to all of the dependencies. - - -Project configuration ---------------------- - -Part of the configuration can also be addressed using a json file. By default, it is named ``project.json`` and some settings can be overridden on command line: - - ==================== ========================== ================================================ ======================================= - Setting Option Description Default - ==================== ========================== ================================================ ======================================= - package_name ``--package-name`` Spack package name **None** - package_version **None** Spack package version **None** - package_final_phase ``--package-final-phase`` Controls after which phase Spack should stop **None** - package_source_dir ``--package-source-dir`` Controls the source directory Spack should use **None** - spack_url **None** Url where to download Spack ``https://github.com/spack/spack.git`` - spack_commit **None** Spack commit to checkout **None** - spack_activate **None** Spack packages to activate **None** - ==================== ========================== ================================================ ======================================= - - -Optimization ------------- - -``uberenv.py`` also features options to optimize the installation - - ==================== ============================================== ================================================ - Option Description Default - ==================== ============================================== ================================================ - ``--mirror`` Location of a Spack mirror **None** - ``--create-mirror`` Creates a Spack mirror at specified location **None** - ``--upstream`` Location of a Spack upstream **None** - ==================== ============================================== ================================================ - - -Project Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A few notes on using ``uberenv.py`` in a new project: - -* For an example of how to craft a ``project.json`` file a target project, see: `Conduit's project.json file `_ - -* ``uberenv.py`` hot copies ``packages`` to the cloned Spack install, this allows you to easily version control any Spack package overrides necessary - - diff --git a/scripts/uberenv/gen_spack_env_script.py b/scripts/uberenv/gen_spack_env_script.py deleted file mode 100644 index a1e6ba5d..00000000 --- a/scripts/uberenv/gen_spack_env_script.py +++ /dev/null @@ -1,128 +0,0 @@ -############################################################################### -# Copyright (c) 2015-2019, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-716457 -# -# All rights reserved. -# -# This file is part of Ascent. -# -# For details, see: http://ascent.readthedocs.io/. -# -# Please also read ascent/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### -import os -import sys -import subprocess - -from os.path import join as pjoin - -# if you have bad luck with spack load, this -# script is for you! -# -# Looks for subdir: spack or uberenv_libs/spack -# queries spack for given package names and -# creates a bash script that adds those to your path -# -# -# usage: -# python gen_spack_env_script.py [spack_pkg_1 spack_pkg_2 ...] -# - -def sexe(cmd,ret_output=False,echo = True): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {}]".format(cmd)) - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - res = p.communicate()[0] - res = res.decode('utf8') - return p.returncode,res - else: - return subprocess.call(cmd,shell=True) - - -def spack_exe(spath=None): - if spath is None: - to_try = [pjoin("uberenv_libs","spack"), "spack"] - for p in to_try: - abs_p = os.path.abspath(p) - print("[looking for spack directory at: {}]".format(abs_p)) - if os.path.isdir(abs_p): - print("[FOUND spack directory at: {}]".format(abs_p)) - return os.path.abspath(pjoin(abs_p,"bin","spack")) - print("[ERROR: failed to find spack directory!]") - sys.exit(-1) - else: - spack_exe = os.path.abspath(spath,"bin","spack") - if not os.path.isfile(spack_exec): - print("[ERROR: failed to find spack directory at spath={}]").format(spath) - sys.exit(-1) - return spack_exe - -def find_pkg(pkg_name): - r,rout = sexe(spack_exe() + " find -p " + pkg_name,ret_output = True) - print(rout) - for l in rout.split("\n"): - print(l) - lstrip = l.strip() - if not lstrip == "" and \ - not lstrip.startswith("==>") and \ - not lstrip.startswith("--"): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - -def path_cmd(pkg): - return('export PATH={}:$PATH\n'.format((pjoin(pkg["path"],"bin")))) - -def write_env_script(pkgs): - ofile = open("s_env.sh","w") - for p in pkgs: - print("[found {} at {}]".format(p["name"],p["path"])) - ofile.write("# {}\n".format(p["name"])) - ofile.write(path_cmd(p)) - print("[created {}]".format(os.path.abspath("s_env.sh"))) - -def main(): - pkgs = [find_pkg(pkg) for pkg in sys.argv[1:]] - if len(pkgs) > 0: - write_env_script(pkgs) - else: - print("usage: python gen_spack_env_script.py spack_pkg_1 spack_pkg_2 ...") - -if __name__ == "__main__": - main() diff --git a/scripts/uberenv/spack_configs b/scripts/uberenv/spack_configs deleted file mode 120000 index 17d3bf7a..00000000 --- a/scripts/uberenv/spack_configs +++ /dev/null @@ -1 +0,0 @@ -../radiuss-spack-configs \ No newline at end of file diff --git a/scripts/uberenv/uberenv.py b/scripts/uberenv/uberenv.py deleted file mode 100755 index 7761be2e..00000000 --- a/scripts/uberenv/uberenv.py +++ /dev/null @@ -1,800 +0,0 @@ -#!/bin/sh -"exec" "python" "-u" "-B" "$0" "$@" -############################################################################### -# Copyright (c) 2014-2020, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-666778 -# -# All rights reserved. -# -# This file is part of Conduit. -# -# For details, see https://lc.llnl.gov/conduit/. -# -# Please also read conduit/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### - -""" - file: uberenv.py - - description: automates using spack to install a project. - -""" - -import os -import sys -import subprocess -import shutil -import socket -import platform -import json -import datetime -import glob -import re - -from optparse import OptionParser - -from os import environ as env -from os.path import join as pjoin - - -def sexe(cmd,ret_output=False,echo=False): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {}]".format(cmd)) - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - out = p.communicate()[0] - out = out.decode('utf8') - return p.returncode,out - else: - return subprocess.call(cmd,shell=True) - - -def parse_args(): - "Parses args from command line" - parser = OptionParser() - parser.add_option("--install", - action="store_true", - dest="install", - default=False, - help="Install `package_name`, not just its dependencies.") - - # where to install - parser.add_option("--prefix", - dest="prefix", - default="uberenv_libs", - help="destination directory") - - # what compiler to use - parser.add_option("--spec", - dest="spec", - default=None, - help="spack compiler spec") - - # optional location of spack mirror - parser.add_option("--mirror", - dest="mirror", - default=None, - help="spack mirror directory") - - # flag to create mirror - parser.add_option("--create-mirror", - action="store_true", - dest="create_mirror", - default=False, - help="Create spack mirror") - - # optional location of spack upstream - parser.add_option("--upstream", - dest="upstream", - default=None, - help="add an external spack instance as upstream") - - # this option allows a user to explicitly to select a - # group of spack settings files (compilers.yaml , packages.yaml) - parser.add_option("--spack-config-dir", - dest="spack_config_dir", - default=None, - help="dir with spack settings files (compilers.yaml, packages.yaml, etc)") - - # overrides package_name - parser.add_option("--package-name", - dest="package_name", - default=None, - help="override the default package name") - - # controls after which package phase spack should stop - parser.add_option("--package-final-phase", - dest="package_final_phase", - default=None, - help="override the default phase after which spack should stop") - - # controls source_dir spack should use to build the package - parser.add_option("--package-source-dir", - dest="package_source_dir", - default=None, - help="override the default source dir spack should use") - - # a file that holds settings for a specific project - # using uberenv.py - parser.add_option("--project-json", - dest="project_json", - default=pjoin(uberenv_script_dir(),"project.json"), - help="uberenv project settings json file") - - # flag to use insecure curl + git - parser.add_option("-k", - action="store_true", - dest="ignore_ssl_errors", - default=False, - help="Ignore SSL Errors") - - # option to force a spack pull - parser.add_option("--pull", - action="store_true", - dest="spack_pull", - default=False, - help="Pull if spack repo already exists") - - # option to force for clean of packages specified to - # be cleaned in the project.json - parser.add_option("--clean", - action="store_true", - dest="spack_clean", - default=False, - help="Force uninstall of packages specified in project.json") - - # option to tell spack to run tests - parser.add_option("--run_tests", - action="store_true", - dest="run_tests", - default=False, - help="Invoke build tests during spack install") - - # option to init osx sdk env flags - parser.add_option("--macos-sdk-env-setup", - action="store_true", - dest="macos_sdk_env_setup", - default=False, - help="Set several env vars to select OSX SDK settings." - "This was necessary for older versions of macOS " - " but can cause issues with macOS versions >= 10.13. " - " so it is disabled by default.") - - - ############### - # parse args - ############### - opts, extras = parser.parse_args() - # we want a dict b/c the values could - # be passed without using optparse - opts = vars(opts) - if not opts["spack_config_dir"] is None: - opts["spack_config_dir"] = os.path.abspath(opts["spack_config_dir"]) - if not os.path.isdir(opts["spack_config_dir"]): - print("[ERROR: invalid spack config dir: {} ]".format(opts["spack_config_dir"])) - sys.exit(-1) - # if rel path is given for the mirror, we need to evaluate here -- before any - # chdirs to avoid confusion related to what it is relative to. - # (it should be relative to where uberenv is run from, so it matches what you expect - # from shell completion, etc) - if not opts["mirror"] is None: - if not opts["mirror"].startswith("http") and not os.path.isabs(opts["mirror"]): - opts["mirror"] = os.path.abspath(opts["mirror"]) - return opts, extras - - -def uberenv_script_dir(): - # returns the directory of the uberenv.py script - return os.path.dirname(os.path.abspath(__file__)) - -def load_json_file(json_file): - # reads json file - return json.load(open(json_file)) - -def is_darwin(): - return "darwin" in platform.system().lower() - -def is_windows(): - return "windows" in platform.system().lower() - -class UberEnv(): - """ Base class for package manager """ - - def __init__(self, opts, extra_opts): - self.opts = opts - self.extra_opts = extra_opts - - # load project settings - self.project_opts = load_json_file(opts["project_json"]) - print("[uberenv project settings: {}]".format(str(self.project_opts))) - print("[uberenv options: {}]".format(str(self.opts))) - - def setup_paths_and_dirs(self): - self.uberenv_path = os.path.dirname(os.path.realpath(__file__)) - - def set_from_args_or_json(self,setting): - try: - setting_value = self.project_opts[setting] - except (KeyError): - print("ERROR: {} must at least be defined in project.json".format(setting)) - raise - else: - if self.opts[setting]: - setting_value = self.opts[setting] - return setting_value - - def set_from_json(self,setting): - try: - setting_value = self.project_opts[setting] - except (KeyError): - print("ERROR: {} must at least be defined in project.json".format(setting)) - raise - return setting_value - - def detect_platform(self): - # find supported sets of compilers.yaml, packages,yaml - res = None - if is_darwin(): - res = "darwin" - elif "SYS_TYPE" in os.environ.keys(): - sys_type = os.environ["SYS_TYPE"].lower() - res = sys_type - return res - - -class SpackEnv(UberEnv): - """ Helper to clone spack and install libraries on MacOS an Linux """ - - def __init__(self, opts, extra_opts): - UberEnv.__init__(self,opts,extra_opts) - - self.pkg_name = self.set_from_args_or_json("package_name") - self.pkg_version = self.set_from_json("package_version") - self.pkg_final_phase = self.set_from_args_or_json("package_final_phase") - self.pkg_src_dir = self.set_from_args_or_json("package_source_dir") - - self.spec_hash = "" - self.use_install = False - - # Some additional setup for macos - if is_darwin(): - if opts["macos_sdk_env_setup"]: - # setup osx deployment target and sdk settings - setup_osx_sdk_env_vars() - else: - print("[skipping MACOSX env var setup]") - - # setup default spec - if opts["spec"] is None: - if is_darwin(): - opts["spec"] = "%clang" - else: - opts["spec"] = "%gcc" - self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) - elif not opts["spec"].startswith("@"): - self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) - else: - self.opts["spec"] = "{}".format(opts["spec"]) - - print("[spack spec: {}]".format(self.opts["spec"])) - - def setup_paths_and_dirs(self): - # get the current working path, and the glob used to identify the - # package files we want to hot-copy to spack - - UberEnv.setup_paths_and_dirs(self) - - self.pkgs = pjoin(self.uberenv_path, "packages","*") - - # setup destination paths - self.dest_dir = os.path.abspath(self.opts["prefix"]) - self.dest_spack = pjoin(self.dest_dir,"spack") - print("[installing to: {0}]".format(self.dest_dir)) - - # print a warning if the dest path already exists - if not os.path.isdir(self.dest_dir): - os.mkdir(self.dest_dir) - else: - print("[info: destination '{}' already exists]".format(self.dest_dir)) - - if os.path.isdir(self.dest_spack): - print("[info: destination '{}' already exists]".format(self.dest_spack)) - - self.pkg_src_dir = os.path.join(self.uberenv_path,self.pkg_src_dir) - if not os.path.isdir(self.pkg_src_dir): - print("[ERROR: package_source_dir '{}' does not exist]".format(self.pkg_src_dir)) - sys.exit(-1) - - - def find_spack_pkg_path_from_hash(self, pkg_name, pkg_hash): - res, out = sexe("spack/bin/spack find -p /{}".format(pkg_hash), ret_output = True) - for l in out.split("\n"): - if l.startswith(pkg_name): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - - def find_spack_pkg_path(self, pkg_name, spec = ""): - res, out = sexe("spack/bin/spack find -p " + pkg_name + spec,ret_output = True) - for l in out.split("\n"): - # TODO: at least print a warning when several choices exist. This will - # pick the first in the list. - if l.startswith(pkg_name): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - - # Extract the first line of the full spec - def read_spack_full_spec(self,pkg_name,spec): - res, out = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True) - for l in out.split("\n"): - if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0: - return l.strip() - - def clone_repo(self): - if not os.path.isdir(self.dest_spack): - - # compose clone command for the dest path, spack url and branch - print("[info: cloning spack develop branch from github]") - - os.chdir(self.dest_dir) - - clone_opts = ("-c http.sslVerify=false " - if self.opts["ignore_ssl_errors"] else "") - - spack_url = self.project_opts.get("spack_url", "https://github.com/spack/spack.git") - spack_branch = self.project_opts.get("spack_branch", "develop") - - clone_cmd = "git {0} clone --single-branch --depth=1 -b {1} {2}".format(clone_opts, spack_branch,spack_url) - sexe(clone_cmd, echo=True) - - if "spack_commit" in self.project_opts: - # optionally, check out a specific commit - os.chdir(pjoin(self.dest_dir,"spack")) - sha1 = self.project_opts["spack_commit"] - res, current_sha1 = sexe("git log -1 --pretty=%H", ret_output=True) - if sha1 != current_sha1: - print("[info: using spack commit {}]".format(sha1)) - sexe("git stash", echo=True) - sexe("git fetch --depth=1 origin {0}".format(sha1),echo=True) - sexe("git checkout {0}".format(sha1),echo=True) - - if self.opts["spack_pull"]: - # do a pull to make sure we have the latest - os.chdir(pjoin(self.dest_dir,"spack")) - sexe("git stash", echo=True) - sexe("git pull", echo=True) - - def config_dir(self): - """ path to compilers.yaml, which we will use for spack's compiler setup""" - spack_config_dir = self.opts["spack_config_dir"] - if spack_config_dir is None: - uberenv_plat = self.detect_platform() - if not uberenv_plat is None: - spack_config_dir = os.path.abspath(pjoin(self.uberenv_path,"spack_configs",uberenv_plat)) - return spack_config_dir - - - def disable_spack_config_scopes(self,spack_dir): - # disables all config scopes except "defaults", which we will - # force our settings into - spack_lib_config = pjoin(spack_dir,"lib","spack","spack","config.py") - print("[disabling config scope (except defaults) in: {}]".format(spack_lib_config)) - cfg_script = open(spack_lib_config).read() - for cfg_scope_stmt in ["('system', os.path.join(spack.paths.system_etc_path, 'spack')),", - "('site', os.path.join(spack.paths.etc_path, 'spack')),", - "('user', spack.paths.user_config_path)"]: - cfg_script = cfg_script.replace(cfg_scope_stmt, - "#DISABLED BY UBERENV: " + cfg_scope_stmt) - open(spack_lib_config,"w").write(cfg_script) - - - def patch(self): - - cfg_dir = self.config_dir() - spack_dir = self.dest_spack - - # force spack to use only "defaults" config scope - self.disable_spack_config_scopes(spack_dir) - spack_etc_defaults_dir = pjoin(spack_dir,"etc","spack","defaults") - - # copy in "defaults" config.yaml - config_yaml = os.path.abspath(pjoin(self.uberenv_path,"spack_configs","config.yaml")) - sexe("cp {} {}/".format(config_yaml, spack_etc_defaults_dir ), echo=True) - - # copy in other settings per platform - if not cfg_dir is None: - print("[copying uberenv compiler and packages settings from {0}]".format(cfg_dir)) - - config_yaml = pjoin(cfg_dir,"config.yaml") - compilers_yaml = pjoin(cfg_dir,"compilers.yaml") - packages_yaml = pjoin(cfg_dir,"packages.yaml") - - if os.path.isfile(config_yaml): - sexe("cp {} {}/".format(config_yaml , spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(compilers_yaml): - sexe("cp {} {}/".format(compilers_yaml, spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(packages_yaml): - sexe("cp {} {}/".format(packages_yaml, spack_etc_defaults_dir ), echo=True) - else: - # let spack try to auto find compilers - sexe("spack/bin/spack compiler find", echo=True) - - # hot-copy our packages into spack - if self.pkgs: - dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages") - print("[copying patched packages from {0}]".format(self.pkgs)) - sexe("cp -Rf {} {}".format(self.pkgs,dest_spack_pkgs)) - - - def clean_build(self): - # clean out any temporary spack build stages - cln_cmd = "spack/bin/spack clean " - res = sexe(cln_cmd, echo=True) - - # clean out any spack cached stuff - cln_cmd = "spack/bin/spack clean --all" - res = sexe(cln_cmd, echo=True) - - # check if we need to force uninstall of selected packages - if self.opts["spack_clean"]: - if self.project_opts.has_key("spack_clean_packages"): - for cln_pkg in self.project_opts["spack_clean_packages"]: - if not self.find_spack_pkg_path(cln_pkg) is None: - unist_cmd = "spack/bin/spack uninstall -f -y --all --dependents " + cln_pkg - res = sexe(unist_cmd, echo=True) - - def show_info(self): - # prints install status and 32 characters hash - options="--install-status --very-long" - spec_cmd = "spack/bin/spack spec {0} {1}{2}".format(options,self.pkg_name,self.opts["spec"]) - - res, out = sexe(spec_cmd, ret_output=True, echo=True) - print(out) - - #Check if spec is already installed - for line in out.split("\n"): - # Example of matching line: ("status" "hash" "package"...) - # [+] hf3cubkgl74ryc3qwen73kl4yfh2ijgd serac@develop%clang@10.0.0-apple~debug~devtools~glvis arch=darwin-mojave-x86_64 - if re.match(r"^(\[\+\]| - ) [a-z0-9]{32} " + re.escape(self.pkg_name), line): - self.spec_hash = line.split(" ")[1] - # if spec already installed - if line.startswith("[+]"): - pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name,self.spec_hash) - install_path = pkg_path["path"] - # testing that the path exists is mandatory until Spack team fixes - # https://github.com/spack/spack/issues/16329 - if os.path.isdir(install_path): - print("[Warning: {} {} has already been installed in {}]".format(self.pkg_name, self.opts["spec"],install_path)) - print("[Warning: Uberenv will proceed using this directory]".format(self.pkg_name)) - self.use_install = True - - return res - - def install(self): - # use the uberenv package to trigger the right builds - # and build an host-config.cmake file - - if not self.use_install: - install_cmd = "spack/bin/spack " - if self.opts["ignore_ssl_errors"]: - install_cmd += "-k " - if not self.opts["install"]: - install_cmd += "dev-build --quiet -d {} -u {} ".format(self.pkg_src_dir,self.pkg_final_phase) - else: - install_cmd += "install " - if self.opts["run_tests"]: - install_cmd += "--test=root " - install_cmd += self.pkg_name + self.opts["spec"] - res = sexe(install_cmd, echo=True) - - if res != 0: - print("[ERROR: failure of spack install/dev-build]") - return res - - full_spec = self.read_spack_full_spec(self.pkg_name,self.opts["spec"]) - if "spack_activate" in self.project_opts: - print("[activating dependent packages]") - # get the full spack spec for our project - pkg_names = self.project_opts["spack_activate"].keys() - for pkg_name in pkg_names: - pkg_spec_requirements = self.project_opts["spack_activate"][pkg_name] - activate=True - for req in pkg_spec_requirements: - if req not in full_spec: - activate=False - break - if activate: - activate_cmd = "spack/bin/spack activate " + pkg_name - sexe(activate_cmd, echo=True) - # note: this assumes package extends python when +python - # this may fail general cases - if self.opts["install"] and "+python" in full_spec: - activate_cmd = "spack/bin/spack activate /" + self.spec_hash - sexe(activate_cmd, echo=True) - # if user opt'd for an install, we want to symlink the final - # install to an easy place: - if self.opts["install"] or self.use_install: - pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name, self.spec_hash) - if self.pkg_name != pkg_path["name"]: - print("[ERROR: Could not find install of {}]".format(self.pkg_name)) - return -1 - else: - # Symlink host-config file - hc_glob = glob.glob(pjoin(pkg_path["path"],"*.cmake")) - if len(hc_glob) > 0: - hc_path = hc_glob[0] - hc_fname = os.path.split(hc_path)[1] - if os.path.islink(hc_fname): - os.unlink(hc_fname) - elif os.path.isfile(hc_fname): - sexe("rm -f {}".format(hc_fname)) - print("[symlinking host config file to {}]".format(pjoin(self.dest_dir,hc_fname))) - os.symlink(hc_path,hc_fname) - - # Symlink install directory - if self.opts["install"]: - pkg_lnk_dir = "{}-install".format(self.pkg_name) - if os.path.islink(pkg_lnk_dir): - os.unlink(pkg_lnk_dir) - print("") - print("[symlinking install to {}]".format(pjoin(self.dest_dir,pkg_lnk_dir))) - os.symlink(pkg_path["path"],os.path.abspath(pkg_lnk_dir)) - print("") - print("[install complete!]") - # otherwise we are in the "only dependencies" case and the host-config - # file has to be copied from the do-be-deleted spack-build dir. - else: - pattern = "*{}.cmake".format(self.pkg_name) - build_dir = pjoin(self.pkg_src_dir,"spack-build") - hc_glob = glob.glob(pjoin(build_dir,pattern)) - if len(hc_glob) > 0: - hc_path = hc_glob[0] - hc_fname = os.path.split(hc_path)[1] - if os.path.islink(hc_fname): - os.unlink(hc_fname) - print("[copying host config file to {}]".format(pjoin(self.dest_dir,hc_fname))) - sexe("cp {} {}".format(hc_path,hc_fname)) - print("[removing project build directory {}]".format(pjoin(build_dir))) - sexe("rm -rf {}".format(build_dir)) - - def get_mirror_path(self): - mirror_path = self.opts["mirror"] - if not mirror_path: - print("[--create-mirror requires a mirror directory]") - sys.exit(-1) - return mirror_path - - def create_mirror(self): - """ - Creates a spack mirror for pkg_name at mirror_path. - """ - - mirror_path = self.get_mirror_path() - - mirror_cmd = "spack/bin/spack " - if self.opts["ignore_ssl_errors"]: - mirror_cmd += "-k " - mirror_cmd += "mirror create -d {} --dependencies {}{}".format(mirror_path, - self.pkg_name, - self.opts["spec"]) - return sexe(mirror_cmd, echo=True) - - def find_spack_mirror(self, mirror_name): - """ - Returns the path of a defaults scoped spack mirror with the - given name, or None if no mirror exists. - """ - res, out = sexe("spack/bin/spack mirror list", ret_output=True) - mirror_path = None - for mirror in out.split('\n'): - if mirror: - parts = mirror.split() - if parts[0] == mirror_name: - mirror_path = parts[1] - return mirror_path - - def use_mirror(self): - """ - Configures spack to use mirror at a given path. - """ - mirror_name = self.pkg_name - mirror_path = self.get_mirror_path() - existing_mirror_path = self.find_spack_mirror(mirror_name) - - if existing_mirror_path and mirror_path != existing_mirror_path: - # Existing mirror has different URL, error out - print("[removing existing spack mirror `{}` @ {}]".format(mirror_name, - existing_mirror_path)) - # - # Note: In this case, spack says it removes the mirror, but we still - # get errors when we try to add a new one, sounds like a bug - # - sexe("spack/bin/spack mirror remove --scope=defaults {} ".format(mirror_name), - echo=True) - existing_mirror_path = None - if not existing_mirror_path: - # Add if not already there - sexe("spack/bin/spack mirror add --scope=defaults {} {}".format( - mirror_name, mirror_path), echo=True) - print("[using mirror {}]".format(mirror_path)) - - def find_spack_upstream(self, upstream_name): - """ - Returns the path of a defaults scoped spack upstream with the - given name, or None if no upstream exists. - """ - upstream_path = None - - res, out = sexe('spack/bin/spack config get upstreams', ret_output=True) - if (not out) and ("upstreams:" in out): - out = out.replace(' ', '') - out = out.replace('install_tree:', '') - out = out.replace(':', '') - out = out.splitlines() - out = out[1:] - upstreams = dict(zip(out[::2], out[1::2])) - - for name in upstreams.keys(): - if name == upstream_name: - upstream_path = upstreams[name] - - return upstream_path - - def use_spack_upstream(self): - """ - Configures spack to use upstream at a given path. - """ - upstream_path = self.opts["upstream"] - if not upstream_path: - print("[--create-upstream requires a upstream directory]") - sys.exit(-1) - upstream_path = os.path.abspath(upstream_path) - upstream_name = self.pkg_name - existing_upstream_path = self.find_spack_upstream(upstream_name) - if (not existing_upstream_path) or (upstream_path != os.path.abspath(existing_upstream_path)): - # Existing upstream has different URL, error out - print("[removing existing spack upstream configuration file]") - sexe("rm spack/etc/spack/defaults/upstreams.yaml") - with open('spack/etc/spack/defaults/upstreams.yaml','w+') as upstreams_cfg_file: - upstreams_cfg_file.write("upstreams:\n") - upstreams_cfg_file.write(" {}:\n".format(upstream_name)) - upstreams_cfg_file.write(" install_tree: {}\n".format(upstream_path)) - - -def find_osx_sdks(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - res = {} - sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") - for sdk in sdks: - sdk_base = os.path.split(sdk)[1] - ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] - res[ver] = sdk - return res - -def setup_osx_sdk_env_vars(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - # find current osx version (10.11.6) - dep_tgt = platform.mac_ver()[0] - # sdk file names use short version (ex: 10.11) - dep_tgt_short = dep_tgt[:dep_tgt.rfind(".")] - # find installed sdks, ideally we want the sdk that matches the current os - sdk_root = None - sdks = find_osx_sdks() - if dep_tgt_short in sdks.keys(): - # matches our osx, use this one - sdk_root = sdks[dep_tgt_short] - elif len(sdks) > 0: - # for now, choose first one: - dep_tgt = sdks.keys()[0] - sdk_root = sdks[dep_tgt] - else: - # no valid sdks, error out - print("[ERROR: Could not find OSX SDK @ /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/]") - sys.exit(-1) - - env["MACOSX_DEPLOYMENT_TARGET"] = dep_tgt - env["SDKROOT"] = sdk_root - print("[setting MACOSX_DEPLOYMENT_TARGET to {}]".format(env["MACOSX_DEPLOYMENT_TARGET"])) - print("[setting SDKROOT to {}]".format(env[ "SDKROOT"])) - - - -def main(): - """ - Clones and runs a package manager to setup third_party libs. - Also creates a host-config.cmake file that can be used by our project. - """ - - # parse args from command line - opts, extra_opts = parse_args() - - # Initialize the environment - env = SpackEnv(opts, extra_opts) - - # Setup the necessary paths and directories - env.setup_paths_and_dirs() - - # Clone the package manager - env.clone_repo() - - os.chdir(env.dest_dir) - - # Patch the package manager, as necessary - env.patch() - - # Clean the build - env.clean_build() - - # Show the spec for what will be built - env.show_info() - - - ########################################################## - # we now have an instance of spack configured how we - # need it to build our tpls at this point there are two - # possible next steps: - # - # *) create a mirror of the packages - # OR - # *) build - # - ########################################################## - if opts["create_mirror"]: - return env.create_mirror() - else: - if not opts["mirror"] is None: - env.use_mirror() - - if not opts["upstream"] is None: - env.use_spack_upstream() - - res = env.install() - - return res - -if __name__ == "__main__": - sys.exit(main()) - - From e56454e0a0d11380c7db4fb60173b94f2bc68f6e Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 3 Aug 2021 14:18:26 -0700 Subject: [PATCH 55/63] Update radiuss-spack-configs --- scripts/radiuss-spack-configs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs index aa21de2d..2f156029 160000 --- a/scripts/radiuss-spack-configs +++ b/scripts/radiuss-spack-configs @@ -1 +1 @@ -Subproject commit aa21de2ddf3576dd7ceac4303508d30632c0c083 +Subproject commit 2f1560293e204c494fc72538161554a552a485be From f2d38b1a1f202bf384d3d1050638029a40608469 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 3 Aug 2021 14:43:38 -0700 Subject: [PATCH 56/63] Point at newer Spack commit --- .uberenv_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.uberenv_config.json b/.uberenv_config.json index 1a0136d9..e3553570 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -5,7 +5,7 @@ "package_source_dir" : "../..", "spack_url": "https://github.com/davidbeckingsale/spack", "spack_branch": "feature/allow-untested-cuda-versions", -"spack_commit": "f96e256bee1948aa030916aae0c1b2645230fb9f", +"spack_commit": "46b22d0f6227f6b12bab712bda5b916a53cfc67d", "spack_activate" : {}, "spack_configs_path": "scripts/radiuss-spack-configs", "spack_packages_path": "scripts/spack_packages"} From c974897fa2b0597f3507a148293d468df9485662 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Tue, 3 Aug 2021 15:13:29 -0700 Subject: [PATCH 57/63] Fix specs and build script --- .gitlab/lassen-jobs.yml | 4 ++-- scripts/gitlab/build_and_test.sh | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index b511be1c..096a326c 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -35,12 +35,12 @@ xl_16_1_1_7_gcc_8_3_1: ibm_clang_9_cuda: variables: - SPEC: "+cuda+allow-untested-versions %clang@9.0.0ibm ^cuda@10.1.168" + SPEC: "+cuda %clang@9.0.0ibm ^cuda@10.1.168" extends: .build_and_test_on_lassen ibm_clang_9_gcc_8_cuda: variables: - SPEC: "+cuda %clang@9.0.0ibm +allow-untested-versions cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" + SPEC: "+cuda %clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" allow_failure: true extends: .build_and_test_on_lassen diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index c801abd9..072f606b 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -15,6 +15,7 @@ project_dir="$(pwd)" build_root=${BUILD_ROOT:-""} hostconfig=${HOST_CONFIG:-""} spec=${SPEC:-""} +job_unique_id=${CI_JOB_ID:-""} raja_version=${UPDATE_RAJA:-""} umpire_version=${UPDATE_UMPIRE:-""} @@ -53,7 +54,16 @@ then if [[ -d /dev/shm ]] then - prefix="/dev/shm/${hostname}/${spec//[ \/\-\=]/_}" + prefix="/dev/shm/${hostname}" + if [[ -z ${job_unique_id} ]]; then + job_unique_id=manual_job_$(date +%s) + while [[ -d ${prefix}/${job_unique_id} ]] ; do + sleep 1 + job_unique_id=manual_job_$(date +%s) + done + fi + + prefix="${prefix}/${job_unique_id}" mkdir -p ${prefix} prefix_opt="--prefix=${prefix}" fi From 9a9063b48eda37dd82631722740f1890c1aedd1c Mon Sep 17 00:00:00 2001 From: Adrien Bernede <51493078+adrienbernede@users.noreply.github.com> Date: Fri, 13 Aug 2021 14:26:16 -0700 Subject: [PATCH 58/63] Fix extra endif in CMakeList --- CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 902c3199..681637a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -44,9 +44,8 @@ set(ENABLE_COPY_HEADERS Off CACHE BOOL "") set(BLT_CXX_STD c++11 CACHE STRING "") if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") - message(STATUS "Setting CMAKE_CXX_EXTENSIONS to ON for PGI Compiler") - set( CMAKE_CXX_EXTENSIONS ON ) - endif() + message(STATUS "Setting CMAKE_CXX_EXTENSIONS to ON for PGI Compiler") + set( CMAKE_CXX_EXTENSIONS ON ) endif() if (ENABLE_CUDA) From a0a9d3cd006b8c327cca7e05df3981ccca2c7b54 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 19 Aug 2021 10:01:34 -0700 Subject: [PATCH 59/63] Update tpls --- blt | 2 +- src/tpl/raja | 2 +- src/tpl/umpire | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/blt b/blt index 7ec2cb80..ddd5a0ca 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit 7ec2cb80525b55d06da683a876b382472ef70661 +Subproject commit ddd5a0ca7c566d0ae14270b66625c8a363630ddb diff --git a/src/tpl/raja b/src/tpl/raja index 3047fa72..357933a4 160000 --- a/src/tpl/raja +++ b/src/tpl/raja @@ -1 +1 @@ -Subproject commit 3047fa720132d19ee143b1fcdacaa72971f5988c +Subproject commit 357933a42842dd91de5c1034204d937fce0a2a44 diff --git a/src/tpl/umpire b/src/tpl/umpire index 447f4640..5f886b42 160000 --- a/src/tpl/umpire +++ b/src/tpl/umpire @@ -1 +1 @@ -Subproject commit 447f4640eff7b8f39d3c59404f3b03629b90c021 +Subproject commit 5f886b4299496b7cb6f9d62dc1372ce6d3832fbc From abdf21d6feb8cd56531c56bad5d4d173c245ae90 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 19 Aug 2021 10:37:26 -0700 Subject: [PATCH 60/63] Fixes for latest RAJA/Umpire --- examples/chai-umpire-allocators.cpp | 6 +++--- src/chai/ManagedArrayView.hpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/chai-umpire-allocators.cpp b/examples/chai-umpire-allocators.cpp index fd999f4f..2e82c6c5 100644 --- a/examples/chai-umpire-allocators.cpp +++ b/examples/chai-umpire-allocators.cpp @@ -5,7 +5,7 @@ // SPDX-License-Identifier: BSD-3-Clause ////////////////////////////////////////////////////////////////////////////// #include "umpire/ResourceManager.hpp" -#include "umpire/strategy/DynamicPool.hpp" +#include "umpire/strategy/QuickPool.hpp" #include "chai/ManagedArray.hpp" #include "../src/util/forall.hpp" @@ -18,12 +18,12 @@ int main(int CHAI_UNUSED_ARG(argc), char** CHAI_UNUSED_ARG(argv)) auto& rm = umpire::ResourceManager::getInstance(); auto cpu_pool = - rm.makeAllocator("cpu_pool", + rm.makeAllocator("cpu_pool", rm.getAllocator("HOST")); #if defined(CHAI_ENABLE_CUDA) || defined(CHAI_ENABLE_HIP) auto gpu_pool = - rm.makeAllocator("gpu_pool", + rm.makeAllocator("gpu_pool", rm.getAllocator("DEVICE")); #endif diff --git a/src/chai/ManagedArrayView.hpp b/src/chai/ManagedArrayView.hpp index 16858f17..26a7326e 100644 --- a/src/chai/ManagedArrayView.hpp +++ b/src/chai/ManagedArrayView.hpp @@ -23,7 +23,7 @@ using ManagedArrayView = template -using TypedManagedArrayView = RAJA::TypedViewBase, LayoutType, IndexTypes...>; From 1842700cc5985e371efd34e6d939c6945cb73827 Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 19 Aug 2021 11:38:50 -0700 Subject: [PATCH 61/63] Fixes for new RAJA --- src/chai/ManagedArrayView.hpp | 2 +- tests/integration/raja-chai-tests.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chai/ManagedArrayView.hpp b/src/chai/ManagedArrayView.hpp index 26a7326e..85ee8407 100644 --- a/src/chai/ManagedArrayView.hpp +++ b/src/chai/ManagedArrayView.hpp @@ -26,7 +26,7 @@ template using TypedManagedArrayView = RAJA::internal::TypedViewBase, LayoutType, - IndexTypes...>; + camp::list >; template using ManagedArrayMultiView = diff --git a/tests/integration/raja-chai-tests.cpp b/tests/integration/raja-chai-tests.cpp index 6e7d6dd3..f43340e6 100644 --- a/tests/integration/raja-chai-tests.cpp +++ b/tests/integration/raja-chai-tests.cpp @@ -94,7 +94,7 @@ CUDA_TEST(ChaiTest, Views) v2(i) *= 2.0f; }); - float* raw_v2 = v2.data.data(); + float* raw_v2 = v2_array.data(); for (int i = 0; i < 10; i++) { ASSERT_FLOAT_EQ(raw_v2[i], i * 1.0f * 2.0f * 2.0f); ; From 7092f2ad57ec9ca07fbe971d381531705e1c387e Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 19 Aug 2021 13:22:59 -0700 Subject: [PATCH 62/63] Remove some quartz CI jobs --- .gitlab/quartz-jobs.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index cc7b409a..0c2cc055 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -19,27 +19,16 @@ gcc_8_1_0: SPEC: "%gcc@8.1.0" extends: .build_and_test_on_quartz -icpc_17_0_2: - variables: - SPEC: "%intel@17.0.2" - extends: .build_and_test_on_quartz - -icpc_18_0_2: - variables: - SPEC: " tests=none %intel@18.0.2" - extends: .build_and_test_on_quartz +# icpc_18_0_2: +# variables: +# SPEC: " tests=none %intel@18.0.2" +# extends: .build_and_test_on_quartz icpc_19_1_0: variables: SPEC: "%intel@19.1.0" extends: .build_and_test_on_quartz -pgi_20_1: - variables: - SPEC: " %pgi@20.1" - PGI_LOCALRC: /collab/usr/global/tools/pgi/etc/localrc-gcc-4.9.3 - extends: .build_and_test_on_quartz - # EXTRAS gcc_4_9_3: From 052f4212b0da591edefca5bc86e911998b5429ff Mon Sep 17 00:00:00 2001 From: David Beckingsale Date: Thu, 19 Aug 2021 13:51:24 -0700 Subject: [PATCH 63/63] Bump release number --- CMakeLists.txt | 2 +- README.md | 2 +- docs/sphinx/conf.py | 4 ++-- docs/sphinx/conf.py.in | 4 ++-- scripts/make_release_tarball.sh | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 681637a6..9ac5b454 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ cmake_policy(SET CMP0057 NEW) cmake_policy(SET CMP0048 NEW) cmake_policy(SET CMP0025 NEW) -project(Chai LANGUAGES CXX VERSION 2.3.0) +project(Chai LANGUAGES CXX VERSION 2.4.0) set(ENABLE_CUDA Off CACHE BOOL "Enable CUDA") set(ENABLE_HIP Off CACHE BOOL "Enable HIP") diff --git a/README.md b/README.md index 874907a7..e69aee09 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# CHAI v2.3 +# CHAI v2.4 [![Azure Build Status](https://dev.azure.com/davidbeckingsale/CHAI/_apis/build/status/LLNL.CHAI?branchName=develop)](https://dev.azure.com/davidbeckingsale/CHAI/_build/latest?definitionId=2&branchName=develop) [![Build Status](https://travis-ci.org/LLNL/CHAI.svg?branch=develop)](https://travis-ci.org/LLNL/CHAI) diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 41c91830..8c3bb894 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -61,9 +61,9 @@ # built documents. # # The short X.Y version. -version = u'2.3' +version = u'2.4' # The full version, including alpha/beta/rc tags. -release = u'2.3.0' +release = u'2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/sphinx/conf.py.in b/docs/sphinx/conf.py.in index a1c90b20..9b08a122 100644 --- a/docs/sphinx/conf.py.in +++ b/docs/sphinx/conf.py.in @@ -60,9 +60,9 @@ author = u'' # built documents. # # The short X.Y version. -version = u'2.3' +version = u'2.4' # The full version, including alpha/beta/rc tags. -release = u'2.3.0' +release = u'2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/scripts/make_release_tarball.sh b/scripts/make_release_tarball.sh index 888c11a0..cce285fe 100755 --- a/scripts/make_release_tarball.sh +++ b/scripts/make_release_tarball.sh @@ -7,7 +7,7 @@ ############################################################################## TAR_CMD=gtar -VERSION=2.3.0 +VERSION=2.4.0 git archive --prefix=chai-${VERSION}/ -o chai-${VERSION}.tar HEAD 2> /dev/null