diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2414f786..5a9da66e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,7 +65,7 @@ stages: # allow pre-allocation the same way slurm does. .build_blueos_3_ppc64le_ib_script: script: - - lalloc 1 -W 10 scripts/gitlab/build_and_test.sh + - lalloc 1 -W 20 scripts/gitlab/build_and_test.sh .build_blueos_3_ppc64le_ib_p9_script: extends: .build_blueos_3_ppc64le_ib_script diff --git a/.gitlab/lassen-jobs.yml b/.gitlab/lassen-jobs.yml index a068e03d..096a326c 100644 --- a/.gitlab/lassen-jobs.yml +++ b/.gitlab/lassen-jobs.yml @@ -5,62 +5,76 @@ # SPDX-License-Identifier: (MIT) ############################################################################## -#### -# All lassen build jobs +# CPU ONLY +ibm_clang_9: + variables: + SPEC: "%clang@9.0.0ibm" + extends: .build_and_test_on_lassen -clang_default (build and test on lassen): +ibm_clang_9_gcc_8: variables: - SPEC: "%clang@default" + SPEC: "%clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen -gcc_default (build and test on lassen): +gcc_8_3_1: variables: - SPEC: "%gcc@default" + SPEC: "%gcc@8.3.1 ^cmake@3.14.5" extends: .build_and_test_on_lassen -xl_default (build and test on lassen): +xl_16_1_1_7: variables: - SPEC: "%xl@default" + SPEC: "%xl@16.1.1.7" extends: .build_and_test_on_lassen -pgi_default (build and test on lassen): +xl_16_1_1_7_gcc_8_3_1: variables: - SPEC: "%pgi@default" + SPEC: "%xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1" extends: .build_and_test_on_lassen - allow_failure: true -gcc_8_3_1 (build and test on lassen): +# CUDA + +ibm_clang_9_cuda: variables: - SPEC: "%gcc@8.3.1" + SPEC: "+cuda %clang@9.0.0ibm ^cuda@10.1.168" extends: .build_and_test_on_lassen -clang_3_9_1 (build and test on lassen): +ibm_clang_9_gcc_8_cuda: variables: - SPEC: "%clang@3.9.1" - extends: .build_and_test_on_lassen_advanced + SPEC: "+cuda %clang@9.0.0ibm cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@10.1.168" + allow_failure: true + extends: .build_and_test_on_lassen -clang_4_0_0 (build and test on lassen): +gcc_8_3_1_cuda: variables: - SPEC: "%clang@4.0.0" - extends: .build_and_test_on_lassen_advanced + SPEC: "+cuda %gcc@8.3.1 ^cuda@10.1.168" + extends: .build_and_test_on_lassen -clang_9_0_0 (build and test on lassen): +xl_16_1_1_7_cuda: variables: - SPEC: "%clang@9.0.0" + SPEC: "+cuda %xl@16.1.1.7 ^cuda@10.1.168 ^cmake@3.14.5" extends: .build_and_test_on_lassen -nvcc_gcc_4_9_3 (build and test on lassen): +xl_16_1_1_7_gcc_8_3_1_cuda_11: variables: - SPEC: "%gcc@4.9.3+cuda" + SPEC: "+cuda %xl@16.1.1.7 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ^cuda@11.0.2 ^cmake@3.14.5" + allow_failure: true extends: .build_and_test_on_lassen -nvcc_gcc_4_9_3_raja (build and test on lassen): +# EXTRAS + +clang_9_0_0_libcpp (build and test on lassen): variables: - SPEC: "%gcc@4.9.3+cuda+raja" + SPEC: "%clang@9.0.0+libcpp" extends: .build_and_test_on_lassen - allow_failure: true -nvcc_xl-beta-2019.06.20 (build and test on lassen): +clang_9_0_0_memleak (build and test on lassen): variables: - SPEC: "%xl@beta2019.06.20+cuda" + SPEC: "%clang@9.0.0 cxxflags=-fsanitize=address" + ASAN_OPTIONS: "detect_leaks=1" extends: .build_and_test_on_lassen + +clang_9_0_0_datarace (build and test on lassen): + variables: + SPEC: "%clang@9.0.0 cxxflags=-fsanitize=thread" + allow_failure: true + extends: .build_and_test_on_lassen \ No newline at end of file diff --git a/.gitlab/quartz-jobs.yml b/.gitlab/quartz-jobs.yml index 31edccb8..0c2cc055 100644 --- a/.gitlab/quartz-jobs.yml +++ b/.gitlab/quartz-jobs.yml @@ -4,93 +4,34 @@ # # SPDX-License-Identifier: (MIT) ############################################################################## - -#### -# Quartz build jobs -clang_3_9_1 (build and test on quartz): - variables: - SPEC: "%clang@3.9.1" - extends: .build_and_test_on_quartz_advanced - -clang_4_0_0 (build and test on quartz): - variables: - SPEC: "%clang@4.0.0" - extends: .build_and_test_on_quartz_advanced - -clang_6_0_0 (build and test on quartz): - variables: - SPEC: "%clang@6.0.0" - extends: .build_and_test_on_quartz_advanced - -clang_9_0_0 (build and test on quartz): +clang_10: variables: - SPEC: "%clang@9.0.0" + SPEC: " %clang@10.0.1" extends: .build_and_test_on_quartz - -gcc_4_9_3 (build and test on quartz): - variables: - SPEC: "%gcc@4.9.3" - extends: .build_and_test_on_quartz_advanced - -gcc_6_1_0 (build and test on quartz): +clang_9: variables: - SPEC: "%gcc@6.1.0" - extends: .build_and_test_on_quartz_advanced - -gcc_7_1_0 (build and test on quartz): - variables: - SPEC: "%gcc@7.1.0" - extends: .build_and_test_on_quartz_advanced - -gcc_7_3_0 (build and test on quartz): - variables: - SPEC: "%gcc@7.3.0" - extends: .build_and_test_on_quartz_advanced + SPEC: " %clang@9.0.0" + extends: .build_and_test_on_quartz -gcc_8_1_0 (build and test on quartz): +gcc_8_1_0: variables: SPEC: "%gcc@8.1.0" extends: .build_and_test_on_quartz +# icpc_18_0_2: +# variables: +# SPEC: " tests=none %intel@18.0.2" +# extends: .build_and_test_on_quartz -icpc_16_0_4 (build and test on quartz): - variables: - SPEC: "%intel@16.0.4" - extends: .build_and_test_on_quartz_advanced - allow_failure: true - -icpc_17_0_2 (build and test on quartz): +icpc_19_1_0: variables: - SPEC: "%intel@17.0.2" - extends: .build_and_test_on_quartz_advanced - -icpc_18_0_0 (build and test on quartz): - variables: - SPEC: "%intel@18.0.0" - extends: .build_and_test_on_quartz_advanced - allow_failure: true - -icpc_19_0_4 (build and test on quartz): - variables: - SPEC: "%intel@19.0.4" + SPEC: "%intel@19.1.0" extends: .build_and_test_on_quartz +# EXTRAS -pgi_17_10 (build and test on quartz): - variables: - SPEC: "%pgi@17.10" - extends: .build_and_test_on_quartz_advanced - allow_failure: true - -pgi_18_5 (build and test on quartz): - variables: - SPEC: "%pgi@18.5" - extends: .build_and_test_on_quartz_advanced - allow_failure: true - -pgi_19_4 (build and test on quartz): +gcc_4_9_3: variables: - SPEC: "%pgi@19.4" + SPEC: "%gcc@4.9.3" extends: .build_and_test_on_quartz - allow_failure: true diff --git a/.gitlab/quartz-templates.yml b/.gitlab/quartz-templates.yml index 744fe246..459bf121 100644 --- a/.gitlab/quartz-templates.yml +++ b/.gitlab/quartz-templates.yml @@ -29,8 +29,7 @@ allocate_resources (on quartz): extends: .on_quartz stage: q_allocate_resources script: - - salloc -N 1 -c 36 -p pdebug -t 10 --no-shell --job-name=${ALLOC_NAME} - needs: [] + - salloc -N 1 -c 36 -p pdebug -t 20 --no-shell --job-name=${ALLOC_NAME} #### # In post-build phase, deallocate resources @@ -47,8 +46,8 @@ release_resources (on quartz): #### # Generic quartz build job, extending build script .build_and_test_on_quartz: - stage: q_build_and_test extends: [.build_toss_3_x86_64_ib_script, .on_quartz] + stage: q_build_and_test .build_and_test_on_quartz_advanced: extends: [.build_and_test_on_quartz, .advanced_pipeline] diff --git a/.gitmodules b/.gitmodules index d77cfd4e..6d879449 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,9 @@ [submodule "src/tpl/raja"] path = src/tpl/raja url = https://github.com/LLNL/RAJA.git +[submodule "scripts/radiuss-spack-configs"] + path = scripts/radiuss-spack-configs + url = https://github.com/LLNL/radiuss-spack-configs +[submodule "scripts/uberenv"] + path = scripts/uberenv + url = https://github.com/LLNL/uberenv.git diff --git a/.uberenv_config.json b/.uberenv_config.json new file mode 100644 index 00000000..e3553570 --- /dev/null +++ b/.uberenv_config.json @@ -0,0 +1,11 @@ +{ +"package_name" : "chai", +"package_version" : "develop", +"package_final_phase" : "hostconfig", +"package_source_dir" : "../..", +"spack_url": "https://github.com/davidbeckingsale/spack", +"spack_branch": "feature/allow-untested-cuda-versions", +"spack_commit": "46b22d0f6227f6b12bab712bda5b916a53cfc67d", +"spack_activate" : {}, +"spack_configs_path": "scripts/radiuss-spack-configs", +"spack_packages_path": "scripts/spack_packages"} diff --git a/CMakeLists.txt b/CMakeLists.txt index 6110b9d3..9ac5b454 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,9 @@ ############################################################################## cmake_policy(SET CMP0057 NEW) cmake_policy(SET CMP0048 NEW) +cmake_policy(SET CMP0025 NEW) -project(Chai LANGUAGES CXX VERSION 2.3.0) +project(Chai LANGUAGES CXX VERSION 2.4.0) set(ENABLE_CUDA Off CACHE BOOL "Enable CUDA") set(ENABLE_HIP Off CACHE BOOL "Enable HIP") @@ -22,6 +23,7 @@ option(ENABLE_UM "Use CUDA unified (managed) memory" Off) option(ENABLE_PINNED "Use pinned host memory" Off) option(ENABLE_RAJA_PLUGIN "Build plugin to set RAJA execution spaces" Off) option(CHAI_ENABLE_GPU_ERROR_CHECKING "Enable GPU error checking" On) +option(CHAI_ENABLE_MANAGED_PTR "Enable managed_ptr" On) option(CHAI_DEBUG "Enable Debug Logging.") set(ENABLE_RAJA_NESTED_TEST ON CACHE BOOL "Enable raja-chai-nested-tests, which fails to build on Debug CUDA builds.") @@ -41,6 +43,11 @@ endif() set(ENABLE_COPY_HEADERS Off CACHE BOOL "") set(BLT_CXX_STD c++11 CACHE STRING "") +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI") + message(STATUS "Setting CMAKE_CXX_EXTENSIONS to ON for PGI Compiler") + set( CMAKE_CXX_EXTENSIONS ON ) +endif() + if (ENABLE_CUDA) cmake_minimum_required(VERSION 3.9) else () diff --git a/README.md b/README.md index 874907a7..e69aee09 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# CHAI v2.3 +# CHAI v2.4 [![Azure Build Status](https://dev.azure.com/davidbeckingsale/CHAI/_apis/build/status/LLNL.CHAI?branchName=develop)](https://dev.azure.com/davidbeckingsale/CHAI/_build/latest?definitionId=2&branchName=develop) [![Build Status](https://travis-ci.org/LLNL/CHAI.svg?branch=develop)](https://travis-ci.org/LLNL/CHAI) diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 14bbc1be..2cb6ec63 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -13,6 +13,7 @@ if (ENABLE_CUDA) ${chai_benchmark_depends} cuda) endif () + if (ENABLE_HIP) set (chai_benchmark_depends ${chai_benchmark_depends} @@ -37,12 +38,13 @@ blt_add_benchmark( NAME managedarray_benchmarks COMMAND managedarray_benchmarks) -blt_add_executable( - NAME managed_ptr_benchmarks - SOURCES chai_managed_ptr_benchmarks.cpp - DEPENDS_ON ${chai_benchmark_depends}) - -blt_add_benchmark( - NAME managed_ptr_benchmarks - COMMAND managed_ptr_benchmarks) +if (CHAI_ENABLE_MANAGED_PTR) + blt_add_executable( + NAME managed_ptr_benchmarks + SOURCES chai_managed_ptr_benchmarks.cpp + DEPENDS_ON ${chai_benchmark_depends}) + blt_add_benchmark( + NAME managed_ptr_benchmarks + COMMAND managed_ptr_benchmarks) +endif () diff --git a/benchmarks/chai_managed_ptr_benchmarks.cpp b/benchmarks/chai_managed_ptr_benchmarks.cpp index 62eb6564..7352f2c5 100644 --- a/benchmarks/chai_managed_ptr_benchmarks.cpp +++ b/benchmarks/chai_managed_ptr_benchmarks.cpp @@ -9,6 +9,7 @@ #include "benchmark/benchmark.h" #include "chai/config.hpp" +#include "chai/ArrayManager.hpp" #include "chai/managed_ptr.hpp" #include "../src/util/forall.hpp" @@ -134,8 +135,8 @@ static void benchmark_use_managed_ptr_cpu(benchmark::State& state) values[i] = i * i; } -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + chai::synchronize(); #endif while (state.KeepRunning()) { @@ -145,8 +146,8 @@ static void benchmark_use_managed_ptr_cpu(benchmark::State& state) free(values); object.free(); -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + chai::synchronize(); #endif } @@ -209,7 +210,7 @@ static void benchmark_pass_copy_to_gpu(benchmark::State& state) while (state.KeepRunning()) { copy_kernel<<<1, 1>>>(helper); - cudaDeviceSynchronize(); + chai::synchronize(); } } @@ -225,10 +226,10 @@ static void benchmark_copy_to_gpu(benchmark::State& state) while (state.KeepRunning()) { ClassWithSize* gpuPointer; - cudaMalloc(&gpuPointer, sizeof(ClassWithSize)); - cudaMemcpy(gpuPointer, cpuPointer, sizeof(ClassWithSize), cudaMemcpyHostToDevice); - cudaFree(gpuPointer); - cudaDeviceSynchronize(); + chai::gpuMalloc((void**)(&gpuPointer), sizeof(ClassWithSize)); + chai::gpuMemcpy(gpuPointer, cpuPointer, sizeof(ClassWithSize), gpuMemcpyHostToDevice); + chai::gpuFree(gpuPointer); + chai::synchronize(); } delete cpuPointer; @@ -258,11 +259,11 @@ static void benchmark_placement_new_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize* address; - cudaMalloc(&address, sizeof(ClassWithSize)); + chai::gpuMalloc((void**)(&address), sizeof(ClassWithSize)); placement_new_kernel<<<1, 1>>>(address); placement_delete_kernel<<<1, 1>>>(address); - cudaFree(address); - cudaDeviceSynchronize(); + chai::gpuFree(address); + chai::synchronize(); } } @@ -290,11 +291,11 @@ static void benchmark_new_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize** buffer; - cudaMalloc(&buffer, sizeof(ClassWithSize*)); + chai::gpuMalloc((void**)(&buffer), sizeof(ClassWithSize*)); create_kernel<<<1, 1>>>(buffer); delete_kernel<<<1, 1>>>(buffer); - cudaFree(buffer); - cudaDeviceSynchronize(); + chai::gpuFree(buffer); + chai::synchronize(); } } @@ -317,15 +318,15 @@ static void benchmark_new_on_gpu_and_copy_to_host(benchmark::State& state) { while (state.KeepRunning()) { ClassWithSize** gpuBuffer; - cudaMalloc(&gpuBuffer, sizeof(ClassWithSize*)); + chai::gpuMalloc((void**)(&gpuBuffer), sizeof(ClassWithSize*)); create_kernel<<<1, 1>>>(gpuBuffer); ClassWithSize** cpuBuffer = (ClassWithSize**) malloc(sizeof(ClassWithSize*)); - cudaMemcpy(cpuBuffer, gpuBuffer, sizeof(ClassWithSize*), cudaMemcpyDeviceToHost); - cudaFree(gpuBuffer); + chai::gpuMemcpy(cpuBuffer, gpuBuffer, sizeof(ClassWithSize*), gpuMemcpyDeviceToHost); + chai::gpuFree(gpuBuffer); ClassWithSize* gpuPointer = cpuBuffer[0]; free(cpuBuffer); delete_kernel_2<<<1, 1>>>(gpuPointer); - cudaDeviceSynchronize(); + chai::synchronize(); } } @@ -348,7 +349,7 @@ static void benchmark_create_on_stack_on_gpu(benchmark::State& state) { while (state.KeepRunning()) { create_on_stack_kernel<<<1, 1>>>(); - cudaDeviceSynchronize(); + chai::synchronize(); } } @@ -379,19 +380,19 @@ void benchmark_use_managed_ptr_gpu(benchmark::State& state) int numValues = 100; int* values; - cudaMalloc(&values, numValues * sizeof(int)); + chai::gpuMalloc((void**)(&values), numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - cudaDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - cudaDeviceSynchronize(); + chai::synchronize(); } - cudaFree(values); + chai::gpuFree(values); object.free(); - cudaDeviceSynchronize(); + chai::synchronize(); } BENCHMARK(benchmark_use_managed_ptr_gpu); @@ -409,19 +410,19 @@ void benchmark_curiously_recurring_template_pattern_gpu(benchmark::State& state) int numValues = 100; int* values; - cudaMalloc(&values, numValues * sizeof(int)); + chai::gpuMalloc((void**)(&values), numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - cudaDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - cudaDeviceSynchronize(); + chai::synchronize(); } - cudaFree(values); + chai::gpuFree(values); delete derivedCRTP; - cudaDeviceSynchronize(); + chai::synchronize(); } BENCHMARK(benchmark_curiously_recurring_template_pattern_gpu); @@ -438,19 +439,19 @@ void benchmark_no_inheritance_gpu(benchmark::State& state) int numValues = 100; int* values; - cudaMalloc(&values, numValues * sizeof(int)); + chai::gpuMalloc((void**)(&values), numValues * sizeof(int)); fill<<<1, 100>>>(numValues, values); - cudaDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<1, 1>>>(object, numValues, values); - cudaDeviceSynchronize(); + chai::synchronize(); } - cudaFree(values); + chai::gpuFree(values); delete noInheritance; - cudaDeviceSynchronize(); + chai::synchronize(); } BENCHMARK(benchmark_no_inheritance_gpu); @@ -471,19 +472,19 @@ void benchmark_bulk_use_managed_ptr_gpu(benchmark::State& state) chai::managed_ptr object = chai::make_managed(2); int* values; - cudaMalloc(&values, N * sizeof(int)); + chai::gpuMalloc((void**)(&values), N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - cudaDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - cudaDeviceSynchronize(); + chai::synchronize(); } - cudaFree(values); + chai::gpuFree(values); object.free(); - cudaDeviceSynchronize(); + chai::synchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_use_managed_ptr_gpu, 1); @@ -519,19 +520,19 @@ void benchmark_bulk_curiously_recurring_template_pattern_gpu(benchmark::State& s auto object = *derivedCRTP; int* values; - cudaMalloc(&values, N * sizeof(int)); + chai::gpuMalloc((void**)(&values), N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - cudaDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - cudaDeviceSynchronize(); + chai::synchronize(); } - cudaFree(values); + chai::gpuFree(values); delete derivedCRTP; - cudaDeviceSynchronize(); + chai::synchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_curiously_recurring_template_pattern_gpu, 1); @@ -567,19 +568,19 @@ void benchmark_bulk_no_inheritance_gpu(benchmark::State& state) auto object = *noInheritance; int* values; - cudaMalloc(&values, N * sizeof(int)); + chai::gpuMalloc((void**)(&values), N * sizeof(int)); fill<<<(N+255)/256, 256>>>(N, values); - cudaDeviceSynchronize(); + chai::synchronize(); while (state.KeepRunning()) { square<<<(N+255)/256, 256>>>(N, values, object); - cudaDeviceSynchronize(); + chai::synchronize(); } - cudaFree(values); + chai::gpuFree(values); delete noInheritance; - cudaDeviceSynchronize(); + chai::synchronize(); } BENCHMARK_TEMPLATE(benchmark_bulk_no_inheritance_gpu, 1); @@ -612,8 +613,8 @@ static void benchmark_bulk_polymorphism_cpu(benchmark::State& state) values[i] = i * i; } -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + chai::synchronize(); #endif while (state.KeepRunning()) { @@ -626,8 +627,8 @@ static void benchmark_bulk_polymorphism_cpu(benchmark::State& state) free(values); delete object; -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + chai::synchronize(); #endif } @@ -659,8 +660,8 @@ static void benchmark_bulk_use_managed_ptr_cpu(benchmark::State& state) values[i] = i * i; } -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + chai::synchronize(); #endif while (state.KeepRunning()) { @@ -673,8 +674,8 @@ static void benchmark_bulk_use_managed_ptr_cpu(benchmark::State& state) free(values); object.free(); -#ifdef __CUDACC__ - cudaDeviceSynchronize(); +#ifdef CHAI_GPUCC + chai::synchronize(); #endif } diff --git a/blt b/blt index cbe99c93..ddd5a0ca 160000 --- a/blt +++ b/blt @@ -1 +1 @@ -Subproject commit cbe99c93d0d83ca04f13018cdfed9a3663ea248c +Subproject commit ddd5a0ca7c566d0ae14270b66625c8a363630ddb diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 41c91830..8c3bb894 100644 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -61,9 +61,9 @@ # built documents. # # The short X.Y version. -version = u'2.3' +version = u'2.4' # The full version, including alpha/beta/rc tags. -release = u'2.3.0' +release = u'2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/sphinx/conf.py.in b/docs/sphinx/conf.py.in index a1c90b20..9b08a122 100644 --- a/docs/sphinx/conf.py.in +++ b/docs/sphinx/conf.py.in @@ -60,9 +60,9 @@ author = u'' # built documents. # # The short X.Y version. -version = u'2.3' +version = u'2.4' # The full version, including alpha/beta/rc tags. -release = u'2.3.0' +release = u'2.4.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/examples/chai-umpire-allocators.cpp b/examples/chai-umpire-allocators.cpp index fd999f4f..2e82c6c5 100644 --- a/examples/chai-umpire-allocators.cpp +++ b/examples/chai-umpire-allocators.cpp @@ -5,7 +5,7 @@ // SPDX-License-Identifier: BSD-3-Clause ////////////////////////////////////////////////////////////////////////////// #include "umpire/ResourceManager.hpp" -#include "umpire/strategy/DynamicPool.hpp" +#include "umpire/strategy/QuickPool.hpp" #include "chai/ManagedArray.hpp" #include "../src/util/forall.hpp" @@ -18,12 +18,12 @@ int main(int CHAI_UNUSED_ARG(argc), char** CHAI_UNUSED_ARG(argv)) auto& rm = umpire::ResourceManager::getInstance(); auto cpu_pool = - rm.makeAllocator("cpu_pool", + rm.makeAllocator("cpu_pool", rm.getAllocator("HOST")); #if defined(CHAI_ENABLE_CUDA) || defined(CHAI_ENABLE_HIP) auto gpu_pool = - rm.makeAllocator("gpu_pool", + rm.makeAllocator("gpu_pool", rm.getAllocator("DEVICE")); #endif diff --git a/scripts/gitlab/build_and_test.sh b/scripts/gitlab/build_and_test.sh index f3dc8e48..072f606b 100755 --- a/scripts/gitlab/build_and_test.sh +++ b/scripts/gitlab/build_and_test.sh @@ -15,6 +15,7 @@ project_dir="$(pwd)" build_root=${BUILD_ROOT:-""} hostconfig=${HOST_CONFIG:-""} spec=${SPEC:-""} +job_unique_id=${CI_JOB_ID:-""} raja_version=${UPDATE_RAJA:-""} umpire_version=${UPDATE_UMPIRE:-""} @@ -53,7 +54,16 @@ then if [[ -d /dev/shm ]] then - prefix="/dev/shm/${hostname}/${spec// /_}" + prefix="/dev/shm/${hostname}" + if [[ -z ${job_unique_id} ]]; then + job_unique_id=manual_job_$(date +%s) + while [[ -d ${prefix}/${job_unique_id} ]] ; do + sleep 1 + job_unique_id=manual_job_$(date +%s) + done + fi + + prefix="${prefix}/${job_unique_id}" mkdir -p ${prefix} prefix_opt="--prefix=${prefix}" fi @@ -120,7 +130,7 @@ then fi # Test -if [[ "${option}" != "--build-only" ]] +if [[ "${option}" != "--build-only" ]] && grep -q -i "ENABLE_TESTS.*ON" ${hostconfig_path} then echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" echo "~~~~~ Testing CHAI" diff --git a/scripts/make_release_tarball.sh b/scripts/make_release_tarball.sh index 888c11a0..cce285fe 100755 --- a/scripts/make_release_tarball.sh +++ b/scripts/make_release_tarball.sh @@ -7,7 +7,7 @@ ############################################################################## TAR_CMD=gtar -VERSION=2.3.0 +VERSION=2.4.0 git archive --prefix=chai-${VERSION}/ -o chai-${VERSION}.tar HEAD 2> /dev/null diff --git a/scripts/radiuss-spack-configs b/scripts/radiuss-spack-configs new file mode 160000 index 00000000..2f156029 --- /dev/null +++ b/scripts/radiuss-spack-configs @@ -0,0 +1 @@ +Subproject commit 2f1560293e204c494fc72538161554a552a485be diff --git a/scripts/uberenv/packages/chai/package.py b/scripts/spack_packages/chai/package.py similarity index 87% rename from scripts/uberenv/packages/chai/package.py rename to scripts/spack_packages/chai/package.py index 0fa189f3..28442947 100644 --- a/scripts/uberenv/packages/chai/package.py +++ b/scripts/spack_packages/chai/package.py @@ -12,6 +12,8 @@ from os import environ as env from os.path import join as pjoin +import re + def cmake_cache_entry(name, value, comment=""): """Generate a string for a cmake cache variable""" @@ -66,18 +68,22 @@ class Chai(CMakePackage, CudaPackage): version('1.1.0', tag='v1.1.0', submodules='True') version('1.0', tag='v1.0', submodules='True') - variant('shared', default=True, description='Build Shared Libs') - variant('raja', default=False, description='Build plugin for RAJA') + variant('shared', default=False, description='Build Shared Libs') + variant('raja', default=True, description='Build plugin for RAJA') variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') + variant('libcpp', default=False, description='Use libc++') depends_on('cmake@3.8:', type='build') - depends_on('umpire') - depends_on('raja', when="+raja") + depends_on('umpire@main') + depends_on('raja@main', when="+raja") depends_on('cmake@3.9:', type='build', when="+cuda") depends_on('umpire+cuda', when="+cuda") depends_on('raja+cuda', when="+raja+cuda") + depends_on('umpire+cuda+allow-untested-versions', when="+cuda+allow-untested-versions") + depends_on('raja+cuda+allow-untested-versions', when="+raja+cuda+allow-untested-versions") + depends_on('umpire+libcpp', when='+libcpp') phases = ['hostconfig', 'cmake', 'build', 'install'] @@ -185,6 +191,8 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) + if "+libcpp" in spec: + cxxflags += ' '.join([cxxflags,"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0"]) if cxxflags: cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) @@ -200,6 +208,17 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("BLT_EXE_LINKER_FLAGS", flags, description)) + gcc_toolchain_regex = re.compile(".*gcc-toolchain.*") + gcc_name_regex = re.compile(".*gcc-name.*") + + using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) + using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) + compilers_using_toolchain = ["pgi", "xl", "icpc"] + if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): + if using_toolchain or using_gcc_name: + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) + if "+cuda" in spec: cfg.write("#------------------{0}\n".format("-" * 60)) cfg.write("# Cuda\n") diff --git a/scripts/uberenv/packages/raja/package.py b/scripts/spack_packages/raja/package.py similarity index 69% rename from scripts/uberenv/packages/raja/package.py rename to scripts/spack_packages/raja/package.py index 605cc046..c7ea7117 100644 --- a/scripts/uberenv/packages/raja/package.py +++ b/scripts/spack_packages/raja/package.py @@ -12,6 +12,8 @@ from os import environ as env from os.path import join as pjoin +import re + def cmake_cache_entry(name, value, comment=""): """Generate a string for a cmake cache variable""" @@ -56,7 +58,9 @@ class Raja(CMakePackage, CudaPackage): git = "https://github.com/LLNL/RAJA.git" version('develop', branch='develop', submodules='True') - version('master', branch='main', submodules='True') + version('main', branch='main', submodules='True') + version('0.12.1', tag='v0.12.1', submodules="True") + version('0.12.0', tag='v0.12.0', submodules="True") version('0.11.0', tag='v0.11.0', submodules="True") version('0.10.1', tag='v0.10.1', submodules="True") version('0.10.0', tag='v0.10.0', submodules="True") @@ -71,13 +75,18 @@ class Raja(CMakePackage, CudaPackage): version('0.4.1', tag='v0.4.1', submodules="True") version('0.4.0', tag='v0.4.0', submodules="True") - variant('openmp', default=True, description='Build OpenMP backend') - variant('shared', default=True, description='Build Shared Libs') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), + variant('openmp', default=False, description='Build OpenMP backend') + variant('shared', default=False, description='Build Shared Libs') + variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') + variant('hip', default=False, description='Build with HIP support') + variant('tests', default='none', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') depends_on('cmake@3.8:', type='build') depends_on('cmake@3.9:', when='+cuda', type='build') + depends_on('hip', when='+hip') + + conflicts('+openmp', when='+hip') phases = ['hostconfig', 'cmake', 'build', 'install'] @@ -92,6 +101,8 @@ def _get_host_config_path(self, spec): var='' if '+cuda' in spec: var= '-'.join([var,'cuda']) + if '+libcpp' in spec: + var='-'.join([var,'libcpp']) host_config_path = "hc-%s-%s-%s%s-%s.cmake" % (socket.gethostname().rstrip('1234567890'), self._get_sys_type(spec), @@ -183,10 +194,14 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): # use global spack compiler flags cflags = ' '.join(spec.compiler_flags['cflags']) + if "+libcpp" in spec: + cflags += ' '.join([cflags,"-DGTEST_HAS_CXXABI_H_=0"]) if cflags: cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) + if "+libcpp" in spec: + cxxflags += ' '.join([cxxflags,"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0"]) if cxxflags: cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) @@ -203,21 +218,18 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("BLT_EXE_LINKER_FLAGS", flags, description)) - if "toss_3_x86_64_ib" in sys_type: - release_flags = "-O3 -msse4.2 -funroll-loops -finline-functions" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) - reldebinf_flags = "-O3 -g -msse4.2 -funroll-loops -finline-functions" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELWITHDEBINFO", reldebinf_flags)) - debug_flags = "-O0 -g" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - - if "blueos_3_ppc64le_ib" in sys_type: - release_flags = "-O3" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) - reldebinf_flags = "-O3 -g" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELWITHDEBINFO", reldebinf_flags)) - debug_flags = "-O0 -g" - cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) + gcc_toolchain_regex = re.compile("--gcc-toolchain=(.*)") + gcc_name_regex = re.compile(".*gcc-name.*") + + using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) + if(using_toolchain): + gcc_toolchain_path = gcc_toolchain_regex.match(using_toolchain[0]) + using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) + compilers_using_toolchain = ["pgi", "xl", "icpc"] + if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): + if using_toolchain or using_gcc_name: + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) if "+cuda" in spec: cfg.write("#------------------{0}\n".format("-" * 60)) @@ -233,14 +245,52 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", cudacompiler)) + cuda_release_flags = "-O3 -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" + cuda_reldebinf_flags = "-O3 -g -Xcompiler -Ofast -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" + cuda_debug_flags = "-O0 -g -Xcompiler -O0 -Xcompiler -finline-functions -Xcompiler -finline-limit=20000" + + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELEASE", cuda_release_flags)) + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_RELWITHDEBINFO", cuda_reldebinf_flags)) + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS_DEBUG", cuda_debug_flags)) + if not spec.satisfies('cuda_arch=none'): cuda_arch = spec.variants['cuda_arch'].value - flag = '-arch sm_{0}'.format(cuda_arch[0]) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", flag)) + cfg.write(cmake_cache_string("CUDA_ARCH", 'sm_{0}'.format(cuda_arch[0]))) else: cfg.write(cmake_cache_option("ENABLE_CUDA", False)) + if "+hip" in spec: + cfg.write("#------------------{0}\n".format("-" * 60)) + cfg.write("# HIP\n") + cfg.write("#------------------{0}\n\n".format("-" * 60)) + + cfg.write(cmake_cache_option("ENABLE_HIP", True)) + + hip_root = spec['hip'].prefix + rocm_root = hip_root + "/.." + cfg.write(cmake_cache_entry("HIP_ROOT_DIR", + hip_root)) + cfg.write(cmake_cache_entry("HIP_CLANG_PATH", + rocm_root + '/llvm/bin')) + cfg.write(cmake_cache_entry("HIP_HIPCC_FLAGS", + '--amdgpu-target=gfx906')) + cfg.write(cmake_cache_entry("HIP_RUNTIME_INCLUDE_DIRS", + "{0}/include;{0}/../hsa/include".format(hip_root))) + if ('%gcc' in spec) or (using_toolchain): + if ('%gcc' in spec): + gcc_bin = os.path.dirname(self.compiler.cxx) + gcc_prefix = join_path(gcc_bin, '..') + else: + gcc_prefix = gcc_toolchain_path.group(1) + cfg.write(cmake_cache_entry("HIP_CLANG_FLAGS", + "--gcc-toolchain={0}".format(gcc_prefix))) + cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", + "-Wl,-rpath {}/lib64".format(gcc_prefix))) + + else: + cfg.write(cmake_cache_option("ENABLE_HIP", False)) + cfg.write("#------------------{0}\n".format("-" * 60)) cfg.write("# Other\n") cfg.write("#------------------{0}\n\n".format("-" * 60)) @@ -255,16 +305,18 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option("BUILD_SHARED_LIBS","+shared" in spec)) cfg.write(cmake_cache_option("ENABLE_OPENMP","+openmp" in spec)) - # Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS which - # is used by the spack compiler wrapper. This can go away when BLT - # removes -Werror from GTest flags + # Note 1: Work around spack adding -march=ppc64le to SPACK_TARGET_ARGS + # which is used by the spack compiler wrapper. This can go away when + # BLT removes -Werror from GTest flags + # Note 2: Tests are either built if variant is set, or if run-tests + # option is passed. if self.spec.satisfies('%clang target=ppc64le:'): cfg.write(cmake_cache_option("ENABLE_TESTS",False)) if 'tests=benchmarks' in spec or not 'tests=none' in spec: print("MSG: no testing supported on %clang target=ppc64le:") else: cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) - cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec)) + cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec or self.run_tests)) ####################### # Close and save diff --git a/scripts/uberenv/packages/umpire/camp_target_umpire_3.0.0.patch b/scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch similarity index 100% rename from scripts/uberenv/packages/umpire/camp_target_umpire_3.0.0.patch rename to scripts/spack_packages/umpire/camp_target_umpire_3.0.0.patch diff --git a/scripts/uberenv/packages/umpire/package.py b/scripts/spack_packages/umpire/package.py similarity index 65% rename from scripts/uberenv/packages/umpire/package.py rename to scripts/spack_packages/umpire/package.py index edeb7f86..8bc83634 100644 --- a/scripts/uberenv/packages/umpire/package.py +++ b/scripts/spack_packages/umpire/package.py @@ -12,6 +12,8 @@ from os import environ as env from os.path import join as pjoin +import re + def cmake_cache_entry(name, value, comment=""): """Generate a string for a cmake cache variable""" @@ -48,7 +50,6 @@ def get_spec_path(spec, package_name, path_replacements = {}, use_bin = False) : return path - class Umpire(CMakePackage, CudaPackage): """An application-focused API for memory management on NUMA & GPU architectures""" @@ -57,7 +58,8 @@ class Umpire(CMakePackage, CudaPackage): git = 'https://github.com/LLNL/Umpire.git' version('develop', branch='develop', submodules='True') - version('master', branch='main', submodules='True') + version('main', branch='main', submodules='True') + version('4.1.2', tag='v4.1.2', submodules='True') version('3.0.0', tag='v3.0.0', submodules='True') version('2.1.0', tag='v2.1.0', submodules='True') version('2.0.0', tag='v2.0.0', submodules='True') @@ -81,22 +83,38 @@ class Umpire(CMakePackage, CudaPackage): patch('camp_target_umpire_3.0.0.patch', when='@3.0.0') variant('fortran', default=False, description='Build C/Fortran API') - variant('c', default=True, description='Build C API') + variant('c', default=False, description='Build C API') + variant('mpi', default=False, description='Enable MPI support') variant('numa', default=False, description='Enable NUMA support') - variant('shared', default=True, description='Enable Shared libs') + variant('shared', default=False, description='Enable Shared libs') variant('openmp', default=False, description='Build with OpenMP support') + variant('openmp_target', default=False, description='Build with OpenMP 4.5 support') variant('deviceconst', default=False, description='Enables support for constant device memory') - variant('tests', default='basic', values=('none', 'basic', 'benchmarks'), + variant('tests', default='none', values=('none', 'basic', 'benchmarks'), multi=False, description='Tests to run') variant('libcpp', default=False, description='Uses libc++ instead of libstdc++') + variant('hip', default=False, description='Build with HIP support') + variant('tools', default=False, description='Enable tools') + variant('dev_benchmarks', default=False, description='Enable Developer Benchmarks') + variant('werror', default=False, description='Enable warnings as errors') + variant('asan', default=False, description='Enable ASAN') + variant('sanitizer_tests', default=False, description='Enable address sanitizer tests') depends_on('cmake@3.8:', type='build') depends_on('cmake@3.9:', when='+cuda', type='build') + depends_on('mpi', when='+mpi') + depends_on('hip', when='+hip') conflicts('+numa', when='@:0.3.2') conflicts('~c', when='+fortran', msg='Fortran API requires C API') + conflicts('~openmp', when='+openmp_target', msg='OpenMP target requires OpenMP') + conflicts('+cuda', when='+hip') + conflicts('+openmp', when='+hip') + conflicts('+openmp_target', when='+hip') + conflicts('+deviceconst', when='~hip~cuda') + conflicts('+sanitizer_tests', when='~asan') phases = ['hostconfig', 'cmake', 'build', 'install'] @@ -157,6 +175,7 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if os.path.isfile(env["SPACK_FC"]): f_compiler = env["SPACK_FC"] + ####################################################################### # By directly fetching the names of the actual compilers we appear # to doing something evil here, but this is necessary to create a @@ -201,6 +220,8 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#------------------\n\n".format("-" * 60)) cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + if '+fortran' in spec: + cfg.write(cmake_cache_entry("CMAKE_Fortran_COMPILER", f_compiler)) # use global spack compiler flags cflags = ' '.join(spec.compiler_flags['cflags']) @@ -215,7 +236,14 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if cxxflags: cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) - if ("gfortran" in f_compiler) and ("clang" in cpp_compiler): + fflags = ' '.join(spec.compiler_flags['fflags']) + cfg.write(cmake_cache_entry("CMAKE_Fortran_FLAGS", fflags)) + + fortran_compilers = ["gfortran", "xlf"] + if any(compiler in f_compiler for compiler in fortran_compilers) and ("clang" in cpp_compiler): + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) + libdir = pjoin(os.path.dirname( os.path.dirname(f_compiler)), "lib") flags = "" @@ -227,6 +255,18 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("BLT_EXE_LINKER_FLAGS", flags, description)) + + gcc_toolchain_regex = re.compile(".*gcc-toolchain.*") + gcc_name_regex = re.compile(".*gcc-name.*") + + using_toolchain = list(filter(gcc_toolchain_regex.match, spec.compiler_flags['cxxflags'])) + using_gcc_name = list(filter(gcc_name_regex.match, spec.compiler_flags['cxxflags'])) + compilers_using_toolchain = ["pgi", "xl", "icpc"] + if any(compiler in cpp_compiler for compiler in compilers_using_toolchain): + if using_toolchain or using_gcc_name: + cfg.write(cmake_cache_entry("BLT_CMAKE_IMPLICIT_LINK_DIRECTORIES_EXCLUDE", + "/usr/tce/packages/gcc/gcc-4.9.3/lib64;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64/gcc/powerpc64le-unknown-linux-gnu/4.9.3;/usr/tce/packages/gcc/gcc-4.9.3/gnu/lib64;/usr/tce/packages/gcc/gcc-4.9.3/lib64/gcc/x86_64-unknown-linux-gnu/4.9.3")) + if "toss_3_x86_64_ib" in sys_type: release_flags = "-O3" reldebinf_flags = "-O3 -g" @@ -240,6 +280,19 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS_RELWITHDEBINFO", reldebinf_flags)) cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) + #Configuration to enable developer benchmarks (i.e. No-Op) + if "+dev_benchmarks" in spec: + cfg.write("#------------------{0}\n".format("-" * 60)) + cfg.write("# Developer Benchmarks\n") + cfg.write("#------------------{0}\n\n".format("-" * 60)) + + cfg.write(cmake_cache_option("ENABLE_DEVELOPER_BENCHMARKS", True)) + cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", True)) #Enable BLT GoogleBenchmark support + else: + cfg.write(cmake_cache_option("ENABLE_DEVELOPER_BENCHMARKS", False)) + cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", False)) + + if "+cuda" in spec: cfg.write("#------------------{0}\n".format("-" * 60)) cfg.write("# Cuda\n") @@ -254,24 +307,76 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_entry("CMAKE_CUDA_COMPILER", cudacompiler)) + cuda_flags = [] + if not spec.satisfies('cuda_arch=none'): cuda_arch = spec.variants['cuda_arch'].value - flag = '-arch sm_{0}'.format(cuda_arch[0]) - cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", flag)) + cuda_flags.append('-arch sm_{0}'.format(cuda_arch[0])) if '+deviceconst' in spec: cfg.write(cmake_cache_option("ENABLE_DEVICE_CONST", True)) + if using_toolchain: + cuda_flags.append("-Xcompiler {}".format(using_toolchain[0])) + + cfg.write(cmake_cache_string("CMAKE_CUDA_FLAGS", ' '.join(cuda_flags))) + else: cfg.write(cmake_cache_option("ENABLE_CUDA", False)) + if "+hip" in spec: + cfg.write("#------------------{0}\n".format("-" * 60)) + cfg.write("# HIP\n") + cfg.write("#------------------{0}\n\n".format("-" * 60)) + + cfg.write(cmake_cache_option("ENABLE_HIP", True)) + +# -DHIP_ROOT_DIR=/opt/rocm-3.6.0/hip -DHIP_CLANG_PATH=/opt/rocm-3.6.0/llvm/bin + + hip_root = spec['hip'].prefix + rocm_root = hip_root + "/.." + cfg.write(cmake_cache_entry("HIP_ROOT_DIR", + hip_root)) + cfg.write(cmake_cache_entry("HIP_CLANG_PATH", + rocm_root + '/llvm/bin')) + cfg.write(cmake_cache_entry("HIP_HIPCC_FLAGS", + '--amdgpu-target=gfx906')) + cfg.write(cmake_cache_entry("HIP_RUNTIME_INCLUDE_DIRS", + "{0}/include;{0}/../hsa/include".format(hip_root))) + if '%gcc' in spec: + gcc_bin = os.path.dirname(self.compiler.cxx) + gcc_prefix = join_path(gcc_bin, '..') + cfg.write(cmake_cache_entry("HIP_CLANG_FLAGS", + "--gcc-toolchain={0}".format(gcc_prefix))) + cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", + "-Wl,-rpath {}/lib64".format(gcc_prefix))) + + if '+deviceconst' in spec: + cfg.write(cmake_cache_option("ENABLE_DEVICE_CONST", True)) + + else: + cfg.write(cmake_cache_option("ENABLE_HIP", False)) + cfg.write(cmake_cache_option("ENABLE_C", '+c' in spec)) cfg.write(cmake_cache_option("ENABLE_FORTRAN", '+fortran' in spec)) + + if "+mpi" in spec: + cfg.write(cmake_cache_option("ENABLE_MPI", '+mpi' in spec)) + cfg.write(cmake_cache_entry("MPI_CXX_COMPILER", spec['mpi'].mpicxx)) + cfg.write(cmake_cache_option("ENABLE_NUMA", '+numa' in spec)) cfg.write(cmake_cache_option("ENABLE_OPENMP", '+openmp' in spec)) + if "+openmp_target" in spec: + cfg.write(cmake_cache_option("ENABLE_OPENMP_TARGET", True)) + if ('%xl' in spec): + cfg.write(cmake_cache_entry("OpenMP_CXX_FLAGS", "-qsmp;-qoffload")) cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec)) cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec)) + cfg.write(cmake_cache_option("ENABLE_TOOLS", '+tools' in spec)) + cfg.write(cmake_cache_option("ENABLE_WARNINGS_AS_ERRORS", '+werror' in spec)) + cfg.write(cmake_cache_option("ENABLE_ASAN", '+asan' in spec)) + cfg.write(cmake_cache_option("ENABLE_SANITIZER_TESTS", '+sanitizer_tests' in spec)) ####################### # Close and save diff --git a/scripts/uberenv b/scripts/uberenv new file mode 160000 index 00000000..105e384f --- /dev/null +++ b/scripts/uberenv @@ -0,0 +1 @@ +Subproject commit 105e384f585e2391c42b2def93124a6580319c1c diff --git a/scripts/uberenv/LICENSE b/scripts/uberenv/LICENSE deleted file mode 100644 index fcd00312..00000000 --- a/scripts/uberenv/LICENSE +++ /dev/null @@ -1,64 +0,0 @@ -Copyright (c) 2014-2018, Lawrence Livermore National Security, LLC. - -Produced at the Lawrence Livermore National Laboratory - -LLNL-CODE-666778 - -All rights reserved. - -This file is part of Conduit. - -For details, see: http://software.llnl.gov/conduit/. - -Please also read conduit/LICENSE - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, - this list of conditions and the disclaimer below. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the disclaimer (as noted below) in the - documentation and/or other materials provided with the distribution. - -* Neither the name of the LLNS/LLNL nor the names of its contributors may - be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -Additional BSD Notice - - 1. This notice is required to be provided under our contract with the U.S. - Department of Energy (DOE). This work was produced at Lawrence - Livermore National Laboratory under Contract No. DE-AC52-07NA27344 with - the DOE. - - 2. Neither the United States Government nor Lawrence Livermore National - Security, LLC nor any of their employees, makes any warranty, express - or implied, or assumes any liability or responsibility for the - accuracy, completeness, or usefulness of any information, apparatus, - product, or process disclosed, or represents that its use would not - infringe privately-owned rights. - - 3. Also, reference herein to any specific commercial products, process, - or services by trade name, trademark, manufacturer or otherwise does - not necessarily constitute or imply its endorsement, recommendation, - or favoring by the United States Government or Lawrence Livermore - National Security, LLC. The views and opinions of authors expressed - herein do not necessarily state or reflect those of the United - States Government or Lawrence Livermore National Security, LLC, and - shall not be used for advertising or product endorsement purposes. - diff --git a/scripts/uberenv/Makefile b/scripts/uberenv/Makefile deleted file mode 100644 index 2760762d..00000000 --- a/scripts/uberenv/Makefile +++ /dev/null @@ -1,6 +0,0 @@ - -default: - sphinx-build -E -a -b html docs/sphinx/ _docs_html - -clean: - rm -rf _docs_html \ No newline at end of file diff --git a/scripts/uberenv/README.md b/scripts/uberenv/README.md deleted file mode 100644 index 82d68201..00000000 --- a/scripts/uberenv/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# uberenv -Automates using Spack (https://www.spack.io/) to build and deploy software. - -Uberenv is a short python script that helps automate using Spack to build -third-party dependencies for development and to deploy Spack packages. - -Uberenv was released as part of the Conduit (https://github.com/LLNL/conduit/). It is included in-source in several projects, this repo is used to hold the latest reference version. - -For more details, see Uberenv's documention: - -https://uberenv.readthedocs.io - -You can also find details about how it is used in Conduit's documentation: - -https://llnl-conduit.readthedocs.io/en/latest/building.html#building-conduit-and-third-party-dependencies - -Conduit's source repo also serves as an example for uberenv and spack configuration files, etc: - -https://github.com/LLNL/conduit/tree/master/scripts/uberenv diff --git a/scripts/uberenv/docs/sphinx/conf.py b/scripts/uberenv/docs/sphinx/conf.py deleted file mode 100644 index a8475c7b..00000000 --- a/scripts/uberenv/docs/sphinx/conf.py +++ /dev/null @@ -1,324 +0,0 @@ -# -*- coding: utf-8 -*- -# -############################################################################### -# Copyright (c) 2015-2019, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-666778 -# -# All rights reserved. -# -# This file is part of Conduit. -# -# For details, see: http://software.llnl.gov/conduit/. -# -# Please also read conduit/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### -# -# Uberenv documentation build configuration file, created by -# sphinx-quickstart on Thu Oct 16 11:23:46 2014. -# -# This file is execfile()d with the current directory set to its -# containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import sys -import os - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.doctest', - 'sphinx.ext.todo', - 'sphinx.ext.coverage', - 'sphinx.ext.mathjax' -] - -# try to add the breathe extension -try: - import breathe - extensions.append('breathe') -except: - pass - -# Add any paths that contain templates here, relative to this directory. -# templates_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Uberenv' -copyright = u'Copyright (c) 2015-2019, LLNS' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = 'current' -# The full version, including alpha/beta/rc tags. -release = 'current' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False - - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { 'logo_only' : True } - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -#html_extra_path = [] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'Uberenvdoc' - - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'Uberenv.tex', u'Uberenv Documentation', - u'LLNS', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'uberenv', u'Uberenv Documentation', - [u'LLNS'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'Uberenv', u'Uberenv Documentation', - u'LLNS', 'Uberenv', 'Automates using spack to build and deploy software.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False - - -# try to use the read the docs theme -try: - import sphinx_rtd_theme - html_theme = "sphinx_rtd_theme" - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -except: - pass diff --git a/scripts/uberenv/docs/sphinx/index.rst b/scripts/uberenv/docs/sphinx/index.rst deleted file mode 100644 index 457ec596..00000000 --- a/scripts/uberenv/docs/sphinx/index.rst +++ /dev/null @@ -1,194 +0,0 @@ -.. ############################################################################ -.. # Copyright (c) 2014-2018, Lawrence Livermore National Security, LLC. -.. # -.. # Produced at the Lawrence Livermore National Laboratory -.. # -.. # LLNL-CODE-666778 -.. # -.. # All rights reserved. -.. # -.. # This file is part of Conduit. -.. # -.. # For details, see: http://software.llnl.gov/conduit/. -.. # -.. # Please also read conduit/LICENSE -.. # -.. # Redistribution and use in source and binary forms, with or without -.. # modification, are permitted provided that the following conditions are met: -.. # -.. # * Redistributions of source code must retain the above copyright notice, -.. # this list of conditions and the disclaimer below. -.. # -.. # * Redistributions in binary form must reproduce the above copyright notice, -.. # this list of conditions and the disclaimer (as noted below) in the -.. # documentation and/or other materials provided with the distribution. -.. # -.. # * Neither the name of the LLNS/LLNL nor the names of its contributors may -.. # be used to endorse or promote products derived from this software without -.. # specific prior written permission. -.. # -.. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -.. # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.. # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.. # ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -.. # LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -.. # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.. # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.. # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.. # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -.. # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -.. # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -.. # POSSIBILITY OF SUCH DAMAGE. -.. # -.. ############################################################################ - -.. _building_with_uberenv: - -Uberenv -~~~~~~~~~~~~~~~ - -**Uberenv** automates using `Spack `_ to build and deploy software. - -Many projects leverage `Spack `_ to help build the software dependencies needed to develop and deploy their projects on HPC systems. Uberenv is a python script that helps automate using Spack to build -third-party dependencies for development and to deploy Spack packages. - -Uberenv was released as part of Conduit (https://github.com/LLNL/conduit/). It is included in-source in several projects. The -https://github.com/llnl/uberenv/ repo is used to hold the latest reference version of Uberenv. - - -uberenv.py -~~~~~~~~~~~~~~~~~~~~~ - -``uberenv.py`` is a single file python script that automates fetching Spack, building and installing third party dependencies, and can optionally install packages as well. To automate the full install process, ``uberenv.py`` uses a target Spack package along with extra settings such as Spack compiler and external third party package details for common HPC platforms. - -``uberenv.py`` is included directly in a project's source code repo in the folder: ``scripts/uberenv/`` -This folder is also used to store extra Spack and Uberenv configuration files unique to the target project. ``uberenv.py`` uses a ``project.json`` file to specify project details, including the target Spack package name and which Spack repo is used. Conduit's source repo serves as an example for Uberenv and Spack configuration files, etc: - -https://github.com/LLNL/conduit/tree/master/scripts/uberenv - - -``uberenv.py`` is developed by LLNL in support of the `Ascent `_, Axom, and `Conduit `_ projects. - - -Command Line Options -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Build configuration -------------------- - -``uberenv.py`` has a few options that allow you to control how dependencies are built: - - ======================= ============================================== ================================================ - Option Description Default - ======================= ============================================== ================================================ - ``--prefix`` Destination directory ``uberenv_libs`` - ``--spec`` Spack spec linux: **%gcc** - osx: **%clang** - ``--spack-config-dir`` Folder with Spack settings files linux: (empty) - osx: ``scripts/uberenv/spack_configs/darwin/`` - ``-k`` Ignore SSL Errors **False** - ``--install`` Fully install target, not just dependencies **False** - ``--run_tests`` Invoke tests during build and against install **False** - ``--project-json`` File for project specific settings ``project.json`` - ======================= ============================================== ================================================ - -The ``-k`` option exists for sites where SSL certificate interception undermines fetching -from github and https hosted source tarballs. When enabled, ``uberenv.py`` clones Spack using: - -.. code:: bash - - git -c http.sslVerify=false clone https://github.com/llnl/spack.git - -And passes ``-k`` to any Spack commands that may fetch via https. - - -Default invocation on Linux: - -.. code:: bash - - python scripts/uberenv/uberenv.py --prefix uberenv_libs \ - --spec %gcc - -Default invocation on OSX: - -.. code:: bash - - python scripts/uberenv/uberenv.py --prefix uberenv_libs \ - --spec %clang \ - --spack-config-dir scripts/uberenv/spack_configs/darwin/ - - -Use the ``--install`` option to install the target package (not just its development dependencies): - -.. code:: bash - - python scripts/uberenv/uberenv.py --install - - -If the target Spack package supports Spack's testing hooks, you can run tests during the build process to validate the build and install, using the ``--run_tests`` option: - -.. code:: bash - - python scripts/uberenv/uberenv.py --install \ - --run_tests - -For details on Spack's spec syntax, see the `Spack Specs & dependencies `_ documentation. - - -Uberenv looks for configuration yaml files under ``scripts/uberenv/spack_config/{platform}`` or you can use the **--spack-config-dir** option to specify a directory with compiler and packages yaml files to use with Spack. See the `Spack Compiler Configuration `_ -and `Spack System Packages -`_ -documentation for details. - -.. note:: - The bootstrapping process ignores ``~/.spack/compilers.yaml`` to avoid conflicts - and surprises from a user's specific Spack settings on HPC platforms. - -When run, ``uberenv.py`` checkouts a specific version of Spack from github as ``spack`` in the -destination directory. It then uses Spack to build and install the target packages' dependencies into -``spack/opt/spack/``. Finally, the target package generates a host-config file ``{hostname}.cmake``, which is -copied to destination directory. This file specifies the compiler settings and paths to all of the dependencies. - - -Project configuration ---------------------- - -Part of the configuration can also be addressed using a json file. By default, it is named ``project.json`` and some settings can be overridden on command line: - - ==================== ========================== ================================================ ======================================= - Setting Option Description Default - ==================== ========================== ================================================ ======================================= - package_name ``--package-name`` Spack package name **None** - package_version **None** Spack package version **None** - package_final_phase ``--package-final-phase`` Controls after which phase Spack should stop **None** - package_source_dir ``--package-source-dir`` Controls the source directory Spack should use **None** - spack_url **None** Url where to download Spack ``https://github.com/spack/spack.git`` - spack_commit **None** Spack commit to checkout **None** - spack_activate **None** Spack packages to activate **None** - ==================== ========================== ================================================ ======================================= - - -Optimization ------------- - -``uberenv.py`` also features options to optimize the installation - - ==================== ============================================== ================================================ - Option Description Default - ==================== ============================================== ================================================ - ``--mirror`` Location of a Spack mirror **None** - ``--create-mirror`` Creates a Spack mirror at specified location **None** - ``--upstream`` Location of a Spack upstream **None** - ==================== ============================================== ================================================ - - -Project Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A few notes on using ``uberenv.py`` in a new project: - -* For an example of how to craft a ``project.json`` file a target project, see: `Conduit's project.json file `_ - -* ``uberenv.py`` hot copies ``packages`` to the cloned Spack install, this allows you to easily version control any Spack package overrides necessary - - diff --git a/scripts/uberenv/gen_spack_env_script.py b/scripts/uberenv/gen_spack_env_script.py deleted file mode 100644 index a1e6ba5d..00000000 --- a/scripts/uberenv/gen_spack_env_script.py +++ /dev/null @@ -1,128 +0,0 @@ -############################################################################### -# Copyright (c) 2015-2019, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-716457 -# -# All rights reserved. -# -# This file is part of Ascent. -# -# For details, see: http://ascent.readthedocs.io/. -# -# Please also read ascent/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### -import os -import sys -import subprocess - -from os.path import join as pjoin - -# if you have bad luck with spack load, this -# script is for you! -# -# Looks for subdir: spack or uberenv_libs/spack -# queries spack for given package names and -# creates a bash script that adds those to your path -# -# -# usage: -# python gen_spack_env_script.py [spack_pkg_1 spack_pkg_2 ...] -# - -def sexe(cmd,ret_output=False,echo = True): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {}]".format(cmd)) - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - res = p.communicate()[0] - res = res.decode('utf8') - return p.returncode,res - else: - return subprocess.call(cmd,shell=True) - - -def spack_exe(spath=None): - if spath is None: - to_try = [pjoin("uberenv_libs","spack"), "spack"] - for p in to_try: - abs_p = os.path.abspath(p) - print("[looking for spack directory at: {}]".format(abs_p)) - if os.path.isdir(abs_p): - print("[FOUND spack directory at: {}]".format(abs_p)) - return os.path.abspath(pjoin(abs_p,"bin","spack")) - print("[ERROR: failed to find spack directory!]") - sys.exit(-1) - else: - spack_exe = os.path.abspath(spath,"bin","spack") - if not os.path.isfile(spack_exec): - print("[ERROR: failed to find spack directory at spath={}]").format(spath) - sys.exit(-1) - return spack_exe - -def find_pkg(pkg_name): - r,rout = sexe(spack_exe() + " find -p " + pkg_name,ret_output = True) - print(rout) - for l in rout.split("\n"): - print(l) - lstrip = l.strip() - if not lstrip == "" and \ - not lstrip.startswith("==>") and \ - not lstrip.startswith("--"): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - -def path_cmd(pkg): - return('export PATH={}:$PATH\n'.format((pjoin(pkg["path"],"bin")))) - -def write_env_script(pkgs): - ofile = open("s_env.sh","w") - for p in pkgs: - print("[found {} at {}]".format(p["name"],p["path"])) - ofile.write("# {}\n".format(p["name"])) - ofile.write(path_cmd(p)) - print("[created {}]".format(os.path.abspath("s_env.sh"))) - -def main(): - pkgs = [find_pkg(pkg) for pkg in sys.argv[1:]] - if len(pkgs) > 0: - write_env_script(pkgs) - else: - print("usage: python gen_spack_env_script.py spack_pkg_1 spack_pkg_2 ...") - -if __name__ == "__main__": - main() diff --git a/scripts/uberenv/project.json b/scripts/uberenv/project.json deleted file mode 100644 index 4cbf5809..00000000 --- a/scripts/uberenv/project.json +++ /dev/null @@ -1,9 +0,0 @@ -{ -"package_name" : "chai", -"package_version" : "develop", -"package_final_phase" : "hostconfig", -"package_source_dir" : "../..", -"spack_url": "https://github.com/spack/spack", -"spack_commit": "bc53bb9b7cbf250c98cfe77d334ed30d6b958c21", -"spack_activate" : {} -} diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/compilers.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/compilers.yaml deleted file mode 100644 index 66bc9290..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/compilers.yaml +++ /dev/null @@ -1,144 +0,0 @@ -compilers: -- compiler: - spec: clang@3.9.1 - paths: - cc: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@4.0.0 - paths: - cc: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@9.0.0 - paths: - cc: /usr/tce/packages/clang/clang-9.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-9.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@coral2018.08.08 - paths: - cc: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang - cxx: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@default - paths: - cc: clang - cxx: clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@8.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@4.9.3 - paths: - cc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@default - paths: - cc: gcc - cxx: g++ - f77: gfortran - fc: gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@default - paths: - cc: xlc - cxx: xlc++ - f77: xlf2003 - fc: xlf2003 - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@beta2019.06.20 - paths: - cc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc - cxx: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@default - paths: - cc: pgcc - cxx: pgc++ - f77: pgfortran - fc: pgfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/packages.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/packages.yaml deleted file mode 100644 index 1297fe96..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib/packages.yaml +++ /dev/null @@ -1,20 +0,0 @@ -packages: - all: - # This defaults us to machine specific flags of ivybridge which allows - # us to run on broadwell as well - target: [ppc64le] - compiler: [gcc, pgi, clang, xl] - - cmake: - version: [3.14.5] - paths: - cmake: /usr/tce/packages/cmake/cmake-3.14.5 - buildable: False - - cuda: - version: [10.1.168,9.2.148,8.0] - paths: - cuda@10.1.168: /usr/tce/packages/cuda/cuda-10.1.168 - cuda@9.2.148: /usr/tce/packages/cuda/cuda-9.2.148 - cuda@8.0: /usr/tce/packages/cuda/cuda-8.0 - buildable: False diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/compilers.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/compilers.yaml deleted file mode 100644 index 66bc9290..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/compilers.yaml +++ /dev/null @@ -1,144 +0,0 @@ -compilers: -- compiler: - spec: clang@3.9.1 - paths: - cc: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-3.9.1/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@4.0.0 - paths: - cc: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang - cxx: /usr/tcetmp/packages/clang/clang-4.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@9.0.0 - paths: - cc: /usr/tce/packages/clang/clang-9.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-9.0.0/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@coral2018.08.08 - paths: - cc: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang - cxx: /usr/tce/packages/clang/clang-coral-2018.08.08/bin/clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@default - paths: - cc: clang - cxx: clang++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@8.3.1 - paths: - cc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@4.9.3 - paths: - cc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@default - paths: - cc: gcc - cxx: g++ - f77: gfortran - fc: gfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@default - paths: - cc: xlc - cxx: xlc++ - f77: xlf2003 - fc: xlf2003 - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: xl@beta2019.06.20 - paths: - cc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc - cxx: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlc++ - f77: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - fc: /usr/tce/packages/xl/xl-beta-2019.06.20/bin/xlf2003_r - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@default - paths: - cc: pgcc - cxx: pgc++ - f77: pgfortran - fc: pgfortran - flags: {} - operating_system: rhel7 - target: ppc64le - modules: [] - environment: {} - extra_rpaths: [] diff --git a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/packages.yaml b/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/packages.yaml deleted file mode 100644 index 1297fe96..00000000 --- a/scripts/uberenv/spack_configs/blueos_3_ppc64le_ib_p9/packages.yaml +++ /dev/null @@ -1,20 +0,0 @@ -packages: - all: - # This defaults us to machine specific flags of ivybridge which allows - # us to run on broadwell as well - target: [ppc64le] - compiler: [gcc, pgi, clang, xl] - - cmake: - version: [3.14.5] - paths: - cmake: /usr/tce/packages/cmake/cmake-3.14.5 - buildable: False - - cuda: - version: [10.1.168,9.2.148,8.0] - paths: - cuda@10.1.168: /usr/tce/packages/cuda/cuda-10.1.168 - cuda@9.2.148: /usr/tce/packages/cuda/cuda-9.2.148 - cuda@8.0: /usr/tce/packages/cuda/cuda-8.0 - buildable: False diff --git a/scripts/uberenv/spack_configs/config.yaml b/scripts/uberenv/spack_configs/config.yaml deleted file mode 100644 index 2095112f..00000000 --- a/scripts/uberenv/spack_configs/config.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# ------------------------------------------------------------------------- -# This is the default spack configuration file. -# -# Settings here are versioned with Spack and are intended to provide -# sensible defaults out of the box. Spack maintainers should edit this -# file to keep it current. -# -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/config.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/config.yaml -# ------------------------------------------------------------------------- -config: - # This is the path to the root of the Spack install tree. - # You can use $spack here to refer to the root of the spack instance. - install_tree: $spack/.. - - # install directory layout - install_path_scheme: "${COMPILERNAME}-${COMPILERVER}/${PACKAGE}-${VERSION}" - -# Locations where templates should be found - template_dirs: - - $spack/templates - - # Locations where different types of modules should be installed. - module_roots: - tcl: $spack/share/spack/modules - lmod: $spack/share/spack/lmod - - - # Temporary locations Spack can try to use for builds. - # - # Spack will use the first one it finds that exists and is writable. - # You can use $tempdir to refer to the system default temp directory - # (as returned by tempfile.gettempdir()). - # - # A value of $spack/var/spack/stage indicates that Spack should run - # builds directly inside its install directory without staging them in - # temporary space. - # - # The build stage can be purged with `spack purge --stage`. - build_stage: - # skipping tempdir b/c running mpi tests fails with local fs - # - $tempdir - - $spack/../builds - - - # Cache directory already downloaded source tarballs and archived - # repositories. This can be purged with `spack purge --downloads`. - source_cache: $spack/var/spack/cache - - - # Cache directory for miscellaneous files, like the package index. - # This can be purged with `spack purge --misc-cache` - misc_cache: .spack/misccache - - - # If this is false, tools like curl that use SSL will not verify - # certifiates. (e.g., curl will use use the -k option) - verify_ssl: true - - - # If set to true, Spack will always check checksums after downloading - # archives. If false, Spack skips the checksum step. - checksum: true - - - # If set to true, `spack install` and friends will NOT clean - # potentially harmful variables from the build environment. Use wisely. - dirty: false - - - # The default number of jobs to use when running `make` in parallel. - # If set to 4, for example, `spack install` will run `make -j4`. - # If not set, all available cores are used by default. - # for uberenv, limit build_jobs to 8 - build_jobs: 8 diff --git a/scripts/uberenv/spack_configs/darwin/compilers.yaml b/scripts/uberenv/spack_configs/darwin/compilers.yaml deleted file mode 100644 index ed5cbf02..00000000 --- a/scripts/uberenv/spack_configs/darwin/compilers.yaml +++ /dev/null @@ -1,65 +0,0 @@ -compilers: -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: elcapitan - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@7.3.0-apple -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: sierra - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@8.0.0-apple - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: highsierra - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@9.0.0-apple - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: mojave - paths: - cc: /usr/bin/clang - cxx: /usr/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@10.0.0-apple - target: x86_64 -- compiler: - environment: {} - extra_rpaths: [] - flags: {} - modules: [] - operating_system: mojave - paths: - cc: /usr/local/opt/llvm/bin/clang - cxx: /usr/local/opt/llvm/bin/clang++ - f77: /usr/local/bin/gfortran - fc: /usr/local/bin/gfortran - spec: clang@10.0.0 - target: x86_64 diff --git a/scripts/uberenv/spack_configs/darwin/packages.yaml b/scripts/uberenv/spack_configs/darwin/packages.yaml deleted file mode 100644 index 6e965957..00000000 --- a/scripts/uberenv/spack_configs/darwin/packages.yaml +++ /dev/null @@ -1,25 +0,0 @@ - -# ------------------------------------------------------------------------- -# This file controls default concretization preferences for Spack. -# -# Settings here are versioned with Spack and are intended to provide -# sensible defaults out of the box. Spack maintainers should edit this -# file to keep it current. -# -# Users can override these settings by editing the following files. -# -# Per-spack-instance settings (overrides defaults): -# $SPACK_ROOT/etc/spack/packages.yaml -# -# Per-user settings (overrides default and site settings): -# ~/.spack/packages.yaml -# ------------------------------------------------------------------------- -packages: - all: - compiler: [clang] - -# cmake: -# version: [3.17.2] -# paths: -# cmake@3.17.2: /usr/local/Cellar/cmake/3.17.2 -# buildable: false diff --git a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/compilers.yaml b/scripts/uberenv/spack_configs/toss_3_x86_64_ib/compilers.yaml deleted file mode 100644 index 27b86198..00000000 --- a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/compilers.yaml +++ /dev/null @@ -1,217 +0,0 @@ -compilers: -- compiler: - spec: clang@3.9.1 - paths: - cc: /usr/tce/packages/clang/clang-3.9.1/bin/clang - cxx: /usr/tce/packages/clang/clang-3.9.1/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@4.0.0 - paths: - cc: /usr/tce/packages/clang/clang-4.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-4.0.0/bin/clang++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@6.0.0 - paths: - cc: /usr/tce/packages/clang/clang-6.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-6.0.0/bin/clang++ - f77: - fc: - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: clang@9.0.0 - paths: - cc: /usr/tce/packages/clang/clang-9.0.0/bin/clang - cxx: /usr/tce/packages/clang/clang-9.0.0/bin/clang++ - f77: - fc: - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@4.9.3 - paths: - cc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - f77: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-4.9.3/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@16.0.4 - paths: - cc: /usr/tce/packages/intel/intel-16.0.4/bin/icc - cxx: /usr/tce/packages/intel/intel-16.0.4/bin/icpc - f77: /usr/tce/packages/intel/intel-16.0.4/bin/ifort - fc: /usr/tce/packages/intel/intel-16.0.4/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@17.0.2 - paths: - cc: /usr/tce/packages/intel/intel-17.0.2/bin/icc - cxx: /usr/tce/packages/intel/intel-17.0.2/bin/icpc - f77: /usr/tce/packages/intel/intel-17.0.2/bin/ifort - fc: /usr/tce/packages/intel/intel-17.0.2/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-4.9.3/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@18.0.0 - paths: - cc: /usr/tce/packages/intel/intel-18.0.0/bin/icc - cxx: /usr/tce/packages/intel/intel-18.0.0/bin/icpc - f77: /usr/tce/packages/intel/intel-18.0.0/bin/ifort - fc: /usr/tce/packages/intel/intel-18.0.0/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: intel@19.0.4 - paths: - cc: /usr/tce/packages/intel/intel-19.0.4/bin/icc - cxx: /usr/tce/packages/intel/intel-19.0.4/bin/icpc - f77: /usr/tce/packages/intel/intel-19.0.4/bin/ifort - fc: /usr/tce/packages/intel/intel-19.0.4/bin/ifort - flags: - cflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/gcc - cxxflags: -gcc-name=/usr/tce/packages/gcc/gcc-7.1.0/bin/g++ - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@17.10 - paths: - cc: /usr/tce/packages/pgi/pgi-17.10/bin/pgcc - cxx: /usr/tce/packages/pgi/pgi-17.10/bin/pgc++ - f77: /usr/tce/packages/pgi/pgi-17.10/bin/pgf77 - fc: /usr/tce/packages/pgi/pgi-17.10/bin/pgf95 - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@18.5 - paths: - cc: /usr/tce/packages/pgi/pgi-18.5/bin/pgcc - cxx: /usr/tce/packages/pgi/pgi-18.5/bin/pgc++ - f77: /usr/tce/packages/pgi/pgi-18.5/bin/pgf77 - fc: /usr/tce/packages/pgi/pgi-18.5/bin/pgf95 - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: pgi@19.4 - paths: - cc: /usr/tce/packages/pgi/pgi-19.4/bin/pgcc - cxx: /usr/tce/packages/pgi/pgi-19.4/bin/pgc++ - f77: /usr/tce/packages/pgi/pgi-19.4/bin/pgfortran - fc: /usr/tce/packages/pgi/pgi-19.4/bin/pgfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@6.1.0 - paths: - cc: /usr/tce/packages/gcc/gcc-6.1.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-6.1.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-6.1.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-6.1.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@7.1.0 - paths: - cc: /usr/tce/packages/gcc/gcc-7.1.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-7.1.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-7.1.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-7.1.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@7.3.0 - paths: - cc: /usr/tce/packages/gcc/gcc-7.3.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-7.3.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-7.3.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-7.3.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] -- compiler: - spec: gcc@8.1.0 - paths: - cc: /usr/tce/packages/gcc/gcc-8.1.0/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.1.0/bin/g++ - f77: /usr/tce/packages/gcc/gcc-8.1.0/bin/gfortran - fc: /usr/tce/packages/gcc/gcc-8.1.0/bin/gfortran - flags: {} - operating_system: rhel7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/packages.yaml b/scripts/uberenv/spack_configs/toss_3_x86_64_ib/packages.yaml deleted file mode 100644 index c67d70f1..00000000 --- a/scripts/uberenv/spack_configs/toss_3_x86_64_ib/packages.yaml +++ /dev/null @@ -1,18 +0,0 @@ -packages: - all: - # This defaults us to machine specific flags of ivybridge which allows - # us to run on broadwell as well - target: [ivybridge] - compiler: [gcc, intel, pgi, clang] - - cmake: - version: [3.14.5] - paths: - cmake: /usr/tce/packages/cmake/cmake-3.14.5 - buildable: False - - cuda: - version: [10.1.168] - paths: - cuda@10.1.168: /usr/tce/packages/cuda/cuda-10.1.168 - buildable: False diff --git a/scripts/uberenv/uberenv.py b/scripts/uberenv/uberenv.py deleted file mode 100755 index 7761be2e..00000000 --- a/scripts/uberenv/uberenv.py +++ /dev/null @@ -1,800 +0,0 @@ -#!/bin/sh -"exec" "python" "-u" "-B" "$0" "$@" -############################################################################### -# Copyright (c) 2014-2020, Lawrence Livermore National Security, LLC. -# -# Produced at the Lawrence Livermore National Laboratory -# -# LLNL-CODE-666778 -# -# All rights reserved. -# -# This file is part of Conduit. -# -# For details, see https://lc.llnl.gov/conduit/. -# -# Please also read conduit/LICENSE -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the disclaimer below. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the disclaimer (as noted below) in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the name of the LLNS/LLNL nor the names of its contributors may -# be used to endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL SECURITY, -# LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE LIABLE FOR ANY -# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING -# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -############################################################################### - -""" - file: uberenv.py - - description: automates using spack to install a project. - -""" - -import os -import sys -import subprocess -import shutil -import socket -import platform -import json -import datetime -import glob -import re - -from optparse import OptionParser - -from os import environ as env -from os.path import join as pjoin - - -def sexe(cmd,ret_output=False,echo=False): - """ Helper for executing shell commands. """ - if echo: - print("[exe: {}]".format(cmd)) - if ret_output: - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - out = p.communicate()[0] - out = out.decode('utf8') - return p.returncode,out - else: - return subprocess.call(cmd,shell=True) - - -def parse_args(): - "Parses args from command line" - parser = OptionParser() - parser.add_option("--install", - action="store_true", - dest="install", - default=False, - help="Install `package_name`, not just its dependencies.") - - # where to install - parser.add_option("--prefix", - dest="prefix", - default="uberenv_libs", - help="destination directory") - - # what compiler to use - parser.add_option("--spec", - dest="spec", - default=None, - help="spack compiler spec") - - # optional location of spack mirror - parser.add_option("--mirror", - dest="mirror", - default=None, - help="spack mirror directory") - - # flag to create mirror - parser.add_option("--create-mirror", - action="store_true", - dest="create_mirror", - default=False, - help="Create spack mirror") - - # optional location of spack upstream - parser.add_option("--upstream", - dest="upstream", - default=None, - help="add an external spack instance as upstream") - - # this option allows a user to explicitly to select a - # group of spack settings files (compilers.yaml , packages.yaml) - parser.add_option("--spack-config-dir", - dest="spack_config_dir", - default=None, - help="dir with spack settings files (compilers.yaml, packages.yaml, etc)") - - # overrides package_name - parser.add_option("--package-name", - dest="package_name", - default=None, - help="override the default package name") - - # controls after which package phase spack should stop - parser.add_option("--package-final-phase", - dest="package_final_phase", - default=None, - help="override the default phase after which spack should stop") - - # controls source_dir spack should use to build the package - parser.add_option("--package-source-dir", - dest="package_source_dir", - default=None, - help="override the default source dir spack should use") - - # a file that holds settings for a specific project - # using uberenv.py - parser.add_option("--project-json", - dest="project_json", - default=pjoin(uberenv_script_dir(),"project.json"), - help="uberenv project settings json file") - - # flag to use insecure curl + git - parser.add_option("-k", - action="store_true", - dest="ignore_ssl_errors", - default=False, - help="Ignore SSL Errors") - - # option to force a spack pull - parser.add_option("--pull", - action="store_true", - dest="spack_pull", - default=False, - help="Pull if spack repo already exists") - - # option to force for clean of packages specified to - # be cleaned in the project.json - parser.add_option("--clean", - action="store_true", - dest="spack_clean", - default=False, - help="Force uninstall of packages specified in project.json") - - # option to tell spack to run tests - parser.add_option("--run_tests", - action="store_true", - dest="run_tests", - default=False, - help="Invoke build tests during spack install") - - # option to init osx sdk env flags - parser.add_option("--macos-sdk-env-setup", - action="store_true", - dest="macos_sdk_env_setup", - default=False, - help="Set several env vars to select OSX SDK settings." - "This was necessary for older versions of macOS " - " but can cause issues with macOS versions >= 10.13. " - " so it is disabled by default.") - - - ############### - # parse args - ############### - opts, extras = parser.parse_args() - # we want a dict b/c the values could - # be passed without using optparse - opts = vars(opts) - if not opts["spack_config_dir"] is None: - opts["spack_config_dir"] = os.path.abspath(opts["spack_config_dir"]) - if not os.path.isdir(opts["spack_config_dir"]): - print("[ERROR: invalid spack config dir: {} ]".format(opts["spack_config_dir"])) - sys.exit(-1) - # if rel path is given for the mirror, we need to evaluate here -- before any - # chdirs to avoid confusion related to what it is relative to. - # (it should be relative to where uberenv is run from, so it matches what you expect - # from shell completion, etc) - if not opts["mirror"] is None: - if not opts["mirror"].startswith("http") and not os.path.isabs(opts["mirror"]): - opts["mirror"] = os.path.abspath(opts["mirror"]) - return opts, extras - - -def uberenv_script_dir(): - # returns the directory of the uberenv.py script - return os.path.dirname(os.path.abspath(__file__)) - -def load_json_file(json_file): - # reads json file - return json.load(open(json_file)) - -def is_darwin(): - return "darwin" in platform.system().lower() - -def is_windows(): - return "windows" in platform.system().lower() - -class UberEnv(): - """ Base class for package manager """ - - def __init__(self, opts, extra_opts): - self.opts = opts - self.extra_opts = extra_opts - - # load project settings - self.project_opts = load_json_file(opts["project_json"]) - print("[uberenv project settings: {}]".format(str(self.project_opts))) - print("[uberenv options: {}]".format(str(self.opts))) - - def setup_paths_and_dirs(self): - self.uberenv_path = os.path.dirname(os.path.realpath(__file__)) - - def set_from_args_or_json(self,setting): - try: - setting_value = self.project_opts[setting] - except (KeyError): - print("ERROR: {} must at least be defined in project.json".format(setting)) - raise - else: - if self.opts[setting]: - setting_value = self.opts[setting] - return setting_value - - def set_from_json(self,setting): - try: - setting_value = self.project_opts[setting] - except (KeyError): - print("ERROR: {} must at least be defined in project.json".format(setting)) - raise - return setting_value - - def detect_platform(self): - # find supported sets of compilers.yaml, packages,yaml - res = None - if is_darwin(): - res = "darwin" - elif "SYS_TYPE" in os.environ.keys(): - sys_type = os.environ["SYS_TYPE"].lower() - res = sys_type - return res - - -class SpackEnv(UberEnv): - """ Helper to clone spack and install libraries on MacOS an Linux """ - - def __init__(self, opts, extra_opts): - UberEnv.__init__(self,opts,extra_opts) - - self.pkg_name = self.set_from_args_or_json("package_name") - self.pkg_version = self.set_from_json("package_version") - self.pkg_final_phase = self.set_from_args_or_json("package_final_phase") - self.pkg_src_dir = self.set_from_args_or_json("package_source_dir") - - self.spec_hash = "" - self.use_install = False - - # Some additional setup for macos - if is_darwin(): - if opts["macos_sdk_env_setup"]: - # setup osx deployment target and sdk settings - setup_osx_sdk_env_vars() - else: - print("[skipping MACOSX env var setup]") - - # setup default spec - if opts["spec"] is None: - if is_darwin(): - opts["spec"] = "%clang" - else: - opts["spec"] = "%gcc" - self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) - elif not opts["spec"].startswith("@"): - self.opts["spec"] = "@{}{}".format(self.pkg_version,opts["spec"]) - else: - self.opts["spec"] = "{}".format(opts["spec"]) - - print("[spack spec: {}]".format(self.opts["spec"])) - - def setup_paths_and_dirs(self): - # get the current working path, and the glob used to identify the - # package files we want to hot-copy to spack - - UberEnv.setup_paths_and_dirs(self) - - self.pkgs = pjoin(self.uberenv_path, "packages","*") - - # setup destination paths - self.dest_dir = os.path.abspath(self.opts["prefix"]) - self.dest_spack = pjoin(self.dest_dir,"spack") - print("[installing to: {0}]".format(self.dest_dir)) - - # print a warning if the dest path already exists - if not os.path.isdir(self.dest_dir): - os.mkdir(self.dest_dir) - else: - print("[info: destination '{}' already exists]".format(self.dest_dir)) - - if os.path.isdir(self.dest_spack): - print("[info: destination '{}' already exists]".format(self.dest_spack)) - - self.pkg_src_dir = os.path.join(self.uberenv_path,self.pkg_src_dir) - if not os.path.isdir(self.pkg_src_dir): - print("[ERROR: package_source_dir '{}' does not exist]".format(self.pkg_src_dir)) - sys.exit(-1) - - - def find_spack_pkg_path_from_hash(self, pkg_name, pkg_hash): - res, out = sexe("spack/bin/spack find -p /{}".format(pkg_hash), ret_output = True) - for l in out.split("\n"): - if l.startswith(pkg_name): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - - def find_spack_pkg_path(self, pkg_name, spec = ""): - res, out = sexe("spack/bin/spack find -p " + pkg_name + spec,ret_output = True) - for l in out.split("\n"): - # TODO: at least print a warning when several choices exist. This will - # pick the first in the list. - if l.startswith(pkg_name): - return {"name": pkg_name, "path": l.split()[-1]} - print("[ERROR: failed to find package named '{}']".format(pkg_name)) - sys.exit(-1) - - # Extract the first line of the full spec - def read_spack_full_spec(self,pkg_name,spec): - res, out = sexe("spack/bin/spack spec " + pkg_name + " " + spec, ret_output=True) - for l in out.split("\n"): - if l.startswith(pkg_name) and l.count("@") > 0 and l.count("arch=") > 0: - return l.strip() - - def clone_repo(self): - if not os.path.isdir(self.dest_spack): - - # compose clone command for the dest path, spack url and branch - print("[info: cloning spack develop branch from github]") - - os.chdir(self.dest_dir) - - clone_opts = ("-c http.sslVerify=false " - if self.opts["ignore_ssl_errors"] else "") - - spack_url = self.project_opts.get("spack_url", "https://github.com/spack/spack.git") - spack_branch = self.project_opts.get("spack_branch", "develop") - - clone_cmd = "git {0} clone --single-branch --depth=1 -b {1} {2}".format(clone_opts, spack_branch,spack_url) - sexe(clone_cmd, echo=True) - - if "spack_commit" in self.project_opts: - # optionally, check out a specific commit - os.chdir(pjoin(self.dest_dir,"spack")) - sha1 = self.project_opts["spack_commit"] - res, current_sha1 = sexe("git log -1 --pretty=%H", ret_output=True) - if sha1 != current_sha1: - print("[info: using spack commit {}]".format(sha1)) - sexe("git stash", echo=True) - sexe("git fetch --depth=1 origin {0}".format(sha1),echo=True) - sexe("git checkout {0}".format(sha1),echo=True) - - if self.opts["spack_pull"]: - # do a pull to make sure we have the latest - os.chdir(pjoin(self.dest_dir,"spack")) - sexe("git stash", echo=True) - sexe("git pull", echo=True) - - def config_dir(self): - """ path to compilers.yaml, which we will use for spack's compiler setup""" - spack_config_dir = self.opts["spack_config_dir"] - if spack_config_dir is None: - uberenv_plat = self.detect_platform() - if not uberenv_plat is None: - spack_config_dir = os.path.abspath(pjoin(self.uberenv_path,"spack_configs",uberenv_plat)) - return spack_config_dir - - - def disable_spack_config_scopes(self,spack_dir): - # disables all config scopes except "defaults", which we will - # force our settings into - spack_lib_config = pjoin(spack_dir,"lib","spack","spack","config.py") - print("[disabling config scope (except defaults) in: {}]".format(spack_lib_config)) - cfg_script = open(spack_lib_config).read() - for cfg_scope_stmt in ["('system', os.path.join(spack.paths.system_etc_path, 'spack')),", - "('site', os.path.join(spack.paths.etc_path, 'spack')),", - "('user', spack.paths.user_config_path)"]: - cfg_script = cfg_script.replace(cfg_scope_stmt, - "#DISABLED BY UBERENV: " + cfg_scope_stmt) - open(spack_lib_config,"w").write(cfg_script) - - - def patch(self): - - cfg_dir = self.config_dir() - spack_dir = self.dest_spack - - # force spack to use only "defaults" config scope - self.disable_spack_config_scopes(spack_dir) - spack_etc_defaults_dir = pjoin(spack_dir,"etc","spack","defaults") - - # copy in "defaults" config.yaml - config_yaml = os.path.abspath(pjoin(self.uberenv_path,"spack_configs","config.yaml")) - sexe("cp {} {}/".format(config_yaml, spack_etc_defaults_dir ), echo=True) - - # copy in other settings per platform - if not cfg_dir is None: - print("[copying uberenv compiler and packages settings from {0}]".format(cfg_dir)) - - config_yaml = pjoin(cfg_dir,"config.yaml") - compilers_yaml = pjoin(cfg_dir,"compilers.yaml") - packages_yaml = pjoin(cfg_dir,"packages.yaml") - - if os.path.isfile(config_yaml): - sexe("cp {} {}/".format(config_yaml , spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(compilers_yaml): - sexe("cp {} {}/".format(compilers_yaml, spack_etc_defaults_dir ), echo=True) - - if os.path.isfile(packages_yaml): - sexe("cp {} {}/".format(packages_yaml, spack_etc_defaults_dir ), echo=True) - else: - # let spack try to auto find compilers - sexe("spack/bin/spack compiler find", echo=True) - - # hot-copy our packages into spack - if self.pkgs: - dest_spack_pkgs = pjoin(spack_dir,"var","spack","repos","builtin","packages") - print("[copying patched packages from {0}]".format(self.pkgs)) - sexe("cp -Rf {} {}".format(self.pkgs,dest_spack_pkgs)) - - - def clean_build(self): - # clean out any temporary spack build stages - cln_cmd = "spack/bin/spack clean " - res = sexe(cln_cmd, echo=True) - - # clean out any spack cached stuff - cln_cmd = "spack/bin/spack clean --all" - res = sexe(cln_cmd, echo=True) - - # check if we need to force uninstall of selected packages - if self.opts["spack_clean"]: - if self.project_opts.has_key("spack_clean_packages"): - for cln_pkg in self.project_opts["spack_clean_packages"]: - if not self.find_spack_pkg_path(cln_pkg) is None: - unist_cmd = "spack/bin/spack uninstall -f -y --all --dependents " + cln_pkg - res = sexe(unist_cmd, echo=True) - - def show_info(self): - # prints install status and 32 characters hash - options="--install-status --very-long" - spec_cmd = "spack/bin/spack spec {0} {1}{2}".format(options,self.pkg_name,self.opts["spec"]) - - res, out = sexe(spec_cmd, ret_output=True, echo=True) - print(out) - - #Check if spec is already installed - for line in out.split("\n"): - # Example of matching line: ("status" "hash" "package"...) - # [+] hf3cubkgl74ryc3qwen73kl4yfh2ijgd serac@develop%clang@10.0.0-apple~debug~devtools~glvis arch=darwin-mojave-x86_64 - if re.match(r"^(\[\+\]| - ) [a-z0-9]{32} " + re.escape(self.pkg_name), line): - self.spec_hash = line.split(" ")[1] - # if spec already installed - if line.startswith("[+]"): - pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name,self.spec_hash) - install_path = pkg_path["path"] - # testing that the path exists is mandatory until Spack team fixes - # https://github.com/spack/spack/issues/16329 - if os.path.isdir(install_path): - print("[Warning: {} {} has already been installed in {}]".format(self.pkg_name, self.opts["spec"],install_path)) - print("[Warning: Uberenv will proceed using this directory]".format(self.pkg_name)) - self.use_install = True - - return res - - def install(self): - # use the uberenv package to trigger the right builds - # and build an host-config.cmake file - - if not self.use_install: - install_cmd = "spack/bin/spack " - if self.opts["ignore_ssl_errors"]: - install_cmd += "-k " - if not self.opts["install"]: - install_cmd += "dev-build --quiet -d {} -u {} ".format(self.pkg_src_dir,self.pkg_final_phase) - else: - install_cmd += "install " - if self.opts["run_tests"]: - install_cmd += "--test=root " - install_cmd += self.pkg_name + self.opts["spec"] - res = sexe(install_cmd, echo=True) - - if res != 0: - print("[ERROR: failure of spack install/dev-build]") - return res - - full_spec = self.read_spack_full_spec(self.pkg_name,self.opts["spec"]) - if "spack_activate" in self.project_opts: - print("[activating dependent packages]") - # get the full spack spec for our project - pkg_names = self.project_opts["spack_activate"].keys() - for pkg_name in pkg_names: - pkg_spec_requirements = self.project_opts["spack_activate"][pkg_name] - activate=True - for req in pkg_spec_requirements: - if req not in full_spec: - activate=False - break - if activate: - activate_cmd = "spack/bin/spack activate " + pkg_name - sexe(activate_cmd, echo=True) - # note: this assumes package extends python when +python - # this may fail general cases - if self.opts["install"] and "+python" in full_spec: - activate_cmd = "spack/bin/spack activate /" + self.spec_hash - sexe(activate_cmd, echo=True) - # if user opt'd for an install, we want to symlink the final - # install to an easy place: - if self.opts["install"] or self.use_install: - pkg_path = self.find_spack_pkg_path_from_hash(self.pkg_name, self.spec_hash) - if self.pkg_name != pkg_path["name"]: - print("[ERROR: Could not find install of {}]".format(self.pkg_name)) - return -1 - else: - # Symlink host-config file - hc_glob = glob.glob(pjoin(pkg_path["path"],"*.cmake")) - if len(hc_glob) > 0: - hc_path = hc_glob[0] - hc_fname = os.path.split(hc_path)[1] - if os.path.islink(hc_fname): - os.unlink(hc_fname) - elif os.path.isfile(hc_fname): - sexe("rm -f {}".format(hc_fname)) - print("[symlinking host config file to {}]".format(pjoin(self.dest_dir,hc_fname))) - os.symlink(hc_path,hc_fname) - - # Symlink install directory - if self.opts["install"]: - pkg_lnk_dir = "{}-install".format(self.pkg_name) - if os.path.islink(pkg_lnk_dir): - os.unlink(pkg_lnk_dir) - print("") - print("[symlinking install to {}]".format(pjoin(self.dest_dir,pkg_lnk_dir))) - os.symlink(pkg_path["path"],os.path.abspath(pkg_lnk_dir)) - print("") - print("[install complete!]") - # otherwise we are in the "only dependencies" case and the host-config - # file has to be copied from the do-be-deleted spack-build dir. - else: - pattern = "*{}.cmake".format(self.pkg_name) - build_dir = pjoin(self.pkg_src_dir,"spack-build") - hc_glob = glob.glob(pjoin(build_dir,pattern)) - if len(hc_glob) > 0: - hc_path = hc_glob[0] - hc_fname = os.path.split(hc_path)[1] - if os.path.islink(hc_fname): - os.unlink(hc_fname) - print("[copying host config file to {}]".format(pjoin(self.dest_dir,hc_fname))) - sexe("cp {} {}".format(hc_path,hc_fname)) - print("[removing project build directory {}]".format(pjoin(build_dir))) - sexe("rm -rf {}".format(build_dir)) - - def get_mirror_path(self): - mirror_path = self.opts["mirror"] - if not mirror_path: - print("[--create-mirror requires a mirror directory]") - sys.exit(-1) - return mirror_path - - def create_mirror(self): - """ - Creates a spack mirror for pkg_name at mirror_path. - """ - - mirror_path = self.get_mirror_path() - - mirror_cmd = "spack/bin/spack " - if self.opts["ignore_ssl_errors"]: - mirror_cmd += "-k " - mirror_cmd += "mirror create -d {} --dependencies {}{}".format(mirror_path, - self.pkg_name, - self.opts["spec"]) - return sexe(mirror_cmd, echo=True) - - def find_spack_mirror(self, mirror_name): - """ - Returns the path of a defaults scoped spack mirror with the - given name, or None if no mirror exists. - """ - res, out = sexe("spack/bin/spack mirror list", ret_output=True) - mirror_path = None - for mirror in out.split('\n'): - if mirror: - parts = mirror.split() - if parts[0] == mirror_name: - mirror_path = parts[1] - return mirror_path - - def use_mirror(self): - """ - Configures spack to use mirror at a given path. - """ - mirror_name = self.pkg_name - mirror_path = self.get_mirror_path() - existing_mirror_path = self.find_spack_mirror(mirror_name) - - if existing_mirror_path and mirror_path != existing_mirror_path: - # Existing mirror has different URL, error out - print("[removing existing spack mirror `{}` @ {}]".format(mirror_name, - existing_mirror_path)) - # - # Note: In this case, spack says it removes the mirror, but we still - # get errors when we try to add a new one, sounds like a bug - # - sexe("spack/bin/spack mirror remove --scope=defaults {} ".format(mirror_name), - echo=True) - existing_mirror_path = None - if not existing_mirror_path: - # Add if not already there - sexe("spack/bin/spack mirror add --scope=defaults {} {}".format( - mirror_name, mirror_path), echo=True) - print("[using mirror {}]".format(mirror_path)) - - def find_spack_upstream(self, upstream_name): - """ - Returns the path of a defaults scoped spack upstream with the - given name, or None if no upstream exists. - """ - upstream_path = None - - res, out = sexe('spack/bin/spack config get upstreams', ret_output=True) - if (not out) and ("upstreams:" in out): - out = out.replace(' ', '') - out = out.replace('install_tree:', '') - out = out.replace(':', '') - out = out.splitlines() - out = out[1:] - upstreams = dict(zip(out[::2], out[1::2])) - - for name in upstreams.keys(): - if name == upstream_name: - upstream_path = upstreams[name] - - return upstream_path - - def use_spack_upstream(self): - """ - Configures spack to use upstream at a given path. - """ - upstream_path = self.opts["upstream"] - if not upstream_path: - print("[--create-upstream requires a upstream directory]") - sys.exit(-1) - upstream_path = os.path.abspath(upstream_path) - upstream_name = self.pkg_name - existing_upstream_path = self.find_spack_upstream(upstream_name) - if (not existing_upstream_path) or (upstream_path != os.path.abspath(existing_upstream_path)): - # Existing upstream has different URL, error out - print("[removing existing spack upstream configuration file]") - sexe("rm spack/etc/spack/defaults/upstreams.yaml") - with open('spack/etc/spack/defaults/upstreams.yaml','w+') as upstreams_cfg_file: - upstreams_cfg_file.write("upstreams:\n") - upstreams_cfg_file.write(" {}:\n".format(upstream_name)) - upstreams_cfg_file.write(" install_tree: {}\n".format(upstream_path)) - - -def find_osx_sdks(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - res = {} - sdks = glob.glob("/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk") - for sdk in sdks: - sdk_base = os.path.split(sdk)[1] - ver = sdk_base[len("MacOSX"):sdk_base.rfind(".")] - res[ver] = sdk - return res - -def setup_osx_sdk_env_vars(): - """ - Finds installed osx sdks, returns dict mapping version to file system path - """ - # find current osx version (10.11.6) - dep_tgt = platform.mac_ver()[0] - # sdk file names use short version (ex: 10.11) - dep_tgt_short = dep_tgt[:dep_tgt.rfind(".")] - # find installed sdks, ideally we want the sdk that matches the current os - sdk_root = None - sdks = find_osx_sdks() - if dep_tgt_short in sdks.keys(): - # matches our osx, use this one - sdk_root = sdks[dep_tgt_short] - elif len(sdks) > 0: - # for now, choose first one: - dep_tgt = sdks.keys()[0] - sdk_root = sdks[dep_tgt] - else: - # no valid sdks, error out - print("[ERROR: Could not find OSX SDK @ /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/]") - sys.exit(-1) - - env["MACOSX_DEPLOYMENT_TARGET"] = dep_tgt - env["SDKROOT"] = sdk_root - print("[setting MACOSX_DEPLOYMENT_TARGET to {}]".format(env["MACOSX_DEPLOYMENT_TARGET"])) - print("[setting SDKROOT to {}]".format(env[ "SDKROOT"])) - - - -def main(): - """ - Clones and runs a package manager to setup third_party libs. - Also creates a host-config.cmake file that can be used by our project. - """ - - # parse args from command line - opts, extra_opts = parse_args() - - # Initialize the environment - env = SpackEnv(opts, extra_opts) - - # Setup the necessary paths and directories - env.setup_paths_and_dirs() - - # Clone the package manager - env.clone_repo() - - os.chdir(env.dest_dir) - - # Patch the package manager, as necessary - env.patch() - - # Clean the build - env.clean_build() - - # Show the spec for what will be built - env.show_info() - - - ########################################################## - # we now have an instance of spack configured how we - # need it to build our tpls at this point there are two - # possible next steps: - # - # *) create a mirror of the packages - # OR - # *) build - # - ########################################################## - if opts["create_mirror"]: - return env.create_mirror() - else: - if not opts["mirror"] is None: - env.use_mirror() - - if not opts["upstream"] is None: - env.use_spack_upstream() - - res = env.install() - - return res - -if __name__ == "__main__": - sys.exit(main()) - - diff --git a/src/chai/ArrayManager.cpp b/src/chai/ArrayManager.cpp index dce8ad96..8a07c7cc 100644 --- a/src/chai/ArrayManager.cpp +++ b/src/chai/ArrayManager.cpp @@ -308,11 +308,13 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre alloc.deallocate(space_ptr); for (int space_t = CPU; space_t < NUM_EXECUTION_SPACES; ++space_t) { - if (space_ptr == pointer_record->m_pointers[space_t]) + if (space_ptr == pointer_record->m_pointers[space_t]) { pointer_record->m_pointers[space_t] = nullptr; + } } - } else { -#elif defined(CHAI_ENABLE_PINNED) + } else +#endif +#if defined(CHAI_ENABLE_PINNED) if (space_ptr == pointer_record->m_pointers[PINNED]) { callback(pointer_record, ACTION_FREE, @@ -323,11 +325,13 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre alloc.deallocate(space_ptr); for (int space_t = CPU; space_t < NUM_EXECUTION_SPACES; ++space_t) { - if (space_ptr == pointer_record->m_pointers[space_t]) + if (space_ptr == pointer_record->m_pointers[space_t]) { pointer_record->m_pointers[space_t] = nullptr; + } } - } else { + } else #endif + { callback(pointer_record, ACTION_FREE, ExecutionSpace(space)); @@ -337,9 +341,7 @@ void ArrayManager::free(PointerRecord* pointer_record, ExecutionSpace spaceToFre alloc.deallocate(space_ptr); pointer_record->m_pointers[space] = nullptr; -#if defined(CHAI_ENABLE_UM) || defined(CHAI_ENABLE_PINNED) } -#endif } else { diff --git a/src/chai/ManagedArray.hpp b/src/chai/ManagedArray.hpp index 2a9057fe..e8c10ca3 100644 --- a/src/chai/ManagedArray.hpp +++ b/src/chai/ManagedArray.hpp @@ -240,6 +240,26 @@ class ManagedArray : public CHAICopyable */ CHAI_HOST T* getPointer(ExecutionSpace space, bool do_move = true) const; + /*! + * \brief Move data to the current execution space (actually determined + * by where the code is executing) and return an iterator to the + * beginning of the array. + * + * \return Iterator (as raw pointer) to the start of the array in the + * current execution space + */ + CHAI_HOST_DEVICE T* begin() const; + + /*! + * \brief Move data to the current execution space (actually determined + * by where the code is executing) and return an iterator to + * one past the end of the array. + * + * \return Iterator (as raw pointer) to the element after the last element + * of the array in the current execution space + */ + CHAI_HOST_DEVICE T* end() const; + /*! * \brief * diff --git a/src/chai/ManagedArray.inl b/src/chai/ManagedArray.inl index 33db39d0..99ed2f44 100644 --- a/src/chai/ManagedArray.inl +++ b/src/chai/ManagedArray.inl @@ -39,6 +39,10 @@ CHAI_HOST_DEVICE ManagedArray::ManagedArray( #if !defined(CHAI_DEVICE_COMPILE) m_pointer_record = new PointerRecord(); int i = 0; + for (int s = CPU; s < NUM_EXECUTION_SPACES; ++s) { + m_pointer_record->m_allocators[s] = m_resource_manager->getAllocatorId(ExecutionSpace(s)); + } + for (const auto& space : spaces) { m_pointer_record->m_allocators[space] = allocators.begin()[i++].getId(); } @@ -253,7 +257,7 @@ template CHAI_INLINE CHAI_HOST void ManagedArray::free(ExecutionSpace space) { - if(!m_is_slice && *this != nullptr) { + if(!m_is_slice) { if (m_resource_manager == nullptr) { m_resource_manager = ArrayManager::getInstance(); } @@ -548,6 +552,18 @@ T* ManagedArray::getPointer(ExecutionSpace space, bool do_move) const { return data(space, do_move); } +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::begin() const { + return data(); +} + +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::end() const { + return data() + size(); +} + //template //ManagedArray::operator ManagedArray< // typename std::conditional::value, diff --git a/src/chai/ManagedArrayView.hpp b/src/chai/ManagedArrayView.hpp index 16858f17..85ee8407 100644 --- a/src/chai/ManagedArrayView.hpp +++ b/src/chai/ManagedArrayView.hpp @@ -23,10 +23,10 @@ using ManagedArrayView = template -using TypedManagedArrayView = RAJA::TypedViewBase, LayoutType, - IndexTypes...>; + camp::list >; template using ManagedArrayMultiView = diff --git a/src/chai/ManagedArray_thin.inl b/src/chai/ManagedArray_thin.inl index 5e3afffc..586c7245 100644 --- a/src/chai/ManagedArray_thin.inl +++ b/src/chai/ManagedArray_thin.inl @@ -136,26 +136,45 @@ T* ManagedArray::getPointer(ExecutionSpace space, bool do_move) const return data(space, do_move); } +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::begin() const { + return data(); +} + +template +CHAI_INLINE +CHAI_HOST_DEVICE T* ManagedArray::end() const { + return data() + size(); +} + template CHAI_INLINE CHAI_HOST void ManagedArray::allocate(size_t elems, ExecutionSpace space, UserCallback const &) { if (!m_is_slice) { - (void) space; // Quiet compiler warning when CHAI_LOG does nothing - CHAI_LOG(Debug, "Allocating array of size " << elems - << " in space " - << space); - - m_elems = elems; - - #if defined(CHAI_ENABLE_UM) - gpuMallocManaged(&m_active_pointer, sizeof(T) * elems); - #else // not CHAI_ENABLE_UM - m_active_pointer = static_cast(malloc(sizeof(T) * elems)); - #endif - - CHAI_LOG(Debug, "m_active_ptr allocated at address: " << m_active_pointer); + if (elems > 0) { + (void) space; // Quiet compiler warning when CHAI_LOG does nothing + CHAI_LOG(Debug, "Allocating array of size " << elems + << " in space " + << space); + + m_elems = elems; + + #if defined(CHAI_ENABLE_UM) + gpuMallocManaged(&m_active_pointer, sizeof(T) * elems); + #else // not CHAI_ENABLE_UM + m_active_pointer = static_cast(malloc(sizeof(T) * elems)); + #endif + + CHAI_LOG(Debug, "m_active_ptr allocated at address: " << m_active_pointer); + + } + else { + m_active_pointer = nullptr; + m_elems = 0; + } } else { CHAI_LOG(Debug, "Attempted to allocate slice!"); diff --git a/src/chai/RajaExecutionSpacePlugin.cpp b/src/chai/RajaExecutionSpacePlugin.cpp index e5f4027f..e914d7f3 100644 --- a/src/chai/RajaExecutionSpacePlugin.cpp +++ b/src/chai/RajaExecutionSpacePlugin.cpp @@ -29,6 +29,10 @@ RajaExecutionSpacePlugin::preCapture(const RAJA::util::PluginContext& p) #if defined(CHAI_ENABLE_CUDA) case RAJA::Platform::cuda: m_arraymanager->setExecutionSpace(chai::GPU); break; +#endif +#if defined(CHAI_ENABLE_HIP) + case RAJA::Platform::hip: + m_arraymanager->setExecutionSpace(chai::GPU); break; #endif default: m_arraymanager->setExecutionSpace(chai::NONE); diff --git a/src/chai/config.hpp.in b/src/chai/config.hpp.in index 26961f34..8325184c 100644 --- a/src/chai/config.hpp.in +++ b/src/chai/config.hpp.in @@ -15,6 +15,7 @@ #cmakedefine CHAI_ENABLE_UM #cmakedefine CHAI_DEBUG #cmakedefine CHAI_ENABLE_GPU_ERROR_CHECKING +#cmakedefine CHAI_ENABLE_MANAGED_PTR #cmakedefine CHAI_ENABLE_RAJA_PLUGIN #cmakedefine CHAI_ENABLE_GPU_SIMULATION_MODE #cmakedefine CHAI_ENABLE_PINNED diff --git a/src/chai/managed_ptr.hpp b/src/chai/managed_ptr.hpp index 1afcf9c2..55a0f122 100644 --- a/src/chai/managed_ptr.hpp +++ b/src/chai/managed_ptr.hpp @@ -9,6 +9,8 @@ #include "chai/config.hpp" +#if defined(CHAI_ENABLE_MANAGED_PTR) + #ifndef CHAI_DISABLE_RM #include "chai/ArrayManager.hpp" #endif @@ -1318,5 +1320,12 @@ namespace chai { } } // namespace chai +#else // defined(CHAI_ENABLE_MANAGED_PTR) + +#error CHAI must be configured with -DCHAI_ENABLE_MANAGED_PTR=ON to use managed_ptr! \ + If CHAI_ENABLE_MANAGED_PTR is defined as a macro, it is safe to include managed_ptr.hpp. + +#endif // defined(CHAI_ENABLE_MANAGED_PTR) + #endif // MANAGED_PTR diff --git a/src/tpl/raja b/src/tpl/raja index 3047fa72..357933a4 160000 --- a/src/tpl/raja +++ b/src/tpl/raja @@ -1 +1 @@ -Subproject commit 3047fa720132d19ee143b1fcdacaa72971f5988c +Subproject commit 357933a42842dd91de5c1034204d937fce0a2a44 diff --git a/src/tpl/umpire b/src/tpl/umpire index 447f4640..5f886b42 160000 --- a/src/tpl/umpire +++ b/src/tpl/umpire @@ -1 +1 @@ -Subproject commit 447f4640eff7b8f39d3c59404f3b03629b90c021 +Subproject commit 5f886b4299496b7cb6f9d62dc1372ce6d3832fbc diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 6825dfeb..b422be2b 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -25,18 +25,20 @@ blt_add_test( NAME managed_array_test COMMAND managed_array_tests) -blt_add_executable( - NAME managed_ptr_tests - SOURCES managed_ptr_tests.cpp - DEPENDS_ON ${chai_integration_test_depends}) +if (CHAI_ENABLE_MANAGED_PTR) + blt_add_executable( + NAME managed_ptr_tests + SOURCES managed_ptr_tests.cpp + DEPENDS_ON ${chai_integration_test_depends}) -target_include_directories( - managed_ptr_tests - PUBLIC ${PROJECT_BINARY_DIR}/include) + target_include_directories( + managed_ptr_tests + PUBLIC ${PROJECT_BINARY_DIR}/include) -blt_add_test( - NAME managed_ptr_test - COMMAND managed_ptr_tests) + blt_add_test( + NAME managed_ptr_test + COMMAND managed_ptr_tests) +endif () if (ENABLE_RAJA_PLUGIN) set(raja_test_depends diff --git a/tests/integration/managed_array_tests.cpp b/tests/integration/managed_array_tests.cpp index dc0fdb23..32a03217 100644 --- a/tests/integration/managed_array_tests.cpp +++ b/tests/integration/managed_array_tests.cpp @@ -29,6 +29,8 @@ #include "chai/ManagedArray.hpp" +#include "umpire/ResourceManager.hpp" + struct my_point { double x; @@ -1130,6 +1132,40 @@ GPU_TEST(ManagedArray, cdataGPU) #endif #endif +TEST(ManagedArray, Iterators) +{ + int length = 10; + chai::ManagedArray array(length); + + forall(sequential(), 0, length, [=] (int i) { + array[i] = i; + }); + + // Make sure the iterator distance is the size of the array + EXPECT_EQ(std::distance(array.begin(), array.end()), length); + + // Double each element with a range-based for loop + for (int& val : array) + { + val *= 2; + } + + // Double each element again with an + std::for_each(array.begin(), array.end(), [](int& val) { val *= 2; }); + + // Make sure a reference to a const array can be iterated over + const chai::ManagedArray& const_array = array; + int i = 0; + for (const int val : const_array) + { + EXPECT_EQ(val, i * 4); + i++; + } + + array.free(); + assert_empty_map(true); +} + TEST(ManagedArray, Reset) { chai::ManagedArray array(20); @@ -1941,6 +1977,8 @@ TEST(ManagedArray, NoAllocation) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } TEST(ManagedArray, NoAllocationNull) @@ -1953,6 +1991,8 @@ TEST(ManagedArray, NoAllocationNull) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } #if defined(CHAI_ENABLE_CUDA) || defined(CHAI_ENABLE_HIP) @@ -1965,6 +2005,25 @@ GPU_TEST(ManagedArray, NoAllocationGPU) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); +} + +GPU_TEST(ManagedArray, NoAllocationGPUList) +{ + auto& rm = umpire::ResourceManager::getInstance(); + chai::ManagedArray array(10, + std::initializer_list{chai::CPU}, + std::initializer_list{rm.getAllocator("HOST")} + ); + + forall(gpu(), 0, 10, [=] __device__ (int i) { + array[i] = i; + }); + + forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } GPU_TEST(ManagedArray, NoAllocationNullGPU) @@ -1977,5 +2036,7 @@ GPU_TEST(ManagedArray, NoAllocationNullGPU) }); forall(sequential(), 0, 10, [=](int i) { ASSERT_EQ(array[i], i); }); + + array.free(); } #endif diff --git a/tests/integration/managed_ptr_tests.cpp b/tests/integration/managed_ptr_tests.cpp index 441bc638..55a51e9d 100644 --- a/tests/integration/managed_ptr_tests.cpp +++ b/tests/integration/managed_ptr_tests.cpp @@ -12,6 +12,7 @@ static void gpu_test_##X##Y() #include "chai/config.hpp" +#include "chai/ArrayManager.hpp" #include "chai/ManagedArray.hpp" #include "chai/managed_ptr.hpp" @@ -326,7 +327,7 @@ TEST(managed_ptr, managed_array_of_managed_ptr) delete[] expectedValues; } -#ifdef __CUDACC__ +#ifdef CHAI_GPUCC template __global__ void deviceNew(T** arr) { @@ -348,21 +349,21 @@ GPU_TEST(managed_ptr, make_on_device) hostArray[0] = nullptr; int** deviceArray = nullptr; - cudaMalloc(&deviceArray, sizeof(int*)); + chai::gpuMalloc((void**)(&deviceArray), sizeof(int*)); int** deviceArray2 = nullptr; - cudaMalloc(&deviceArray2, sizeof(int*)); + chai::gpuMalloc((void**)(&deviceArray2), sizeof(int*)); deviceNew<<<1, 1>>>(deviceArray); - cudaMemcpy(hostArray, deviceArray, sizeof(int*), cudaMemcpyDeviceToHost); - cudaMemcpy(deviceArray2, hostArray, sizeof(int*), cudaMemcpyHostToDevice); + chai::gpuMemcpy(hostArray, deviceArray, sizeof(int*), gpuMemcpyDeviceToHost); + chai::gpuMemcpy(deviceArray2, hostArray, sizeof(int*), gpuMemcpyHostToDevice); ASSERT_NE(hostArray[0], nullptr); deviceDelete<<<1, 1>>>(deviceArray2); free(hostArray); - cudaFree(deviceArray); - cudaFree(deviceArray2); + chai::gpuFree(deviceArray); + chai::gpuFree(deviceArray2); } GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) @@ -373,16 +374,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) // Initialize device side memory to hold a pointer RawArrayClass** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(RawArrayClass*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), cudaMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); @@ -402,16 +403,16 @@ GPU_TEST(managed_ptr, gpu_build_managed_ptr) // Initialize device side memory to hold a pointer RawArrayClass** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(RawArrayClass*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(RawArrayClass*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), cudaMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(RawArrayClass*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); diff --git a/tests/integration/raja-chai-nested.cpp b/tests/integration/raja-chai-nested.cpp index 68f26cc6..152a18a2 100644 --- a/tests/integration/raja-chai-nested.cpp +++ b/tests/integration/raja-chai-nested.cpp @@ -27,6 +27,7 @@ using namespace std; #include "gtest/gtest.h" +// TODO: add hip policy for these tests. #if defined(RAJA_ENABLE_CUDA) #define PARALLEL_RAJA_DEVICE __device__ #elif defined(RAJA_ENABLE_OPENMP) @@ -108,6 +109,9 @@ CUDA_TEST(Chai, NestedSimple) ASSERT_FLOAT_EQ(v1[index], index * 1.0f); ASSERT_FLOAT_EQ(v2[index], index * 2.0f); }); + + v1.free(); + v2.free(); } CUDA_TEST(Chai, NestedView) @@ -168,6 +172,9 @@ CUDA_TEST(Chai, NestedView) [=](int i, int j) { ASSERT_FLOAT_EQ(v2(i, j), v1(i, j) * 2.0f); }); + + v1_array.free(); + v2_array.free(); } CUDA_TEST(Chai, NestedMultiView) @@ -236,6 +243,8 @@ CUDA_TEST(Chai, NestedMultiView) [=](int i, int j) { ASSERT_FLOAT_EQ(mview(1, i, j), mview(0, i, j) * 2.0f); }); + v1_array.free(); + v2_array.free(); } /////////////////////////////////////////////////////////////////////////// @@ -367,7 +376,9 @@ void runLTimesTests(Index_type num_moments, } }); - //rm->setExecutionSpace(chai::NONE); + L_data.free(); + psi_data.free(); + phi_data.free(); } TEST(Chai, LTimes) diff --git a/tests/integration/raja-chai-tests.cpp b/tests/integration/raja-chai-tests.cpp index b40a436c..f43340e6 100644 --- a/tests/integration/raja-chai-tests.cpp +++ b/tests/integration/raja-chai-tests.cpp @@ -16,6 +16,7 @@ #include "gtest/gtest.h" +// TODO: add hip policy for these tests. #define CUDA_TEST(X, Y) \ static void cuda_test_##X##_##Y(); \ TEST(X, Y) { cuda_test_##X##_##Y(); } \ @@ -62,6 +63,9 @@ CUDA_TEST(ChaiTest, Simple) ASSERT_FLOAT_EQ(raw_v2[i], i * 2.0f * 2.0f); ; } + + v1.free(); + v2.free(); } CUDA_TEST(ChaiTest, Views) @@ -90,11 +94,14 @@ CUDA_TEST(ChaiTest, Views) v2(i) *= 2.0f; }); - float* raw_v2 = v2.data.data(); + float* raw_v2 = v2_array.data(); for (int i = 0; i < 10; i++) { ASSERT_FLOAT_EQ(raw_v2[i], i * 1.0f * 2.0f * 2.0f); ; } + + v1_array.free(); + v2_array.free(); } CUDA_TEST(ChaiTest, MultiView) @@ -137,4 +144,7 @@ CUDA_TEST(ChaiTest, MultiView) ASSERT_FLOAT_EQ(raw_v2[i], i * 1.0f * 2.0f * 2.0f); ; } + + v1_array.free(); + v2_array.free(); } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 2438289e..d2236989 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -36,15 +36,17 @@ blt_add_test( NAME array_manager_unit_test COMMAND array_manager_unit_tests) -blt_add_executable( - NAME managed_ptr_unit_tests - SOURCES managed_ptr_unit_tests.cpp - DEPENDS_ON ${chai_unit_test_depends}) - -target_include_directories( - managed_ptr_unit_tests - PUBLIC ${PROJECT_BINARY_DIR}/include) - -blt_add_test( - NAME managed_ptr_unit_test - COMMAND managed_ptr_unit_tests) +if (CHAI_ENABLE_MANAGED_PTR) + blt_add_executable( + NAME managed_ptr_unit_tests + SOURCES managed_ptr_unit_tests.cpp + DEPENDS_ON ${chai_unit_test_depends}) + + target_include_directories( + managed_ptr_unit_tests + PUBLIC ${PROJECT_BINARY_DIR}/include) + + blt_add_test( + NAME managed_ptr_unit_test + COMMAND managed_ptr_unit_tests) +endif () diff --git a/tests/unit/array_manager_unit_tests.cpp b/tests/unit/array_manager_unit_tests.cpp index 5e1e68ea..3ea44ddc 100644 --- a/tests/unit/array_manager_unit_tests.cpp +++ b/tests/unit/array_manager_unit_tests.cpp @@ -72,6 +72,9 @@ TEST(ArrayManager, getPointerMap) ASSERT_EQ(rm->getTotalNumArrays(), 2); ASSERT_EQ(rm->getTotalSize(), (sizeOfArray1 * sizeof(int)) + (sizeOfArray2 * sizeof(double))); + + array1.free(); + array2.free(); } /*! diff --git a/tests/unit/managed_ptr_unit_tests.cpp b/tests/unit/managed_ptr_unit_tests.cpp index f6743251..f3cfe4ae 100644 --- a/tests/unit/managed_ptr_unit_tests.cpp +++ b/tests/unit/managed_ptr_unit_tests.cpp @@ -12,6 +12,7 @@ static void gpu_test_##X_##Y() #include "chai/config.hpp" +#include "chai/ArrayManager.hpp" #include "chai/ManagedArray.hpp" #include "chai/managed_ptr.hpp" @@ -464,7 +465,7 @@ TEST(managed_ptr, reinterpret_pointer_cast) derived.free(); } -#ifdef __CUDACC__ +#ifdef CHAI_GPUCC GPU_TEST(managed_ptr, gpu_default_constructor) { @@ -603,16 +604,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device) // Initialize device side memory to hold a pointer Simple** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(Simple*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(Simple*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), cudaMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr); @@ -632,16 +633,16 @@ GPU_TEST(managed_ptr, gpu_new_and_delete_on_device_2) // Initialize device side memory to hold a pointer Simple** gpuPointerHolder = nullptr; - cudaMalloc(&gpuPointerHolder, sizeof(Simple*)); + chai::gpuMalloc((void**)(&gpuPointerHolder), sizeof(Simple*)); // Create on the device chai::detail::make_on_device<<<1, 1>>>(gpuPointerHolder); // Copy to the host side memory - cudaMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), cudaMemcpyDeviceToHost); + chai::gpuMemcpy(cpuPointerHolder, gpuPointerHolder, sizeof(Simple*), gpuMemcpyDeviceToHost); // Free device side memory - cudaFree(gpuPointerHolder); + chai::gpuFree(gpuPointerHolder); // Save the pointer ASSERT_NE(cpuPointerHolder[0], nullptr);