From bb688e088e08c36181cf0f242ac2d8224ea316b5 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 10 Jan 2024 14:41:46 -0800 Subject: [PATCH 1/3] Add SYCL CI (#1371) --- .github/workflows/sycl.yml | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/sycl.yml diff --git a/.github/workflows/sycl.yml b/.github/workflows/sycl.yml new file mode 100644 index 000000000..efaa57f5d --- /dev/null +++ b/.github/workflows/sycl.yml @@ -0,0 +1,50 @@ +name: ERF CI (sycl) + +on: + push: + # branches: [development] + paths-ignore: + - Docs + - README.rst + - license.txt + + pull_request: + branches: [development] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-sycl + cancel-in-progress: true + +jobs: + Build-And-Test-SYCL: + name: oneAPI SYCL + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Dependencies + run: Submodules/AMReX/.github/workflows/dependencies/dependencies_dpcpp.sh + + - name: Build & Install + run: | + set +e + source /opt/intel/oneapi/setvars.sh + set -e + mkdir build + cd build + cmake .. \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/ERF/install \ + -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ + -DERF_DIM:STRING=3 \ + -DERF_ENABLE_MPI:BOOL=ON \ + -DERF_ENABLE_SYCL:BOOL=ON \ + -DERF_ENABLE_TESTS:BOOL=ON \ + -DERF_ENABLE_ALL_WARNINGS:BOOL=ON \ + -DERF_ENABLE_FCOMPARE:BOOL=ON \ + -DCMAKE_C_COMPILER=$(which icx) \ + -DCMAKE_CXX_COMPILER=$(which icpx) \ + -DCMAKE_CXX_STANDARD=17 + make -j 2; From 41a5483297c851668cd4fc18904bc4216a09db29 Mon Sep 17 00:00:00 2001 From: Eliot Quon Date: Wed, 10 Jan 2024 15:42:14 -0700 Subject: [PATCH 2/3] Account for terrain cellsize in SFS models (#1370) * Account for terrain in cellVol calculation * oops --- .../Diffusion/ComputeTurbulentViscosity.cpp | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/Source/Diffusion/ComputeTurbulentViscosity.cpp b/Source/Diffusion/ComputeTurbulentViscosity.cpp index 2a0389277..217cad3f9 100644 --- a/Source/Diffusion/ComputeTurbulentViscosity.cpp +++ b/Source/Diffusion/ComputeTurbulentViscosity.cpp @@ -4,6 +4,7 @@ #include #include #include +#include using namespace amrex; @@ -45,11 +46,14 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu amrex::MultiFab& Hfx1, amrex::MultiFab& Hfx2, amrex::MultiFab& Hfx3, amrex::MultiFab& Diss, const amrex::Geometry& geom, const amrex::MultiFab& mapfac_u, const amrex::MultiFab& mapfac_v, + const std::unique_ptr& z_phys_nd, const TurbChoice& turbChoice, const Real const_grav) { - const amrex::GpuArray dxInv = geom.InvCellSizeArray(); + const amrex::GpuArray cellSizeInv = geom.InvCellSizeArray(); const Box& domain = geom.Domain(); + const bool use_terrain = (z_phys_nd != nullptr); + // SMAGORINSKY: Fill Kturb for momentum in horizontal and vertical //*********************************************************************************** if (turbChoice.les_type == LESType::Smagorinsky) @@ -78,10 +82,19 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu Array4 mf_u = mapfac_u.array(mfi); Array4 mf_v = mapfac_v.array(mfi); + Array4 z_nd_arr = (use_terrain) ? z_phys_nd->const_array(mfi) : Array4{}; + ParallelFor(bxcc, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { Real SmnSmn = ComputeSmnSmn(i,j,k,tau11,tau22,tau33,tau12,tau13,tau23); - Real cellVolMsf = 1.0 / (dxInv[0] * mf_u(i,j,0) * dxInv[1] * mf_v(i,j,0) * dxInv[2]); + Real dxInv = cellSizeInv[0]; + Real dyInv = cellSizeInv[1]; + Real dzInv = cellSizeInv[2]; + if (use_terrain) { + // the terrain grid is only deformed in z for now + dzInv /= Compute_h_zeta_AtCellCenter(i,j,k, cellSizeInv, z_nd_arr); + } + Real cellVolMsf = 1.0 / (dxInv * mf_u(i,j,0) * dyInv * mf_v(i,j,0) * dzInv); Real DeltaMsf = std::pow(cellVolMsf,1.0/3.0); Real CsDeltaSqrMsf = Cs*Cs*DeltaMsf*DeltaMsf; @@ -119,15 +132,24 @@ void ComputeTurbulentViscosityLES (const amrex::MultiFab& Tau11, const amrex::Mu Array4 mf_u = mapfac_u.array(mfi); Array4 mf_v = mapfac_v.array(mfi); + Array4 z_nd_arr = (use_terrain) ? z_phys_nd->const_array(mfi) : Array4{}; + ParallelFor(bxcc, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - Real cellVolMsf = 1.0 / (dxInv[0] * mf_u(i,j,0) * dxInv[1] * mf_v(i,j,0) * dxInv[2]); + Real dxInv = cellSizeInv[0]; + Real dyInv = cellSizeInv[1]; + Real dzInv = cellSizeInv[2]; + if (use_terrain) { + // the terrain grid is only deformed in z for now + dzInv /= Compute_h_zeta_AtCellCenter(i,j,k, cellSizeInv, z_nd_arr); + } + Real cellVolMsf = 1.0 / (dxInv * mf_u(i,j,0) * dyInv * mf_v(i,j,0) * dzInv); Real DeltaMsf = std::pow(cellVolMsf,1.0/3.0); // Calculate stratification-dependent mixing length (Deardorff 1980) Real eps = std::numeric_limits::epsilon(); Real dtheta_dz = 0.5 * ( cell_data(i,j,k+1,RhoTheta_comp)/cell_data(i,j,k+1,Rho_comp) - - cell_data(i,j,k-1,RhoTheta_comp)/cell_data(i,j,k-1,Rho_comp) )*dxInv[2]; + - cell_data(i,j,k-1,RhoTheta_comp)/cell_data(i,j,k-1,Rho_comp) )*dzInv; Real E = cell_data(i,j,k,RhoKE_comp) / cell_data(i,j,k,Rho_comp); Real strat = l_abs_g * dtheta_dz * l_inv_theta0; // stratification Real length; @@ -403,6 +425,7 @@ void ComputeTurbulentViscosity (const amrex::MultiFab& xvel , const amrex::Multi cons_in, eddyViscosity, Hfx1, Hfx2, Hfx3, Diss, geom, mapfac_u, mapfac_v, + z_phys_nd, turbChoice, const_grav); } From 60668b2bc5cbb1815873f2f3e4da1b073f415ac2 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 10 Jan 2024 15:26:53 -0800 Subject: [PATCH 3/3] Use amrex's scripts to install cuda, hip, gcc, and mac dependencies (#1372) Co-authored-by: Ann Almgren --- .github/workflows/cuda-ci.yml | 16 ++--- .../workflows/dependencies/dependencies.sh | 16 ----- .../dependencies/dependencies_hip.sh | 58 ------------------- .../dependencies/dependencies_mac.sh | 14 ----- .github/workflows/gcc.yml | 2 +- .github/workflows/hip.yml | 2 +- .github/workflows/macos.yml | 2 +- 7 files changed, 7 insertions(+), 103 deletions(-) delete mode 100755 .github/workflows/dependencies/dependencies.sh delete mode 100755 .github/workflows/dependencies/dependencies_hip.sh delete mode 100755 .github/workflows/dependencies/dependencies_mac.sh diff --git a/.github/workflows/cuda-ci.yml b/.github/workflows/cuda-ci.yml index ea9b0df02..aa9a9f153 100644 --- a/.github/workflows/cuda-ci.yml +++ b/.github/workflows/cuda-ci.yml @@ -25,7 +25,6 @@ jobs: include: - cuda_ver: "11.0" cuda_pkg: 11-0 - cuda_extra: libcurand-dev-11-0 cuda-cupti-dev-11-0 libcusolver-dev-11-0 libcublas-dev-11-0 libcusparse-dev-11-0 steps: - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.6.0 @@ -35,19 +34,12 @@ jobs: with: submodules: true - name: Prepare CUDA environment - run: | - export DEBIAN_FRONTEND=noninteractive - sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub - sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64/7fa2af80.pub - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/cuda.list - echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu2004/x86_64 /" | sudo tee /etc/apt/sources.list.d/nvidia-ml.list - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - libopenmpi-dev cuda-command-line-tools-${{matrix.cuda_pkg}} cuda-compiler-${{matrix.cuda_pkg}} cuda-minimal-build-${{matrix.cuda_pkg}} cuda-nvml-dev-${{matrix.cuda_pkg}} cuda-nvtx-${{matrix.cuda_pkg}} ${{matrix.cuda_extra}} + run: Submodules/AMReX/.github/workflows/dependencies/dependencies_nvcc.sh ${{matrix.cuda_ver}} - name: Configure and build run: | - export PATH=/usr/local/nvidia/bin:/usr/local/cuda-${{matrix.cuda_ver}}/bin:${PATH} - export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda-${{matrix.cuda_ver}}/lib:${LD_LIBRARY_PATH} + export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} + export LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:${LD_LIBRARY_PATH} + which nvcc || echo "nvcc not in PATH!" cmake -Bbuild-${{matrix.cuda_pkg}} \ -DCMAKE_BUILD_TYPE:STRING=RelWithDebInfo \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ diff --git a/.github/workflows/dependencies/dependencies.sh b/.github/workflows/dependencies/dependencies.sh deleted file mode 100755 index 8fea96d48..000000000 --- a/.github/workflows/dependencies/dependencies.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2020 The AMReX Community -# -# License: BSD-3-Clause-LBNL -# Authors: Axel Huebl - -set -eu -o pipefail - -sudo apt-get update - -sudo apt-get install -y --no-install-recommends\ - build-essential \ - g++ \ - libopenmpi-dev \ - openmpi-bin diff --git a/.github/workflows/dependencies/dependencies_hip.sh b/.github/workflows/dependencies/dependencies_hip.sh deleted file mode 100755 index 4fb2c87fc..000000000 --- a/.github/workflows/dependencies/dependencies_hip.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2020 The AMReX Community -# -# License: BSD-3-Clause-LBNL -# Authors: Axel Huebl - -# search recursive inside a folder if a file contains tabs -# -# @result 0 if no files are found, else 1 -# - -set -eu -o pipefail - -# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu -wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key \ - | sudo apt-key add - -echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \ - | sudo tee /etc/apt/sources.list.d/rocm.list - -echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ - | sudo tee -a /etc/profile.d/rocm.sh -# we should not need to export HIP_PATH=/opt/rocm/hip with those installs - -sudo apt-get update - -# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#installing-development-packages-for-cross-compilation -# meta-package: rocm-dkms -# OpenCL: rocm-opencl -# other: rocm-dev rocm-utils -sudo apt-get install -y --no-install-recommends \ - build-essential \ - gfortran \ - libnuma-dev \ - libopenmpi-dev \ - openmpi-bin \ - rocm-dev \ - roctracer-dev \ - rocprofiler-dev \ - rocrand-dev \ - rocprim-dev \ - rocfft \ - rocprim \ - rocrand \ - hiprand-dev - -# activate -# -source /etc/profile.d/rocm.sh -hipcc --version -which clang -which clang++ - -# cmake-easyinstall -# -sudo curl -L -o /usr/local/bin/cmake-easyinstall https://git.io/JvLxY -sudo chmod a+x /usr/local/bin/cmake-easyinstall -export CEI_SUDO="sudo" diff --git a/.github/workflows/dependencies/dependencies_mac.sh b/.github/workflows/dependencies/dependencies_mac.sh deleted file mode 100755 index 95f5786b1..000000000 --- a/.github/workflows/dependencies/dependencies_mac.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright 2020 The AMReX Community -# -# License: BSD-3-Clause-LBNL -# Authors: Axel Huebl - -set -eu -o pipefail - -brew update -brew install gfortran || true -brew install libomp || true -brew install open-mpi || true -brew install ccache || true diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml index f149d99b3..6fad6e536 100644 --- a/.github/workflows/gcc.yml +++ b/.github/workflows/gcc.yml @@ -17,7 +17,7 @@ jobs: submodules: true - name: Install Dependencies - run: .github/workflows/dependencies/dependencies.sh + run: Submodules/AMReX/.github/workflows/dependencies/dependencies.sh - name: Configure Project and Generate Build System run: | diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 8addaf8cb..918d0f1e4 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -40,7 +40,7 @@ jobs: submodules: true - name: Dependencies - run: .github/workflows/dependencies/dependencies_hip.sh + run: Submodules/AMReX/.github/workflows/dependencies/dependencies_hip.sh - name: Build & Install run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 92acd4d6c..548621c4f 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -17,7 +17,7 @@ jobs: submodules: true - name: Install Dependencies - run: .github/workflows/dependencies/dependencies_mac.sh + run: Submodules/AMReX/.github/workflows/dependencies/dependencies_mac.sh - name: Configure Project and Generate Build System run: |