From 8840330cd932599bcbf44906662e40dc11a37190 Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Thu, 25 Jul 2024 13:40:31 -0700
Subject: [PATCH 01/10] Partition tests between azure and GItHub Actions + some
 goodies.

---
 .github/ISSUE_TEMPLATE/bug_report.md      |  25 ++++
 .github/ISSUE_TEMPLATE/feature_request.md |  22 ++++
 .github/workflows/build.yml               |  86 +++++++++++++
 Dockerfile                                | 141 ++++++++++------------
 azure-pipelines.yml                       |  32 ++---
 5 files changed, 208 insertions(+), 98 deletions(-)
 create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md
 create mode 100644 .github/workflows/build.yml

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 0000000000..41e69454f8
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,25 @@
+---
+name: Bug report
+about: Create a report to help us improve
+
+---
+
+**Describe the bug**
+
+A clear and concise description of what the bug is:
+
+**To Reproduce**
+
+Steps to reproduce the behavior:
+
+**Expected behavior**
+
+A clear and concise description of what you expected to happen:
+
+**Compilers & Libraries (please complete the following information):**
+ - Compiler & version: [e.g. GCC 4.9.3]:
+ - CUDA version (if applicable):
+
+**Additional context**
+
+Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 0000000000..cd2ac307ce
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,22 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+
+A clear and concise description of what the problem is.
+
+**Describe the solution you'd like**
+
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+
+Add any other context or information about the feature request here.
+
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 0000000000..6724b82280
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,86 @@
+on: push
+name: Build
+jobs:
+  build_docker:
+    strategy:
+      matrix:
+        target: [gcc12, gcc13, clang13, clang15, hip5.6, intel2024, intel2024_sycl]
+    runs-on: ubuntu-latest
+    steps:
+    - run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /opt/hostedtoolcache/CodeQL
+        sudo docker image prune --all --force
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+    - uses: docker/setup-buildx-action@v3
+    - uses: docker/build-push-action@v6
+      with:
+        target: ${{ matrix.target }}
+  build_mac:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+      - uses: threeal/cmake-action@v1.3.0
+        with:
+          build-dir: build
+          options:
+            CMAKE_CXX_STANDARD=14
+            ENABLE_OPENMP=Off
+            CMAKE_BUILD_TYPE=Release
+          run-build: true
+          build-args: '--parallel 16'
+      - uses: threeal/ctest-action@v1.1.0
+  build_windows:
+    strategy:
+      matrix:
+        shared: 
+        - args: 
+            BUILD_SHARED_LIBS=On 
+            CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=On
+        - args: BUILD_SHARED_LIBS=Off
+          
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+## ====================================
+## Config and build action
+      - uses: threeal/cmake-action@v1.3.0
+        with:
+          build-dir: build
+          options:
+            ENABLE_WARNINGS_AS_ERRORS=Off
+            BLT_CXX_STD=""
+            CMAKE_CXX_STANDARD=17
+            CMAKE_BUILD_TYPE=Release
+            ${{ matrix.shared.args }}
+          run-build: true
+          build-args: '--parallel 16'
+## ====================================
+## Print the contents of the test directory in the build space (debugging)
+##    - run: | 
+##        dir -r D:\a\RAJA\RAJA\build\test
+## ====================================
+## Run tests action
+      - uses: threeal/ctest-action@v1.1.0
+        with:
+          build-config: Debug
+## ====================================
+## Attempt to run tests by invoking ctest manually.
+## Generates errors similar to above -- cannot find test executables.
+##    - name: Test
+##      working-directory: ${{github.workspace}}/build
+##      run: ctest -C Release --output-on-failure
+## ====================================
+## Another attempt.
+## This produces no errors, but test executables are not found.
+##    - name: Run Tests
+##      run: ctest.exe -C Release
+##      shell: bash
diff --git a/Dockerfile b/Dockerfile
index 89e6a83f70..32c3cd397b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -5,114 +5,107 @@
 # SPDX-License-Identifier: (BSD-3-Clause)
 ###############################################################################
 
-FROM ghcr.io/rse-ops/gcc-ubuntu-20.04:gcc-7.3.0 AS gcc7.3.0
+##
+## Note that we build with 'make -j 16' on GitHub Actions and
+## with 'make -j 6' on Azure. This is reflected in the 'make' commands below.
+## This seems to work best for throughput.
+##
+
+FROM ghcr.io/llnl/radiuss:gcc-11-ubuntu-22.04 AS gcc11
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=g++ -DRAJA_ENABLE_WARNINGS=On -DRAJA_DEPRECATED_TESTS=On -DENABLE_OPENMP=On .. && \
-    make -j 6 &&\
-    ctest -T test --output-on-failure && \
-    cd .. && rm -rf build
+RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Release -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On .. && \
+    make -j 16 &&\
+    ctest -T test --output-on-failure
 
-FROM ghcr.io/rse-ops/gcc-ubuntu-20.04:gcc-8.1.0 AS gcc8.1.0
+FROM ghcr.io/llnl/radiuss:gcc-12-ubuntu-22.04 AS gcc12
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=g++ -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_COVERAGE=On -DENABLE_OPENMP=On .. && \
-    make -j 6 &&\
-    ctest -T test --output-on-failure && \
-    cd .. && rm -rf build
+RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Release -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On .. && \
+    make -j 16 &&\
+    ctest -T test --output-on-failure
 
-FROM ghcr.io/rse-ops/gcc-ubuntu-20.04:gcc-9.4.0 AS gcc9.4.0
+FROM ghcr.io/llnl/radiuss:gcc-12-ubuntu-22.04 AS gcc12_debug
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=g++ -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_RUNTIME_PLUGINS=On -DENABLE_OPENMP=On .. && \
+RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Release -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On .. && \
     make -j 6 &&\
-    ctest -T test --output-on-failure && \
-    cd .. && rm -rf build
+    ctest -T test --output-on-failure
 
-FROM ghcr.io/rse-ops/gcc-ubuntu-20.04:gcc-11.2.0 AS gcc11.2.0
+FROM ghcr.io/llnl/radiuss:gcc-13-ubuntu-22.04 AS gcc13
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_COMPILER=g++ -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_BOUNDS_CHECK=ON -DENABLE_OPENMP=On .. && \
-    make -j 6 &&\
-    ctest -T test --output-on-failure && \
-    cd .. && rm -rf build
+RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Release -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On .. && \
+    make -j 16 &&\
+    ctest -T test --output-on-failure
 
-FROM ghcr.io/rse-ops/clang-ubuntu-20.04:llvm-11.0.0 AS clang11.0.0
+FROM ghcr.io/llnl/radiuss:clang-13-ubuntu-22.04 AS clang13
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN . /opt/spack/share/spack/setup-env.sh && export LD_LIBRARY_PATH=/opt/view/lib:$LD_LIBRARY_PATH && \
-    cmake -DCMAKE_CXX_COMPILER=clang++ -DENABLE_OPENMP=On .. && \
-    make -j 6 &&\
-    ctest -T test --output-on-failure && \
-    cd .. && rm -rf build
+RUN cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=On .. && \
+    make -j 16 &&\
+    ctest -T test --output-on-failure
 
-FROM ghcr.io/rse-ops/clang-ubuntu-20.04:llvm-11.0.0 AS clang11.0.0-debug
+FROM ghcr.io/llnl/radiuss:clang-14-ubuntu-22.04 AS clang14_debug
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN . /opt/spack/share/spack/setup-env.sh && export LD_LIBRARY_PATH=/opt/view/lib:$LD_LIBRARY_PATH && \
-    cmake -DCMAKE_CXX_COMPILER=clang++ -DENABLE_OPENMP=On -DCMAKE_BUILD_TYPE=Debug .. && \
+RUN cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Debug  -DENABLE_OPENMP=On .. && \
     make -j 6 &&\
-    ctest -T test --output-on-failure && \
-    cd .. && rm -rf build
+    ctest -T test --output-on-failure
 
-FROM ghcr.io/rse-ops/clang-ubuntu-22.04:llvm-13.0.0 AS clang13.0.0
+FROM ghcr.io/llnl/radiuss:clang-15-ubuntu-22.04 AS clang15
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN . /opt/spack/share/spack/setup-env.sh && export LD_LIBRARY_PATH=/opt/view/lib:$LD_LIBRARY_PATH && \
-    cmake -DCMAKE_CXX_COMPILER=clang++ -DENABLE_OPENMP=On -DCMAKE_BUILD_TYPE=Release .. && \
-    make -j 6 &&\
-    ctest -T test --output-on-failure && \
-    cd .. && rm -rf build
+RUN cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=On .. && \
+    make -j 16 &&\
+    ctest -T test --output-on-failure
 
-##FROM ghcr.io/rse-ops/cuda:cuda-10.1.243-ubuntu-18.04 AS nvcc10.1.243
-##ENV GTEST_COLOR=1
-##COPY . /home/raja/workspace
-##WORKDIR /home/raja/workspace/build
-##RUN . /opt/spack/share/spack/setup-env.sh && spack load cuda && \
-##    cmake -DCMAKE_CXX_COMPILER=g++ -DENABLE_CUDA=On -DCMAKE_CUDA_STANDARD=14 -DCMAKE_CUDA_ARCHITECTURES=70 -DENABLE_OPENMP=On .. && \
-##    make -j 4 && \
-##    cd .. && rm -rf build
+## Test run failure in RAJA launch tests with new reducer interface.
+## Need to figure out best way to handle that.
+FROM ghcr.io/llnl/radiuss:ubuntu-20.04-intel-2024.0 AS intel2024
+ENV GTEST_COLOR=1
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN /bin/bash -c "source /opt/intel/oneapi/setvars.sh 2>&1 > /dev/null && \
+    cmake -DCMAKE_CXX_COMPILER=icpx -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=On .. && \
+    make -j 16"
+##    make -j 16 &&\
+##    ctest -T test --output-on-failure"
 
-##FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.1.1 AS nvcc11.1.1
-##ENV GTEST_COLOR=1
-##COPY . /home/raja/workspace
-##WORKDIR /home/raja/workspace/build
-##RUN . /opt/spack/share/spack/setup-env.sh && spack load cuda && \
-##    cmake -DCMAKE_CXX_COMPILER=g++ -DENABLE_CUDA=On -DCMAKE_CUDA_STANDARD=14 -DCMAKE_CUDA_ARCHITECTURES=70 -DENABLE_OPENMP=On .. && \
-##    make -j 4 && \
-##    cd .. && rm -rf build
+##
+## Need to find a viable cuda image to test...
+## 
 
-##FROM ghcr.io/rse-ops/cuda-ubuntu-20.04:cuda-11.1.1 AS nvcc11.1.-debug
-##ENV GTEST_COLOR=1
-##COPY . /home/raja/workspace
-##WORKDIR /home/raja/workspace/build
-##RUN . /opt/spack/share/spack/setup-env.sh && spack load cuda && \
-##    cmake -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER=g++ -DENABLE_CUDA=On -DCMAKE_CUDA_STANDARD=14 -DCMAKE_CUDA_ARCHITECTURES=70 -DENABLE_OPENMP=On .. && \
-##    make -j 4 && \
-##    cd .. && rm -rf build
+# TODO: We should switch to ROCm 6 -- where to get an image??
+FROM ghcr.io/llnl/radiuss:ubuntu-20.04-hip-5.6.1 AS rocm5.6
+ENV GTEST_COLOR=1
+ENV HCC_AMDGPU_TARGET=gfx900
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 6
 
-##FROM ghcr.io/rse-ops/hip-ubuntu-20.04:hip-5.1.3 AS hip5.1.3
-##ENV GTEST_COLOR=1
-##ENV HCC_AMDGPU_TARGET=gfx900
-##COPY . /home/raja/workspace
-##WORKDIR /home/raja/workspace/build
-##RUN . /opt/spack/share/spack/setup-env.sh && spack load hip llvm-amdgpu && \
-##    cmake -DCMAKE_CXX_COMPILER=clang++ -DHIP_PATH=/opt -DENABLE_HIP=On -DENABLE_CUDA=Off -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
-##    make -j 6 && \
-##    cd .. && rm -rf build
+# TODO: We should switch to ROCm 6 -- where to get an image??
+FROM ghcr.io/llnl/radiuss:ubuntu-20.04-hip-5.6.1 AS rocm5.6_desul
+ENV GTEST_COLOR=1
+ENV HCC_AMDGPU_TARGET=gfx900
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 6
 
-FROM ghcr.io/rse-ops/intel-ubuntu-22.04:intel-2023.2.1 AS sycl
+FROM ghcr.io/llnl/radiuss:intel-2024.0-ubuntu-20.04 AS intel2024_sycl
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN /bin/bash -c "source /opt/view/setvars.sh && \
-    cmake -DCMAKE_CXX_COMPILER=dpcpp -DRAJA_ENABLE_SYCL=On -DENABLE_OPENMP=Off -DENABLE_ALL_WARNINGS=Off -DBLT_CXX_STD=c++17 .. && \
-    make -j 6" && \
-    cd .. && rm -rf build
+RUN /bin/bash -c "source /opt/intel/oneapi/setvars.sh 2>&1 > /dev/null && \
+    cmake -DCMAKE_CXX_COMPILER=dpcpp -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=Off -DRAJA_ENABLE_SYCL=On -DBLT_CXX_STD=c++17 -DRAJA_ENABLE_DESUL_ATOMICS .. && \
+    make -j 16"
+
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 33e3bfa8c0..13797d75d4 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -34,30 +34,14 @@ jobs:
   timeoutInMinutes: 360
   strategy:
     matrix: 
-      gcc7.3.0:
-        docker_target: gcc7.3.0
-      gcc8.1.0:
-        docker_target: gcc8.1.0
-      gcc9.4.0:
-        docker_target: gcc9.4.0
-      gcc11.2.0:
-        docker_target: gcc11.2.0
-      clang11.0.0:
-        docker_target: clang11.0.0
-      clang11.0.0-debug:
-        docker_target: clang11.0.0-debug
-      clang13.0.0:
-        docker_target: clang13.0.0
-##      nvcc10.1.243:
-##        docker_target: nvcc10.1.243
-##      nvcc11.1.1:
-##        docker_target: nvcc11.1.1
-##      nvcc11.1.1-debug:
-##        docker_target: nvcc11.1.1-debug
-##      hip5.1.3:
-##        docker_target: hip5.1.3
-      sycl:
-        docker_target: sycl
+      gcc12_debug:
+        docker_target: gcc12_debug
+      clang14_debug:
+        docker_target: clang14_debug
+      rocm5.6:
+        docker_target: rocm5.6
+      rocm5.6_desul:
+        docker_target: rocm5.6_desul
   pool:
     vmImage: 'ubuntu-latest'
   variables:

From 9fb05f08daf5a2fc9a733277d88485201c5c4ed3 Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Thu, 25 Jul 2024 13:45:20 -0700
Subject: [PATCH 02/10] Fix job name

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 6724b82280..fae97ee712 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ jobs:
   build_docker:
     strategy:
       matrix:
-        target: [gcc12, gcc13, clang13, clang15, hip5.6, intel2024, intel2024_sycl]
+        target: [gcc12, gcc13, clang13, clang15, rocm5.6, rocm5.6_desul, intel2024, intel2024_sycl]
     runs-on: ubuntu-latest
     steps:
     - run: |

From 1171d473c4fd63ffe694e19e70a0abfcf5ca7eb9 Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Thu, 25 Jul 2024 13:50:14 -0700
Subject: [PATCH 03/10] Fix another bug in dockerfile

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 32c3cd397b..3fae795ada 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -98,7 +98,7 @@ ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
     make -j 6
 
 FROM ghcr.io/llnl/radiuss:intel-2024.0-ubuntu-20.04 AS intel2024_sycl

From fd0857cba7e350c49f726f728a746921093b2e36 Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Thu, 25 Jul 2024 13:57:58 -0700
Subject: [PATCH 04/10] Fix another typo....grrrfff

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 3fae795ada..c7b22aac29 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -106,6 +106,6 @@ ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
 RUN /bin/bash -c "source /opt/intel/oneapi/setvars.sh 2>&1 > /dev/null && \
-    cmake -DCMAKE_CXX_COMPILER=dpcpp -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=Off -DRAJA_ENABLE_SYCL=On -DBLT_CXX_STD=c++17 -DRAJA_ENABLE_DESUL_ATOMICS .. && \
+    cmake -DCMAKE_CXX_COMPILER=dpcpp -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=Off -DRAJA_ENABLE_SYCL=On -DBLT_CXX_STD=c++17 -DRAJA_ENABLE_DESUL_ATOMICS=On .. && \
     make -j 16"
 

From b3a198696ff29d60798de8a5b4d70456823c044e Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Thu, 25 Jul 2024 15:00:58 -0700
Subject: [PATCH 05/10] Add some desul checks, move some builds around

---
 .github/workflows/build.yml |  2 +-
 Dockerfile                  | 18 +++++++++++++++++-
 azure-pipelines.yml         |  6 ++++--
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index fae97ee712..4f9d9a67c1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ jobs:
   build_docker:
     strategy:
       matrix:
-        target: [gcc12, gcc13, clang13, clang15, rocm5.6, rocm5.6_desul, intel2024, intel2024_sycl]
+        target: [gcc12, gcc13, clang13, clang15, rocm5.6_desul, intel2024, intel2024_sycl]
     runs-on: ubuntu-latest
     steps:
     - run: |
diff --git a/Dockerfile b/Dockerfile
index c7b22aac29..0fd23f163b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -31,7 +31,15 @@ FROM ghcr.io/llnl/radiuss:gcc-12-ubuntu-22.04 AS gcc12_debug
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Release -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On .. && \
+RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Debug -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On .. && \
+    make -j 6 &&\
+    ctest -T test --output-on-failure
+
+FROM ghcr.io/llnl/radiuss:gcc-12-ubuntu-22.04 AS gcc12_desul
+ENV GTEST_COLOR=1
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Release -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On -DRAJA_ENABLE_DESUL_ATOMICS=On .. && \
     make -j 6 &&\
     ctest -T test --output-on-failure
 
@@ -67,6 +75,14 @@ RUN cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENM
     make -j 16 &&\
     ctest -T test --output-on-failure
 
+FROM ghcr.io/llnl/radiuss:clang-15-ubuntu-22.04 AS clang15_desul
+ENV GTEST_COLOR=1
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=On -DRAJA_ENABLE_DESUL_ATOMICS=On .. && \
+    make -j 16 &&\
+    ctest -T test --output-on-failure
+
 ## Test run failure in RAJA launch tests with new reducer interface.
 ## Need to figure out best way to handle that.
 FROM ghcr.io/llnl/radiuss:ubuntu-20.04-intel-2024.0 AS intel2024
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 13797d75d4..096b0cbcfc 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -36,12 +36,14 @@ jobs:
     matrix: 
       gcc12_debug:
         docker_target: gcc12_debug
+      gcc12_desul:
+        docker_target: gcc12_desul
       clang14_debug:
         docker_target: clang14_debug
+      clang15_desul:
+        docker_target: clang15_desul
       rocm5.6:
         docker_target: rocm5.6
-      rocm5.6_desul:
-        docker_target: rocm5.6_desul
   pool:
     vmImage: 'ubuntu-latest'
   variables:

From 512cf7051d5826ada51615c8cb2780a25db8e528 Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Thu, 25 Jul 2024 16:04:50 -0700
Subject: [PATCH 06/10] add intel debug to GH actions, remove rocm from azure

---
 .github/workflows/build.yml |  2 +-
 Dockerfile                  | 12 ++++++++++++
 azure-pipelines.yml         |  2 --
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4f9d9a67c1..544c2bca5e 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ jobs:
   build_docker:
     strategy:
       matrix:
-        target: [gcc12, gcc13, clang13, clang15, rocm5.6_desul, intel2024, intel2024_sycl]
+        target: [gcc12, gcc13, clang13, clang15, rocm5.6, rocm5.6_desul, intel2024, intel2024_debug, intel2024_sycl]
     runs-on: ubuntu-latest
     steps:
     - run: |
diff --git a/Dockerfile b/Dockerfile
index 0fd23f163b..b7bf6370a3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -95,6 +95,18 @@ RUN /bin/bash -c "source /opt/intel/oneapi/setvars.sh 2>&1 > /dev/null && \
 ##    make -j 16 &&\
 ##    ctest -T test --output-on-failure"
 
+## Test run failure in RAJA launch tests with new reducer interface.
+## Need to figure out best way to handle that.
+FROM ghcr.io/llnl/radiuss:ubuntu-20.04-intel-2024.0 AS intel2024_debug
+ENV GTEST_COLOR=1
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN /bin/bash -c "source /opt/intel/oneapi/setvars.sh 2>&1 > /dev/null && \
+    cmake -DCMAKE_CXX_COMPILER=icpx -DCMAKE_BUILD_TYPE=Debug -DENABLE_OPENMP=On .. && \
+    make -j 16"
+##    make -j 16 &&\
+##    ctest -T test --output-on-failure"
+
 ##
 ## Need to find a viable cuda image to test...
 ## 
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 096b0cbcfc..6cb4f3537e 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -42,8 +42,6 @@ jobs:
         docker_target: clang14_debug
       clang15_desul:
         docker_target: clang15_desul
-      rocm5.6:
-        docker_target: rocm5.6
   pool:
     vmImage: 'ubuntu-latest'
   variables:

From c99b902382b43b1f118ffd3e9e11f8601dda8c0d Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Wed, 14 Aug 2024 10:38:52 -0700
Subject: [PATCH 07/10] Try rocm 6.0 image

---
 .github/workflows/build.yml |  2 +-
 Dockerfile                  | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 544c2bca5e..28fd192300 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ jobs:
   build_docker:
     strategy:
       matrix:
-        target: [gcc12, gcc13, clang13, clang15, rocm5.6, rocm5.6_desul, intel2024, intel2024_debug, intel2024_sycl]
+        target: [gcc12, gcc13, clang13, clang15, rocm6.0, rocm6.0_desul, intel2024, intel2024_debug, intel2024_sycl]
     runs-on: ubuntu-latest
     steps:
     - run: |
diff --git a/Dockerfile b/Dockerfile
index b7bf6370a3..11229220ba 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -129,6 +129,22 @@ WORKDIR /home/raja/workspace/build
 RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
     make -j 6
 
+FROM ghcr.io/llnl/radiuss:hip-6.0.2-ubuntu-20.04 AS rocm6.0
+ENV GTEST_COLOR=1
+ENV HCC_AMDGPU_TARGET=gfx900
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 6
+
+FROM ghcr.io/llnl/radiuss:hip-6.0.2-ubuntu-20.04 AS rocm6.0_desul
+ENV GTEST_COLOR=1
+ENV HCC_AMDGPU_TARGET=gfx900
+COPY . /home/raja/workspace
+WORKDIR /home/raja/workspace/build
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 6
+
 FROM ghcr.io/llnl/radiuss:intel-2024.0-ubuntu-20.04 AS intel2024_sycl
 ENV GTEST_COLOR=1
 COPY . /home/raja/workspace

From 3af08e6f80eb6a104776fd4dc978bfd7901c626d Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Wed, 14 Aug 2024 10:50:12 -0700
Subject: [PATCH 08/10] Point at correct ROCm compiler

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 11229220ba..72333e14da 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -134,7 +134,7 @@ ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-6.0.2/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
     make -j 6
 
 FROM ghcr.io/llnl/radiuss:hip-6.0.2-ubuntu-20.04 AS rocm6.0_desul
@@ -142,7 +142,7 @@ ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-6.0.2/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
     make -j 6
 
 FROM ghcr.io/llnl/radiuss:intel-2024.0-ubuntu-20.04 AS intel2024_sycl

From 5c38a9496eeaa2a5cae820c3a73e4874595e57e3 Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Wed, 14 Aug 2024 13:52:50 -0700
Subject: [PATCH 09/10] CI cleanup

---
 .github/workflows/build.yml | 14 +-------------
 Dockerfile                  |  6 ++++--
 azure-pipelines.yml         |  2 ++
 3 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 28fd192300..8aabaf94c1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ jobs:
   build_docker:
     strategy:
       matrix:
-        target: [gcc12, gcc13, clang13, clang15, rocm6.0, rocm6.0_desul, intel2024, intel2024_debug, intel2024_sycl]
+        target: [gcc12, gcc13, clang13, clang15, rocm5.6, rocm5.6_desul, intel2024, intel2024_debug, intel2024_sycl]
     runs-on: ubuntu-latest
     steps:
     - run: |
@@ -72,15 +72,3 @@ jobs:
       - uses: threeal/ctest-action@v1.1.0
         with:
           build-config: Debug
-## ====================================
-## Attempt to run tests by invoking ctest manually.
-## Generates errors similar to above -- cannot find test executables.
-##    - name: Test
-##      working-directory: ${{github.workspace}}/build
-##      run: ctest -C Release --output-on-failure
-## ====================================
-## Another attempt.
-## This produces no errors, but test executables are not found.
-##    - name: Run Tests
-##      run: ctest.exe -C Release
-##      shell: bash
diff --git a/Dockerfile b/Dockerfile
index 72333e14da..55f3815f50 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,7 +16,7 @@ ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
 RUN cmake -DCMAKE_CXX_COMPILER=g++ -DCMAKE_BUILD_TYPE=Release -DRAJA_ENABLE_WARNINGS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=On -DENABLE_OPENMP=On .. && \
-    make -j 16 &&\
+    make -j 6 &&\
     ctest -T test --output-on-failure
 
 FROM ghcr.io/llnl/radiuss:gcc-12-ubuntu-22.04 AS gcc12
@@ -80,7 +80,7 @@ ENV GTEST_COLOR=1
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
 RUN cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_OPENMP=On -DRAJA_ENABLE_DESUL_ATOMICS=On .. && \
-    make -j 16 &&\
+    make -j 6 &&\
     ctest -T test --output-on-failure
 
 ## Test run failure in RAJA launch tests with new reducer interface.
@@ -129,6 +129,7 @@ WORKDIR /home/raja/workspace/build
 RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
     make -j 6
 
+## ROCm 6 image is broken
 FROM ghcr.io/llnl/radiuss:hip-6.0.2-ubuntu-20.04 AS rocm6.0
 ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
@@ -137,6 +138,7 @@ WORKDIR /home/raja/workspace/build
 RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-6.0.2/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
     make -j 6
 
+## ROCm 6 image is broken
 FROM ghcr.io/llnl/radiuss:hip-6.0.2-ubuntu-20.04 AS rocm6.0_desul
 ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 6cb4f3537e..4535138152 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -34,6 +34,8 @@ jobs:
   timeoutInMinutes: 360
   strategy:
     matrix: 
+      gcc11:
+        docker_target: gcc11
       gcc12_debug:
         docker_target: gcc12_debug
       gcc12_desul:

From 1be9c30f7f5339f4ea786d0bdb07254e75ab098e Mon Sep 17 00:00:00 2001
From: Rich Hornung <hornung1@llnl.gov>
Date: Fri, 16 Aug 2024 14:16:52 -0700
Subject: [PATCH 10/10] Make build type explicit for CI checks

---
 Dockerfile | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 55f3815f50..4e224371eb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -117,8 +117,8 @@ ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
-    make -j 6
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 16
 
 # TODO: We should switch to ROCm 6 -- where to get an image??
 FROM ghcr.io/llnl/radiuss:ubuntu-20.04-hip-5.6.1 AS rocm5.6_desul
@@ -126,8 +126,8 @@ ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
-    make -j 6
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-5.6.1/bin/amdclang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 16
 
 ## ROCm 6 image is broken
 FROM ghcr.io/llnl/radiuss:hip-6.0.2-ubuntu-20.04 AS rocm6.0
@@ -135,8 +135,8 @@ ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-6.0.2/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
-    make -j 6
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-6.0.2/bin/amdclang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_HIP=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 16
 
 ## ROCm 6 image is broken
 FROM ghcr.io/llnl/radiuss:hip-6.0.2-ubuntu-20.04 AS rocm6.0_desul
@@ -144,8 +144,8 @@ ENV GTEST_COLOR=1
 ENV HCC_AMDGPU_TARGET=gfx900
 COPY . /home/raja/workspace
 WORKDIR /home/raja/workspace/build
-RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-6.0.2/bin/amdclang++ -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
-    make -j 6
+RUN cmake -DCMAKE_CXX_COMPILER=/opt/rocm-6.0.2/bin/amdclang++ -DCMAKE_BUILD_TYPE=Release -DENABLE_HIP=On -DRAJA_ENABLE_DESUL_ATOMICS=On -DRAJA_ENABLE_WARNINGS_AS_ERRORS=Off .. && \
+    make -j 16
 
 FROM ghcr.io/llnl/radiuss:intel-2024.0-ubuntu-20.04 AS intel2024_sycl
 ENV GTEST_COLOR=1