update setonix configuration (#362)

* update setonix configuration * update config * use un-hidden cmake * Update setonix.cmake --------- Co-authored-by: Piyush Sharda <[email protected]>
quokka-astro · Sep 9, 2023 · d17b5e4 · d17b5e4
1 parent 7ffd65f
commit d17b5e4
Show file tree

Hide file tree

Showing 7 changed files with 13 additions and 67 deletions.
diff --git a/cmake/setonix.cmake b/cmake/setonix.cmake
@@ -3,7 +3,8 @@
 
 set(AMReX_GPU_BACKEND HIP CACHE STRING "")
 set(AMReX_AMD_ARCH gfx90a CACHE STRING "") # MI250X
-set(AMReX_ASCENT OFF BOOL STRING "")
+#set(AMReX_ASCENT ON CACHE BOOL "")
+#set(AMReX_CONDUIT ON CACHE BOOL "")
 
 option(QUOKKA_PYTHON ON)
-set(AMReX_SPACEDIM 3 CACHE STRING "")
+set(AMReX_SPACEDIM 3 CACHE STRING "")
diff --git a/scripts/setonix-1node.submit b/scripts/setonix-1node.submit
@@ -8,13 +8,12 @@
 #SBATCH --exclusive
 #SBATCH --ntasks-per-node=8
 #SBATCH --gpus-per-node=8
-#SBATCH --cpus-per-task=8
 ##SBATCH --core-spec=8
 #SBATCH -N 1
 
 # load modules
 module load craype-accel-amd-gfx90a
-module load rocm/5.4.3
+module load rocm/5.2.3
 
 # workaround no longer needed for AMReX 23.07+
 #export FI_MR_CACHE_MAX_COUNT=0  # libfabric disable caching

diff --git a/scripts/setonix-64nodes.submit b/scripts/setonix-64nodes.submit
@@ -8,13 +8,12 @@
 #SBATCH --exclusive
 #SBATCH --ntasks-per-node=8
 #SBATCH --gpus-per-node=8
-#SBATCH --cpus-per-task=8
 ##SBATCH --core-spec=8
 #SBATCH -N 64
 
 # load modules
 module load craype-accel-amd-gfx90a
-module load rocm/5.4.3
+module load rocm/5.2.3
 
 # workaround no longer needed for AMReX 23.07+
 #export FI_MR_CACHE_MAX_COUNT=0  # libfabric disable caching

diff --git a/scripts/setonix-8nodes.submit b/scripts/setonix-8nodes.submit
@@ -8,13 +8,12 @@
 #SBATCH --exclusive
 #SBATCH --ntasks-per-node=8
 #SBATCH --gpus-per-node=8
-#SBATCH --cpus-per-task=8
 ##SBATCH --core-spec=8
 #SBATCH -N 8
 
 # load modules
 module load craype-accel-amd-gfx90a
-module load rocm/5.4.3
+module load rocm/5.2.3
 
 # workaround no longer needed for AMReX 23.07+
 #export FI_MR_CACHE_MAX_COUNT=0  # libfabric disable caching

diff --git a/scripts/setonix-rocm5.3.0.profile b/scripts/setonix-rocm5.3.0.profile
diff --git a/scripts/setonix-rocm5.4.3.profile b/scripts/setonix-rocm5.4.3.profile
diff --git a/scripts/setonix.profile b/scripts/setonix.profile
@@ -1,12 +1,12 @@
 #!/bin/bash
 
-source /opt/cray/pe/cpe/22.09/restore_lmod_system_defaults.sh
+source /opt/cray/pe/cpe/23.03/restore_lmod_system_defaults.sh
 
-module load cmake/3.21.4
+module load cmake/3.24.3
 module load craype-accel-amd-gfx90a
-module load rocm/5.0.2
+module load rocm/5.2.3
 module load cray-mpich
-module load cce/14.0.3
+module load cce/15.0.1
 module load cray-hdf5
 module load cray-python/3.9.13.1
 
@@ -16,6 +16,9 @@ export MPICH_GPU_SUPPORT_ENABLED=1
 # optimize ROCm/HIP compilation for MI250X
 export AMREX_AMD_ARCH=gfx90a
 
+# allow CMake to find Ascent
+export Ascent_DIR=/software/projects/pawsey0807/bwibking/ascent_06082023/install/ascent-develop/lib/cmake/ascent/
+
 # compiler environment hints
 export CC=$(which cc)
 export CXX=$(which CC)