diff --git a/cmake/setonix.cmake b/cmake/setonix.cmake index 5574a1110..3e6ec011f 100644 --- a/cmake/setonix.cmake +++ b/cmake/setonix.cmake @@ -3,7 +3,8 @@ set(AMReX_GPU_BACKEND HIP CACHE STRING "") set(AMReX_AMD_ARCH gfx90a CACHE STRING "") # MI250X -set(AMReX_ASCENT OFF BOOL STRING "") +#set(AMReX_ASCENT ON CACHE BOOL "") +#set(AMReX_CONDUIT ON CACHE BOOL "") option(QUOKKA_PYTHON ON) -set(AMReX_SPACEDIM 3 CACHE STRING "") \ No newline at end of file +set(AMReX_SPACEDIM 3 CACHE STRING "") diff --git a/scripts/setonix-1node.submit b/scripts/setonix-1node.submit index 99a763aed..220e1b357 100644 --- a/scripts/setonix-1node.submit +++ b/scripts/setonix-1node.submit @@ -8,13 +8,12 @@ #SBATCH --exclusive #SBATCH --ntasks-per-node=8 #SBATCH --gpus-per-node=8 -#SBATCH --cpus-per-task=8 ##SBATCH --core-spec=8 #SBATCH -N 1 # load modules module load craype-accel-amd-gfx90a -module load rocm/5.4.3 +module load rocm/5.2.3 # workaround no longer needed for AMReX 23.07+ #export FI_MR_CACHE_MAX_COUNT=0 # libfabric disable caching diff --git a/scripts/setonix-64nodes.submit b/scripts/setonix-64nodes.submit index aad12d01c..886a41e22 100644 --- a/scripts/setonix-64nodes.submit +++ b/scripts/setonix-64nodes.submit @@ -8,13 +8,12 @@ #SBATCH --exclusive #SBATCH --ntasks-per-node=8 #SBATCH --gpus-per-node=8 -#SBATCH --cpus-per-task=8 ##SBATCH --core-spec=8 #SBATCH -N 64 # load modules module load craype-accel-amd-gfx90a -module load rocm/5.4.3 +module load rocm/5.2.3 # workaround no longer needed for AMReX 23.07+ #export FI_MR_CACHE_MAX_COUNT=0 # libfabric disable caching diff --git a/scripts/setonix-8nodes.submit b/scripts/setonix-8nodes.submit index a2fdbbc80..5b3a3b322 100644 --- a/scripts/setonix-8nodes.submit +++ b/scripts/setonix-8nodes.submit @@ -8,13 +8,12 @@ #SBATCH --exclusive #SBATCH --ntasks-per-node=8 #SBATCH --gpus-per-node=8 -#SBATCH --cpus-per-task=8 ##SBATCH --core-spec=8 #SBATCH -N 8 # load modules module load craype-accel-amd-gfx90a -module load rocm/5.4.3 +module load rocm/5.2.3 # workaround no longer needed for AMReX 23.07+ #export FI_MR_CACHE_MAX_COUNT=0 # libfabric disable caching diff --git a/scripts/setonix-rocm5.3.0.profile b/scripts/setonix-rocm5.3.0.profile deleted file mode 100644 index ba34627c7..000000000 --- a/scripts/setonix-rocm5.3.0.profile +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -source /opt/cray/pe/cpe/22.09/restore_lmod_system_defaults.sh - -module load cmake/3.21.4 -module load craype-accel-amd-gfx90a - -. /software/projects/pawsey0807/bwibking/spack/share/spack/setup-env.sh -spack load hip@5.3.0 -spack load rocrand@5.3.0 -spack load rocprim@5.3.0 - -module load cray-mpich -module load cce/14.0.3 -module load cray-hdf5 -module load cray-python/3.9.13.1 - -# GPU-aware MPI -export MPICH_GPU_SUPPORT_ENABLED=1 - -# optimize ROCm/HIP compilation for MI250X -export AMREX_AMD_ARCH=gfx90a - -# compiler environment hints -export CC=$(which hipcc) -export CXX=$(which hipcc) -export FC=$(which ftn) -export CFLAGS="-I${ROCM_PATH}/include -I${MPICH_DIR}/include -I${HDF5_DIR}/include" -export CXXFLAGS="-I${ROCM_PATH}/include -I${MPICH_DIR}/include -I${HDF5_DIR}/include" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 -L${MPICH_DIR}/lib -lmpi -L${HDF5_DIR}/lib -lhdf5 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" diff --git a/scripts/setonix-rocm5.4.3.profile b/scripts/setonix-rocm5.4.3.profile deleted file mode 100644 index 185819f41..000000000 --- a/scripts/setonix-rocm5.4.3.profile +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -source /opt/cray/pe/cpe/22.09/restore_lmod_system_defaults.sh - -module load PrgEnv-gnu -module load cmake/3.21.4 -module load craype-accel-amd-gfx90a -module load rocm/5.4.3 -module load cray-mpich -module load cray-hdf5 -module load cray-python/3.9.13.1 - -# GPU-aware MPI -export MPICH_GPU_SUPPORT_ENABLED=1 - -# optimize ROCm/HIP compilation for MI250X -export AMREX_AMD_ARCH=gfx90a - -# compiler environment hints -export CC=$(which hipcc) -export CXX=$(which hipcc) -export FC=$(which ftn) -export CFLAGS="-I${ROCM_PATH}/include -I${MPICH_DIR}/include -I${HDF5_DIR}/include" -export CXXFLAGS="-I${ROCM_PATH}/include -I${MPICH_DIR}/include -I${HDF5_DIR}/include" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 -L${MPICH_DIR}/lib -lmpi -L${HDF5_DIR}/lib -lhdf5 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" diff --git a/scripts/setonix.profile b/scripts/setonix.profile index 7138354ec..d4b89564a 100644 --- a/scripts/setonix.profile +++ b/scripts/setonix.profile @@ -1,12 +1,12 @@ #!/bin/bash -source /opt/cray/pe/cpe/22.09/restore_lmod_system_defaults.sh +source /opt/cray/pe/cpe/23.03/restore_lmod_system_defaults.sh -module load cmake/3.21.4 +module load cmake/3.24.3 module load craype-accel-amd-gfx90a -module load rocm/5.0.2 +module load rocm/5.2.3 module load cray-mpich -module load cce/14.0.3 +module load cce/15.0.1 module load cray-hdf5 module load cray-python/3.9.13.1 @@ -16,6 +16,9 @@ export MPICH_GPU_SUPPORT_ENABLED=1 # optimize ROCm/HIP compilation for MI250X export AMREX_AMD_ARCH=gfx90a +# allow CMake to find Ascent +export Ascent_DIR=/software/projects/pawsey0807/bwibking/ascent_06082023/install/ascent-develop/lib/cmake/ascent/ + # compiler environment hints export CC=$(which cc) export CXX=$(which CC)