From 92d3f94ee1400e7d924f2cd628f3afc93c465c00 Mon Sep 17 00:00:00 2001 From: seshadri levante Date: Fri, 17 May 2024 10:24:20 +0200 Subject: [PATCH 1/7] Added changes so fesom2 can compile with OpenACC. Have to check the namelist files --- CMakeLists.txt | 2 +- env.sh | 9 +++++++-- env/levante.dkrz.de/shell.nvhpc | 7 ++++--- src/CMakeLists.txt | 13 +++++++------ src/ice_fct.F90 | 4 ++++ 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 42be9544a..92d473451 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.16) # set default build type cache entry (do so before project(...) is called, which would create this cache entry on its own) if(NOT CMAKE_BUILD_TYPE) message(STATUS "setting default build type: Release") - set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") + set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") endif() project(FESOM2.0) diff --git a/env.sh b/env.sh index 6fff86847..4b3aace25 100755 --- a/env.sh +++ b/env.sh @@ -54,7 +54,12 @@ elif [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then STRATEGY="mistral.dkrz.de" elif [[ $LOGINHOST =~ ^levante ]] || [[ $LOGINHOST =~ ^l[:alnum:]+\.lvt\.dkrz\.de$ ]]; then STRATEGY="levante.dkrz.de" -elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then + # following regex only matches if input is 2 word like levante.nvhpc, this enables using different shells for a machine directly + compid_regex="^([[:alnum:]]+)\.([[:alnum:]]+)$" + if [[ $LOGINHOST =~ $compid_regex ]]; then + COMPILERID="${BASH_REMATCH[2]}" + fi + elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then STRATEGY="ollie" elif [[ $LOGINHOST =~ ^albedo[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then STRATEGY="albedo" @@ -112,4 +117,4 @@ else echo "Sourcing $(realpath $SHELLFILE) for environment" source $SHELLFILE echo "$(realpath ${SHELLFILE})" > $DIR/bin/current_shell_path -fi +fi \ No newline at end of file diff --git a/env/levante.dkrz.de/shell.nvhpc b/env/levante.dkrz.de/shell.nvhpc index eb2b776f6..5bf73e871 100755 --- a/env/levante.dkrz.de/shell.nvhpc +++ b/env/levante.dkrz.de/shell.nvhpc @@ -5,9 +5,10 @@ export CPU_MODEL=AMD_EPYC_ZEN3 module --force purge # module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0 # module load openmpi/4.1.2-intel-2021.5.0 -module load nvhpc/22.5-gcc-11.2.0 -module load openmpi/.4.1.4-nvhpc-22.5 +module load nvhpc/23.9-gcc-11.2.0 +module load openmpi/4.1.6-nvhpc-23.9 export FC=mpif90 CC=mpicc CXX=mpicxx; +# export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0 module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0 @@ -28,4 +29,4 @@ export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self export UCX_UNIFIED_MODE=y export HDF5_USE_FILE_LOCKING=FALSE export OMPI_MCA_io="romio321" -export UCX_HANDLE_ERRORS=bt +export UCX_HANDLE_ERRORS=bt \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 509e899a4..c7159844f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -269,16 +269,17 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray ) endif() elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) target_compile_definitions(${PROJECT_NAME} PRIVATE ENABLE_NVHPC_WORKAROUNDS) - target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs) + target_compile_options(${PROJECT_NAME} PRIVATE -Mnofma -Mallocatable=95 -Mr8 -pgf90libs) if(ENABLE_OPENACC) # additional compiler settings - target_compile_options(${PROJECT_NAME} PRIVATE -acc -ta=tesla:${NV_GPU_ARCH} -Minfo=accel) - set(CMAKE_EXE_LINKER_FLAGS "-acc -ta=tesla:${NV_GPU_ARCH}") + message("Taking ENABLE_OPENACC = ON") + target_compile_options(${PROJECT_NAME} PRIVATE -O2 -acc -gpu=${NV_GPU_ARCH} -Minfo=accel) + set(CMAKE_EXE_LINKER_FLAGS "-acc -gpu=${NV_GPU_ARCH}") endif() if(ENABLE_OPENMP) target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast) - else() - target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) + # else() + # target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) endif() endif() @@ -316,4 +317,4 @@ target_link_libraries(${PROJECT_NAME}.x PUBLIC ${PROJECT_NAME}) ### Export and installation -fesom_export(TARGETS ${PROJECT_NAME} parms fesom.x) +fesom_export(TARGETS ${PROJECT_NAME} parms fesom.x) \ No newline at end of file diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index d60bf352d..df7fe3948 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -1122,7 +1122,11 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) call exchange_nod(ice_temp, partit, luse_g2g = .true.) #endif +#ifndef ENABLE_OPENACC +!$OMP PARALLEL DO +#else !$ACC END DATA +#endif !$OMP BARRIER end subroutine ice_fem_fct From 59b0765c657ad9d05af113f9f288990664a6c458 Mon Sep 17 00:00:00 2001 From: seshadri levante Date: Fri, 17 May 2024 14:46:36 +0200 Subject: [PATCH 2/7] Working OpenACC code after commenting 'set(CMAKE_EXE_LINKER_FLAGS)' --- CMakeLists.txt | 2 +- src/CMakeLists.txt | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92d473451..6fb86280a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.16) # set default build type cache entry (do so before project(...) is called, which would create this cache entry on its own) if(NOT CMAKE_BUILD_TYPE) message(STATUS "setting default build type: Release") - set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") endif() project(FESOM2.0) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c7159844f..cc34a529f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,8 +58,9 @@ endif() option(ENABLE_OPENACC "compile with OpenACC support" OFF) message(STATUS "ENABLE_OPENACC: ${ENABLE_OPENACC}") - -set(NV_GPU_ARCH "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") +option(DISABLE_OPENACC_ATOMICS "disable kernels using atomic statement for reproducible results" ON) +set(GPU_COMPUTE_CAPABILITY "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") +set(GPU_FLAGS "cuda12.2,${GPU_COMPUTE_CAPABILITY}" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)") option(ENABLE_OPENMP "build FESOM with OpenMP" OFF) message(STATUS "ENABLE_OPENMP: ${ENABLE_OPENMP}") @@ -256,7 +257,7 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU ) elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray ) #target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O3) target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O2 -hnoacc -M878) #-hnoacc is a workaround for cray automatically activate -hacc, -M878 is to suppress ftn-878 warning - if(ENABLE_OPENMP) + if(${ENABLE_OPENMP}) target_compile_options(${PROJECT_NAME} PRIVATE -homp) else() target_compile_options(${PROJECT_NAME} PRIVATE -hnoomp) @@ -269,14 +270,19 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray ) endif() elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) target_compile_definitions(${PROJECT_NAME} PRIVATE ENABLE_NVHPC_WORKAROUNDS) + #target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs) target_compile_options(${PROJECT_NAME} PRIVATE -Mnofma -Mallocatable=95 -Mr8 -pgf90libs) - if(ENABLE_OPENACC) + if(${ENABLE_OPENACC}) # additional compiler settings message("Taking ENABLE_OPENACC = ON") - target_compile_options(${PROJECT_NAME} PRIVATE -O2 -acc -gpu=${NV_GPU_ARCH} -Minfo=accel) - set(CMAKE_EXE_LINKER_FLAGS "-acc -gpu=${NV_GPU_ARCH}") + target_compile_options(${PROJECT_NAME} PRIVATE -acc -O2 -gpu=${GPU_FLAGS} -Minfo=accel) + # set(CMAKE_EXE_LINKER_FLAGS "-acc -gpu=${GPU_FLAGS}") + if(${DISABLE_OPENACC_ATOMICS}) + message("Taking DISABLE_OPENACC_ATOMICS = ON") + target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_OPENACC_ATOMICS) + endif() endif() - if(ENABLE_OPENMP) + if(${ENABLE_OPENMP}) target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast) # else() # target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) From ff4b4252e412d49a5b280e837f7c6f02d0b7d375 Mon Sep 17 00:00:00 2001 From: seshadri levante Date: Tue, 21 May 2024 13:13:04 +0200 Subject: [PATCH 3/7] Updated the changes regarding unneccesarry spaces and changed all ENABLE_OPENMP to in src/CMakeLists.txt --- CMakeLists.txt | 2 +- env.sh | 2 +- env/levante.dkrz.de/shell.nvhpc | 2 +- src/CMakeLists.txt | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fb86280a..42be9544a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.16) # set default build type cache entry (do so before project(...) is called, which would create this cache entry on its own) if(NOT CMAKE_BUILD_TYPE) message(STATUS "setting default build type: Release") - set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") + set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build, options are: None(CMAKE_CXX_FLAGS or CMAKE_C_FLAGS used) Debug Release RelWithDebInfo MinSizeRel.") endif() project(FESOM2.0) diff --git a/env.sh b/env.sh index 4b3aace25..6c5d9b1f5 100755 --- a/env.sh +++ b/env.sh @@ -117,4 +117,4 @@ else echo "Sourcing $(realpath $SHELLFILE) for environment" source $SHELLFILE echo "$(realpath ${SHELLFILE})" > $DIR/bin/current_shell_path -fi \ No newline at end of file +fi diff --git a/env/levante.dkrz.de/shell.nvhpc b/env/levante.dkrz.de/shell.nvhpc index 5bf73e871..f0ae54531 100755 --- a/env/levante.dkrz.de/shell.nvhpc +++ b/env/levante.dkrz.de/shell.nvhpc @@ -29,4 +29,4 @@ export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self export UCX_UNIFIED_MODE=y export HDF5_USE_FILE_LOCKING=FALSE export OMPI_MCA_io="romio321" -export UCX_HANDLE_ERRORS=bt \ No newline at end of file +export UCX_HANDLE_ERRORS=bt diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cc34a529f..ae6283a09 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -64,7 +64,7 @@ set(GPU_FLAGS "cuda12.2,${GPU_COMPUTE_CAPABILITY}" CACHE STRING "GPU arch for nv option(ENABLE_OPENMP "build FESOM with OpenMP" OFF) message(STATUS "ENABLE_OPENMP: ${ENABLE_OPENMP}") -if(ENABLE_OPENMP) +if(${ENABLE_OPENMP}) find_package(OpenMP REQUIRED COMPONENTS Fortran) endif() @@ -185,7 +185,7 @@ target_link_libraries(${PROJECT_NAME} PRIVATE parms) #metis target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran) set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran) -if(ENABLE_OPENMP) +if(${ENABLE_OPENMP}) target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran) endif() @@ -284,8 +284,8 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC ) endif() if(${ENABLE_OPENMP}) target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast) - # else() - # target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) + else() + target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline) endif() endif() @@ -323,4 +323,4 @@ target_link_libraries(${PROJECT_NAME}.x PUBLIC ${PROJECT_NAME}) ### Export and installation -fesom_export(TARGETS ${PROJECT_NAME} parms fesom.x) \ No newline at end of file +fesom_export(TARGETS ${PROJECT_NAME} parms fesom.x) From 720cff61134523654d04a96c3edcee622ee01ade Mon Sep 17 00:00:00 2001 From: seshadri levante Date: Tue, 16 Jul 2024 14:39:18 +0200 Subject: [PATCH 4/7] Updated with Miguels's comment --- src/ice_fct.F90 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ice_fct.F90 b/src/ice_fct.F90 index df7fe3948..7acb4ff1f 100755 --- a/src/ice_fct.F90 +++ b/src/ice_fct.F90 @@ -1122,9 +1122,7 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh) call exchange_nod(ice_temp, partit, luse_g2g = .true.) #endif -#ifndef ENABLE_OPENACC -!$OMP PARALLEL DO -#else +#ifdef ENABLE_OPENACC !$ACC END DATA #endif From 3389483e3fb0134b6ed86d57f07524157ed58c2d Mon Sep 17 00:00:00 2001 From: seshadri levante Date: Wed, 17 Jul 2024 16:11:05 +0200 Subject: [PATCH 5/7] Now, we can compile on levante with intel and gnu --- env/levante.dkrz.de/shell.intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/env/levante.dkrz.de/shell.intel b/env/levante.dkrz.de/shell.intel index e087c7f84..777e55f0d 100755 --- a/env/levante.dkrz.de/shell.intel +++ b/env/levante.dkrz.de/shell.intel @@ -9,7 +9,7 @@ module load openmpi/4.1.2-intel-2021.5.0 export FC=mpif90 CC=mpicc CXX=mpicxx ; spack load intel-oneapi-mkl@2022.0.1%gcc@11.2.0 # this handles adding to path elegantly then using hardcoded path below #module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0 -#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0 module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0 From 372b5de70b77e1e319efb403e474d72dd1bcf425 Mon Sep 17 00:00:00 2001 From: seshadri levante Date: Wed, 17 Jul 2024 16:17:17 +0200 Subject: [PATCH 6/7] Added suvi's changes back to env.sh (compilerId regex for levante) --- env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/env.sh b/env.sh index 6c5d9b1f5..6ab2d3799 100755 --- a/env.sh +++ b/env.sh @@ -54,12 +54,12 @@ elif [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then STRATEGY="mistral.dkrz.de" elif [[ $LOGINHOST =~ ^levante ]] || [[ $LOGINHOST =~ ^l[:alnum:]+\.lvt\.dkrz\.de$ ]]; then STRATEGY="levante.dkrz.de" - # following regex only matches if input is 2 word like levante.nvhpc, this enables using different shells for a machine directly + # following regex only matches if input is 2 word like levante.nvhpc compid_regex="^([[:alnum:]]+)\.([[:alnum:]]+)$" if [[ $LOGINHOST =~ $compid_regex ]]; then COMPILERID="${BASH_REMATCH[2]}" fi - elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then +elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then STRATEGY="ollie" elif [[ $LOGINHOST =~ ^albedo[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then STRATEGY="albedo" From dd467cb0bdf878fd09e75ae080ad553f32a16814 Mon Sep 17 00:00:00 2001 From: seshadri levante Date: Wed, 25 Sep 2024 12:13:55 +0200 Subject: [PATCH 7/7] Updated shell.intel and shell.nvhpc --- env/levante.dkrz.de/shell.intel | 2 +- env/levante.dkrz.de/shell.nvhpc | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/env/levante.dkrz.de/shell.intel b/env/levante.dkrz.de/shell.intel index 777e55f0d..e087c7f84 100755 --- a/env/levante.dkrz.de/shell.intel +++ b/env/levante.dkrz.de/shell.intel @@ -9,7 +9,7 @@ module load openmpi/4.1.2-intel-2021.5.0 export FC=mpif90 CC=mpicc CXX=mpicxx ; spack load intel-oneapi-mkl@2022.0.1%gcc@11.2.0 # this handles adding to path elegantly then using hardcoded path below #module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0 -export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH +#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0 module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0 diff --git a/env/levante.dkrz.de/shell.nvhpc b/env/levante.dkrz.de/shell.nvhpc index f0ae54531..5f9bf063b 100755 --- a/env/levante.dkrz.de/shell.nvhpc +++ b/env/levante.dkrz.de/shell.nvhpc @@ -8,7 +8,6 @@ module --force purge module load nvhpc/23.9-gcc-11.2.0 module load openmpi/4.1.6-nvhpc-23.9 export FC=mpif90 CC=mpicc CXX=mpicxx; -# export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0 module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0