Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Production de openacc test #589

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Production de openacc test #589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
92d3f94
Added changes so fesom2 can compile with OpenACC. Have to check the n…
May 17, 2024
59b0765
Working OpenACC code after commenting 'set(CMAKE_EXE_LINKER_FLAGS)'
May 17, 2024
ff4b425
Updated the changes regarding unneccesarry spaces and changed all ENA…
May 21, 2024
24127a9
Merge branch 'production_DE' into production_DE_openacc_test
Jun 24, 2024
720cff6
Updated with Miguels's comment
Jul 16, 2024
3389483
Now, we can compile on levante with intel and gnu
Jul 17, 2024
372b5de
Added suvi's changes back to env.sh (compilerId regex for levante)
Jul 17, 2024
624a780
Merge branch 'main' into production_DE_openacc_test
basava70 Aug 26, 2024
a6a0ba4
Merge branch 'main' into production_DE_openacc_test
basava70 Sep 25, 2024
dd467cb
Updated shell.intel and shell.nvhpc
Sep 25, 2024
25692cf
Merge branch 'production_DE_openacc_test' of github.com:FESOM/fesom2 …
Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions env/levante.dkrz.de/shell.nvhpc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ export CPU_MODEL=AMD_EPYC_ZEN3
module --force purge
# module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0
# module load openmpi/4.1.2-intel-2021.5.0
module load nvhpc/22.5-gcc-11.2.0
module load openmpi/.4.1.4-nvhpc-22.5
module load nvhpc/23.9-gcc-11.2.0
module load openmpi/4.1.6-nvhpc-23.9
export FC=mpif90 CC=mpicc CXX=mpicxx;

module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
Expand Down
27 changes: 17 additions & 10 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,13 @@ endif()

option(ENABLE_OPENACC "compile with OpenACC support" OFF)
message(STATUS "ENABLE_OPENACC: ${ENABLE_OPENACC}")

set(NV_GPU_ARCH "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)")
option(DISABLE_OPENACC_ATOMICS "disable kernels using atomic statement for reproducible results" ON)
set(GPU_COMPUTE_CAPABILITY "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)")
set(GPU_FLAGS "cuda12.2,${GPU_COMPUTE_CAPABILITY}" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)")
Comment on lines -67 to +69
Copy link
Collaborator

@mandresm mandresm May 21, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@suvarchal, this was different from a previous version that was working on Levante. @basava70 replaced this line to have a variable that contains not only cc80 but cuda12.2,cc80 that is then passed to the -gpu flag during compilation. Does this change make sense to you?


option(ENABLE_OPENMP "build FESOM with OpenMP" OFF)
message(STATUS "ENABLE_OPENMP: ${ENABLE_OPENMP}")
if(ENABLE_OPENMP)
if(${ENABLE_OPENMP})
find_package(OpenMP REQUIRED COMPONENTS Fortran)
endif()

Expand Down Expand Up @@ -186,7 +187,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC $<INSTALL_INTERFACE:module/fes
target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran)

set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran)
if(ENABLE_OPENMP)
if(${ENABLE_OPENMP})
target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran)
endif()

Expand Down Expand Up @@ -275,7 +276,7 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL GNU )
elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray )
#target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O3)
target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O2 -hnoacc -M878) #-hnoacc is a workaround for cray automatically activate -hacc, -M878 is to suppress ftn-878 warning
if(ENABLE_OPENMP)
if(${ENABLE_OPENMP})
target_compile_options(${PROJECT_NAME} PRIVATE -homp)
else()
target_compile_options(${PROJECT_NAME} PRIVATE -hnoomp)
Expand All @@ -288,13 +289,19 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray )
endif()
elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC )
target_compile_definitions(${PROJECT_NAME} PRIVATE ENABLE_NVHPC_WORKAROUNDS)
target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs)
if(ENABLE_OPENACC)
#target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs)
target_compile_options(${PROJECT_NAME} PRIVATE -Mnofma -Mallocatable=95 -Mr8 -pgf90libs)
if(${ENABLE_OPENACC})
# additional compiler settings
target_compile_options(${PROJECT_NAME} PRIVATE -acc -ta=tesla:${NV_GPU_ARCH} -Minfo=accel)
set(CMAKE_EXE_LINKER_FLAGS "-acc -ta=tesla:${NV_GPU_ARCH}")
message("Taking ENABLE_OPENACC = ON")
target_compile_options(${PROJECT_NAME} PRIVATE -acc -O2 -gpu=${GPU_FLAGS} -Minfo=accel)
# set(CMAKE_EXE_LINKER_FLAGS "-acc -gpu=${GPU_FLAGS}")
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@suvarchal @dsidoren @mandresm
I had to comment this particular line for the OpenACC code to work. I am not sure why. If anyone can let me know why, it will be great. But the code is running with this change.

if(${DISABLE_OPENACC_ATOMICS})
message("Taking DISABLE_OPENACC_ATOMICS = ON")
target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_OPENACC_ATOMICS)
endif()
endif()
if(ENABLE_OPENMP)
if(${ENABLE_OPENMP})
target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast)
else()
target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline)
Expand Down
2 changes: 2 additions & 0 deletions src/ice_fct.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1128,7 +1128,9 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh)
call exchange_nod(ice_temp, partit, luse_g2g = .true.)
#endif

#ifdef ENABLE_OPENACC
!$ACC END DATA
#endif

!$OMP BARRIER
end subroutine ice_fem_fct
Expand Down
Loading