diff --git a/env/levante.dkrz.de/shell.nvhpc b/env/levante.dkrz.de/shell.nvhpc
index eb2b776f6..5f9bf063b 100755
--- a/env/levante.dkrz.de/shell.nvhpc
+++ b/env/levante.dkrz.de/shell.nvhpc
@@ -5,8 +5,8 @@ export CPU_MODEL=AMD_EPYC_ZEN3
 module --force purge 
 # module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0
 # module load openmpi/4.1.2-intel-2021.5.0
-module load nvhpc/22.5-gcc-11.2.0
-module load openmpi/.4.1.4-nvhpc-22.5
+module load nvhpc/23.9-gcc-11.2.0
+module load openmpi/4.1.6-nvhpc-23.9
 export FC=mpif90 CC=mpicc CXX=mpicxx;
 
 module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c3e698e7f..c526b573a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -64,12 +64,13 @@ endif()
 
 option(ENABLE_OPENACC "compile with OpenACC support" OFF)
 message(STATUS "ENABLE_OPENACC: ${ENABLE_OPENACC}")
-
-set(NV_GPU_ARCH "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)")
+option(DISABLE_OPENACC_ATOMICS "disable kernels using atomic statement for reproducible results" ON)
+set(GPU_COMPUTE_CAPABILITY "cc80" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)")
+set(GPU_FLAGS "cuda12.2,${GPU_COMPUTE_CAPABILITY}" CACHE STRING "GPU arch for nvfortran compiler (cc35,cc50,cc60,cc70,cc80,...)")
 
 option(ENABLE_OPENMP "build FESOM with OpenMP" OFF)
 message(STATUS "ENABLE_OPENMP: ${ENABLE_OPENMP}")
-if(ENABLE_OPENMP)
+if(${ENABLE_OPENMP})
    find_package(OpenMP REQUIRED COMPONENTS Fortran)
 endif()
 
@@ -186,7 +187,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC $<INSTALL_INTERFACE:module/fes
 target_link_libraries(${PROJECT_NAME} PRIVATE MPI::MPI_Fortran)
 
 set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE Fortran)
-if(ENABLE_OPENMP)
+if(${ENABLE_OPENMP})
    target_link_libraries(${PROJECT_NAME} PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 
@@ -275,7 +276,7 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL  GNU )
 elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray )
 	#target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O3)
    target_compile_options(${PROJECT_NAME} PRIVATE -c -emf -hbyteswapio -hflex_mp=conservative -hfp1 -hadd_paren -Ounroll0 -hipa0 -r am -s real64 -N 1023 -g -G2 -O2 -hnoacc -M878)  #-hnoacc is a workaround for cray automatically activate -hacc, -M878 is to suppress ftn-878 warning
-   if(ENABLE_OPENMP)
+   if(${ENABLE_OPENMP})
       target_compile_options(${PROJECT_NAME} PRIVATE -homp)
    else()
       target_compile_options(${PROJECT_NAME} PRIVATE -hnoomp)
@@ -288,13 +289,19 @@ elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL Cray )
    endif()
 elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL NVHPC )
    target_compile_definitions(${PROJECT_NAME} PRIVATE ENABLE_NVHPC_WORKAROUNDS)
-   target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs)
-   if(ENABLE_OPENACC)
+   #target_compile_options(${PROJECT_NAME} PRIVATE -fast -fastsse -O3 -Mallocatable=95 -Mr8 -pgf90libs)
+   target_compile_options(${PROJECT_NAME} PRIVATE -Mnofma -Mallocatable=95 -Mr8 -pgf90libs)
+   if(${ENABLE_OPENACC})
       # additional compiler settings
-      target_compile_options(${PROJECT_NAME} PRIVATE -acc -ta=tesla:${NV_GPU_ARCH} -Minfo=accel)
-      set(CMAKE_EXE_LINKER_FLAGS "-acc -ta=tesla:${NV_GPU_ARCH}")
+      message("Taking ENABLE_OPENACC = ON")
+      target_compile_options(${PROJECT_NAME} PRIVATE -acc -O2 -gpu=${GPU_FLAGS} -Minfo=accel)
+      # set(CMAKE_EXE_LINKER_FLAGS "-acc -gpu=${GPU_FLAGS}")
+      if(${DISABLE_OPENACC_ATOMICS})
+         message("Taking DISABLE_OPENACC_ATOMICS = ON")
+          target_compile_definitions(${PROJECT_NAME} PRIVATE DISABLE_OPENACC_ATOMICS)
+      endif()
    endif()
-   if(ENABLE_OPENMP)
+   if(${ENABLE_OPENMP})
       target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast)
    else()
       target_compile_options(${PROJECT_NAME} PRIVATE -Mipa=fast,inline)
diff --git a/src/ice_fct.F90 b/src/ice_fct.F90
index 5e0b87093..be4d92518 100755
--- a/src/ice_fct.F90
+++ b/src/ice_fct.F90
@@ -1128,7 +1128,9 @@ subroutine ice_fem_fct(tr_array_id, ice, partit, mesh)
     call exchange_nod(ice_temp, partit, luse_g2g = .true.)
 #endif
 
+#ifdef ENABLE_OPENACC
 !$ACC END DATA
+#endif
 
 !$OMP BARRIER
 end subroutine ice_fem_fct