diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index 6835515164f0..1ad0d6e6b61a 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake index 6835515164f0..1ad0d6e6b61a 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake @@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt index c771a0f9a1d3..d728158c213f 100644 --- a/components/omega/CMakeLists.txt +++ b/components/omega/CMakeLists.txt @@ -44,7 +44,7 @@ if (NOT DEFINED PROJECT_NAME) set(CMAKE_CXX_STANDARD 17) # used in E3SM set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(LINKER_LANGUAGE C) # needed to support Cray C compiler wrapper + set(LINKER_LANGUAGE CXX) # needed to support Cray C compiler wrapper # update variables for standalone build setup_standalone_build() diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index 5e1b67487103..a81da6e59d41 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -29,6 +29,10 @@ macro(common) option(OMEGA_DEBUG "Turn on error message throwing (default OFF)." OFF) option(OMEGA_LOG_FLUSH "Turn on unbuffered logging (default OFF)." OFF) + if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG") + set(OMEGA_DEBUG ON) + endif() + if(NOT DEFINED OMEGA_CXX_FLAGS) set(OMEGA_CXX_FLAGS "") endif() @@ -120,7 +124,7 @@ macro(read_cime_config) break() elseif("${arg}" STREQUAL "-n" OR "${arg}" STREQUAL "-N" OR - "${arg}" STREQUAL "-c") + "${arg}" STREQUAL "-c" OR "${arg}" STREQUAL "-np") set(SKIP_ARG TRUE) else() @@ -219,6 +223,9 @@ macro(init_standalone_build) elseif(USE_HIP) set(OMEGA_ARCH "HIP") + elseif(USE_SYCL) + set(OMEGA_ARCH "SYCL") + else() execute_process( @@ -294,7 +301,11 @@ macro(init_standalone_build) set(_CtestScript ${OMEGA_BUILD_DIR}/omega_ctest.sh) file(WRITE ${_CtestScript} "#!/usr/bin/env bash\n\n") file(APPEND ${_CtestScript} "source ./omega_env.sh\n\n") - file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n") + if(OMEGA_DEBUG) + file(APPEND ${_CtestScript} "ctest --output-on-failure --verbose $* # --rerun-failed\n\n") + else() + file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n") + endif() # create a profile script set(_ProfileScript ${OMEGA_BUILD_DIR}/omega_profile.sh) @@ -422,10 +433,14 @@ macro(init_standalone_build) file(APPEND ${_ProfileScript} " -o \$OUTFILE ./src/omega.exe 1000") elseif("${OMEGA_ARCH}" STREQUAL "SYCL") - set(CMAKE_CXX_COMPILER ${OMEGA_SYCL_COMPILER}) + set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) - if(OMEGA_SYCL_FLAGS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_SYCL_FLAGS}") + # add flags from upstream-E3SM + if(SYCL_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SYCL_FLAGS}") + endif() + if(OMEGA_SYCL_EXE_LINKER_FLAGS) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OMEGA_SYCL_EXE_LINKER_FLAGS}") endif() else() @@ -458,7 +473,7 @@ macro(init_standalone_build) message(STATUS "CMAKE_CXX_COMPILER = ${CMAKE_CXX_COMPILER}") message(STATUS "CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}") -# message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}") + message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}") endmacro() @@ -512,10 +527,6 @@ macro(update_variables) add_definitions(-DOMEGA_BUILD_MODE=${OMEGA_BUILD_MODE}) - if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG") - set(OMEGA_DEBUG ON) - endif() - if(NOT DEFINED OMEGA_LOG_LEVEL) set(OMEGA_LOG_LEVEL "INFO") endif() diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt index fc53ed481a0c..2c70b81bcc70 100644 --- a/components/omega/src/CMakeLists.txt +++ b/components/omega/src/CMakeLists.txt @@ -53,7 +53,7 @@ target_link_libraries( if(GKlib_FOUND) target_link_libraries( OmegaLibFlags - PUBLIC + INTERFACE gklib ) endif() @@ -106,6 +106,6 @@ if(OMEGA_BUILD_EXECUTABLE) OmegaLibFlags ) - set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE C) + set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE CXX) endif() diff --git a/components/omega/src/base/DataTypes.h b/components/omega/src/base/DataTypes.h index 11e35aa404e8..4b91c3592581 100644 --- a/components/omega/src/base/DataTypes.h +++ b/components/omega/src/base/DataTypes.h @@ -43,6 +43,8 @@ KOKKOS_INLINE_FUNCTION constexpr Real operator""_Real(long double x) { using MemSpace = Kokkos::CudaSpace; #elif OMEGA_ENABLE_HIP using MemSpace = Kokkos::Experimental::HIPSpace; +#elif OMEGA_ENABLE_SYCL +using MemSpace = Kokkos::Experimental::SYCLDeviceUSMSpace; #elif OMEGA_ENABLE_OPENMP using MemSpace = Kokkos::HostSpace; #elif OMEGA_ENABLE_SERIAL diff --git a/components/omega/src/base/IO.h b/components/omega/src/base/IO.h index 83ed72a3f6c2..8ca557b246f0 100644 --- a/components/omega/src/base/IO.h +++ b/components/omega/src/base/IO.h @@ -71,7 +71,7 @@ enum FileFmt { FmtHDF5 = PIO_IOTYPE_HDF5, ///< native HDF5 format FmtADIOS = PIO_IOTYPE_ADIOS, ///< ADIOS format FmtUnknown = -1, ///< Unknown or undefined - FmtDefault = PIO_IOTYPE_NETCDF4C, ///< NetCDF4 is default + FmtDefault = PIO_IOTYPE_PNETCDF, ///< work around HDF5 in SYCL runs }; /// File operations diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 91be32856b4a..eaf2890bd224 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -46,14 +46,22 @@ function(add_omega_test test_name exe_name source_files mpi_args) ) endif() - set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE C) + set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE CXX) # Add the test command if (mpi_args) - add_test( - NAME ${test_name} - COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name} - ) + + if("${OMEGA_ARCH}" STREQUAL "SYCL") + add_test( + NAME ${test_name} + COMMAND ${OMEGA_MPI_EXEC} ${mpi_args} ${OMEGA_MPI_ARGS} ./${exe_name} + ) + else() + add_test( + NAME ${test_name} + COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name} + ) + endif() else() add_test( @@ -281,7 +289,7 @@ add_omega_test( TEND_PLANE_TEST testTendencyTermsPlane.exe ocn/TendencyTermsTest.cpp - "-n 8;--cpu-bind=cores" + "-n 8;" ) target_compile_definitions( testTendencyTermsPlane.exe @@ -293,7 +301,7 @@ add_omega_test( TEND_PLANE_SINGLE_PRECISION_TEST testTendencyTermsPlaneSinglePrecision.exe ocn/TendencyTermsTest.cpp - "-n 8;--cpu-bind=cores" + "-n 8;" single_precision ) @@ -301,7 +309,7 @@ add_omega_test( TEND_SPHERE_TEST testTendencyTermsSphere.exe ocn/TendencyTermsTest.cpp - "-n 8;--cpu-bind=cores" + "-n 8;" ) target_compile_definitions( testTendencyTermsSphere.exe @@ -350,7 +358,7 @@ add_omega_test( REDUCTIONS_TEST testReductions.exe base/ReductionsTest.cpp - "-n;2;--cpu-bind=cores" + "-n;2;" ) ###################