From cd8a9e5f12c7d82a387666ef7164a072c3ad54be Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 29 Oct 2024 03:52:00 +0000 Subject: [PATCH 1/5] Move -Xsycl-target-backend to link flags --- cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake | 3 ++- cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index 6835515164f0..f83db48a244c 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake index 6835515164f0..f83db48a244c 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake @@ -3,5 +3,6 @@ string(APPEND CMAKE_EXE_LINKER_FLAGS " -lmkl_intel_lp64 -lmkl_sequential -lmkl_c if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() -string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 -Xsycl-target-backend \"-device 12.60.7\"") +string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") +string(APPEND CMAKE_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") From 8dd30e273d5be77c9f8691a9dc62a551ac312b10 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 29 Oct 2024 03:57:56 +0000 Subject: [PATCH 2/5] Fix SYCL and PBS flags --- components/omega/CMakeLists.txt | 2 +- components/omega/OmegaBuild.cmake | 27 +++++++++++++++++---------- components/omega/src/CMakeLists.txt | 4 ++-- components/omega/src/base/DataTypes.h | 2 ++ components/omega/test/CMakeLists.txt | 24 ++++++++++++++++-------- 5 files changed, 38 insertions(+), 21 deletions(-) diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt index c771a0f9a1d3..d728158c213f 100644 --- a/components/omega/CMakeLists.txt +++ b/components/omega/CMakeLists.txt @@ -44,7 +44,7 @@ if (NOT DEFINED PROJECT_NAME) set(CMAKE_CXX_STANDARD 17) # used in E3SM set(CMAKE_CXX_STANDARD_REQUIRED ON) - set(LINKER_LANGUAGE C) # needed to support Cray C compiler wrapper + set(LINKER_LANGUAGE CXX) # needed to support Cray C compiler wrapper # update variables for standalone build setup_standalone_build() diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index 5e1b67487103..2051d3ab49b8 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -29,6 +29,10 @@ macro(common) option(OMEGA_DEBUG "Turn on error message throwing (default OFF)." OFF) option(OMEGA_LOG_FLUSH "Turn on unbuffered logging (default OFF)." OFF) + if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG") + set(OMEGA_DEBUG ON) + endif() + if(NOT DEFINED OMEGA_CXX_FLAGS) set(OMEGA_CXX_FLAGS "") endif() @@ -120,7 +124,7 @@ macro(read_cime_config) break() elseif("${arg}" STREQUAL "-n" OR "${arg}" STREQUAL "-N" OR - "${arg}" STREQUAL "-c") + "${arg}" STREQUAL "-c" OR "${arg}" STREQUAL "-np") set(SKIP_ARG TRUE) else() @@ -219,6 +223,9 @@ macro(init_standalone_build) elseif(USE_HIP) set(OMEGA_ARCH "HIP") + elseif(USE_SYCL) + set(OMEGA_ARCH "SYCL") + else() execute_process( @@ -294,7 +301,11 @@ macro(init_standalone_build) set(_CtestScript ${OMEGA_BUILD_DIR}/omega_ctest.sh) file(WRITE ${_CtestScript} "#!/usr/bin/env bash\n\n") file(APPEND ${_CtestScript} "source ./omega_env.sh\n\n") - file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n") + if(OMEGA_DEBUG) + file(APPEND ${_CtestScript} "ctest --output-on-failure --verbose $* # --rerun-failed\n\n") + else() + file(APPEND ${_CtestScript} "ctest --output-on-failure $* # --rerun-failed\n\n") + endif() # create a profile script set(_ProfileScript ${OMEGA_BUILD_DIR}/omega_profile.sh) @@ -422,10 +433,10 @@ macro(init_standalone_build) file(APPEND ${_ProfileScript} " -o \$OUTFILE ./src/omega.exe 1000") elseif("${OMEGA_ARCH}" STREQUAL "SYCL") - set(CMAKE_CXX_COMPILER ${OMEGA_SYCL_COMPILER}) + set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) - if(OMEGA_SYCL_FLAGS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OMEGA_SYCL_FLAGS}") + if(SYCL_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SYCL_FLAGS}") endif() else() @@ -458,7 +469,7 @@ macro(init_standalone_build) message(STATUS "CMAKE_CXX_COMPILER = ${CMAKE_CXX_COMPILER}") message(STATUS "CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}") -# message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}") + message(STATUS "CMAKE_EXE_LINKER_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}") endmacro() @@ -512,10 +523,6 @@ macro(update_variables) add_definitions(-DOMEGA_BUILD_MODE=${OMEGA_BUILD_MODE}) - if("${OMEGA_BUILD_TYPE}" STREQUAL "Debug" OR "${OMEGA_BUILD_TYPE}" STREQUAL "DEBUG") - set(OMEGA_DEBUG ON) - endif() - if(NOT DEFINED OMEGA_LOG_LEVEL) set(OMEGA_LOG_LEVEL "INFO") endif() diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt index fc53ed481a0c..2c70b81bcc70 100644 --- a/components/omega/src/CMakeLists.txt +++ b/components/omega/src/CMakeLists.txt @@ -53,7 +53,7 @@ target_link_libraries( if(GKlib_FOUND) target_link_libraries( OmegaLibFlags - PUBLIC + INTERFACE gklib ) endif() @@ -106,6 +106,6 @@ if(OMEGA_BUILD_EXECUTABLE) OmegaLibFlags ) - set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE C) + set_target_properties(${OMEGA_EXE_NAME} PROPERTIES LINKER_LANGUAGE CXX) endif() diff --git a/components/omega/src/base/DataTypes.h b/components/omega/src/base/DataTypes.h index 11e35aa404e8..4b91c3592581 100644 --- a/components/omega/src/base/DataTypes.h +++ b/components/omega/src/base/DataTypes.h @@ -43,6 +43,8 @@ KOKKOS_INLINE_FUNCTION constexpr Real operator""_Real(long double x) { using MemSpace = Kokkos::CudaSpace; #elif OMEGA_ENABLE_HIP using MemSpace = Kokkos::Experimental::HIPSpace; +#elif OMEGA_ENABLE_SYCL +using MemSpace = Kokkos::Experimental::SYCLDeviceUSMSpace; #elif OMEGA_ENABLE_OPENMP using MemSpace = Kokkos::HostSpace; #elif OMEGA_ENABLE_SERIAL diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 91be32856b4a..66c385d9667c 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -46,14 +46,22 @@ function(add_omega_test test_name exe_name source_files mpi_args) ) endif() - set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE C) + set_target_properties(${exe_name} PROPERTIES LINKER_LANGUAGE CXX) # Add the test command if (mpi_args) - add_test( - NAME ${test_name} - COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name} - ) + + if("${OMEGA_ARCH}" STREQUAL "SYCL") + add_test( + NAME ${test_name} + COMMAND ${OMEGA_MPI_EXEC} ${mpi_args} ${OMEGA_MPI_ARGS} ./${exe_name} + ) + else() + add_test( + NAME ${test_name} + COMMAND ${OMEGA_MPI_EXEC} ${OMEGA_MPI_ARGS} ${mpi_args} -- ./${exe_name} + ) + endif() else() add_test( @@ -281,7 +289,7 @@ add_omega_test( TEND_PLANE_TEST testTendencyTermsPlane.exe ocn/TendencyTermsTest.cpp - "-n 8;--cpu-bind=cores" + "-n 8;" ) target_compile_definitions( testTendencyTermsPlane.exe @@ -301,7 +309,7 @@ add_omega_test( TEND_SPHERE_TEST testTendencyTermsSphere.exe ocn/TendencyTermsTest.cpp - "-n 8;--cpu-bind=cores" + "-n 8;" ) target_compile_definitions( testTendencyTermsSphere.exe @@ -350,7 +358,7 @@ add_omega_test( REDUCTIONS_TEST testReductions.exe base/ReductionsTest.cpp - "-n;2;--cpu-bind=cores" + "-n;2;" ) ################### From a9c2fd834926bbe9ca634b4342326c43d6624aa2 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Tue, 29 Oct 2024 04:01:43 +0000 Subject: [PATCH 3/5] Default to PIO_IOTYPE_PNETCDF to work around HDF5 in SYCL runs --- components/omega/src/base/IO.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/omega/src/base/IO.h b/components/omega/src/base/IO.h index 83ed72a3f6c2..8ca557b246f0 100644 --- a/components/omega/src/base/IO.h +++ b/components/omega/src/base/IO.h @@ -71,7 +71,7 @@ enum FileFmt { FmtHDF5 = PIO_IOTYPE_HDF5, ///< native HDF5 format FmtADIOS = PIO_IOTYPE_ADIOS, ///< ADIOS format FmtUnknown = -1, ///< Unknown or undefined - FmtDefault = PIO_IOTYPE_NETCDF4C, ///< NetCDF4 is default + FmtDefault = PIO_IOTYPE_PNETCDF, ///< work around HDF5 in SYCL runs }; /// File operations From be5c90cd249070c154fb00b2f395c61e19df3a53 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Wed, 11 Dec 2024 21:27:41 +0000 Subject: [PATCH 4/5] Move SYCL link flags to OMEGA_SYCL_EXE_LINKER_FLAGS --- cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake | 2 +- cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake | 2 +- components/omega/OmegaBuild.cmake | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake index f83db48a244c..1ad0d6e6b61a 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_aurora.cmake @@ -4,5 +4,5 @@ if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake index f83db48a244c..1ad0d6e6b61a 100644 --- a/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake +++ b/cime_config/machines/cmake_macros/oneapi-ifxgpu_sunspot.cmake @@ -4,5 +4,5 @@ if (compile_threaded) string(APPEND CMAKE_EXE_LINKER_FLAGS " -fiopenmp -fopenmp-targets=spir64") endif() string(APPEND SYCL_FLAGS " -\-intel -fsycl -fsycl-targets=spir64_gen -mlong-double-64 ") -string(APPEND CMAKE_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") +string(APPEND OMEGA_SYCL_EXE_LINKER_FLAGS " -Xsycl-target-backend \"-device 12.60.7\" ") string(APPEND CMAKE_CXX_FLAGS " -Xclang -fsycl-allow-virtual-functions") diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake index 2051d3ab49b8..a81da6e59d41 100644 --- a/components/omega/OmegaBuild.cmake +++ b/components/omega/OmegaBuild.cmake @@ -435,9 +435,13 @@ macro(init_standalone_build) elseif("${OMEGA_ARCH}" STREQUAL "SYCL") set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) + # add flags from upstream-E3SM if(SYCL_FLAGS) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SYCL_FLAGS}") endif() + if(OMEGA_SYCL_EXE_LINKER_FLAGS) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OMEGA_SYCL_EXE_LINKER_FLAGS}") + endif() else() set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER}) From e4f899d1d822b9054eea909f4973929bc832c568 Mon Sep 17 00:00:00 2001 From: Azamat Mametjanov Date: Fri, 13 Dec 2024 22:02:12 +0000 Subject: [PATCH 5/5] Clean-up slurm-specific core binding in a ctest --- components/omega/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/omega/test/CMakeLists.txt b/components/omega/test/CMakeLists.txt index 66c385d9667c..eaf2890bd224 100644 --- a/components/omega/test/CMakeLists.txt +++ b/components/omega/test/CMakeLists.txt @@ -301,7 +301,7 @@ add_omega_test( TEND_PLANE_SINGLE_PRECISION_TEST testTendencyTermsPlaneSinglePrecision.exe ocn/TendencyTermsTest.cpp - "-n 8;--cpu-bind=cores" + "-n 8;" single_precision )