Merge branch 'jgfouca/cmake_names_in_macros' into master (PR #6014)

Use CMake variables/names in our cmake macros

This should be the last major PR in CMAKE V2 effort.

Changes Summary:
* Macro structure changes
* Add post-process step to set final values for a few items
* Make cmake conversion more robust by storing pre-macro values and comparing instead of just looking at new variables
* Change all macros to set variables that cmake knows about! This is a huge change with wide impacts
* We no longer have to set flag properties on all files. This simplifies things but makes it hard to support e3sm_remove_flags so that feature has been removed. You should always be able to append flags to achieve the same effect as removing a flag.
* Try to remove as much of the linker flag micromanagement as possible
* We now default to cxx linker. Intel and PGI still have to use fortran unfortunately since our main function is in fortran
* Remove as many non-cmake settings as possible from the macros
* Add script for converting macros from the old style to new style
* Upgrade the compare-flags tool to support parsing the e3sm.bldlog file. This is a tougher way to compare flags since you have to build the cases with -j 1, but it's more robust than looking at cmake internal files.
* Remove lots of unneeded stuff from main cmake files (build_model and common_setup)
* Remove unused old makefiles for COSP and MPAS
* Move gptl build to cmake

Testing:
* mappy_gnu: e3sm_developer, with full flag compare on a couple big cases
* anlgce: e3sm_developer builds
* pm-cpu_intel: e3sm_developer, with full flag compare on one big case
* chrysalis_intel: e3sm_developer, with full flag compare on one big case
* summit: build one big case with pgigpu, ibmgpu, pgi

[BFB]

cime_config/customize/e3sm_compile_wrap.py

@@ -21,10 +21,7 @@ def run_cmd(args):
 
     arglen = len(args)
     target = None
-    for (
-        idx,
-        arg,
-    ) in enumerate(args):
+    for idx, arg in enumerate(args):
         if arg == "-o" and idx + 1 < arglen:
             target = args[idx + 1]
             break
@@ -41,13 +38,11 @@ def run_cmd(args):
 
     return result
 
-
 ###############################################################################
 def parse_command_line(args, _):
 ###############################################################################
     return args[1:]
 
-
 ###############################################################################
 def _main_func(description):
 ###############################################################################
@@ -57,7 +52,6 @@ def _main_func(description):
 
     sys.exit(result)
 
-
 ###############################################################################
 
 if __name__ == "__main__":

cime_config/machines/Depends.anlgce.gnu.cmake

@@ -4,8 +4,7 @@ set(NOOPT
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS NOOPT)
-    e3sm_remove_flags("${ITEM}" "-O2")
-    e3sm_remove_flags("${ITEM}" "-O")
+    e3sm_deoptimize_file(${ITEM})
   endforeach()
 endif()
 

cime_config/machines/Depends.chrysalis.intel.cmake

@@ -10,7 +10,6 @@ set(O2MODELSRC
 )
 if (NOT DEBUG)
   foreach(ITEM IN LISTS O2MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O3")
     e3sm_add_flags("${ITEM}" "-O2")
   endforeach()
 endif()

cime_config/machines/Depends.cray.cmake

@@ -8,7 +8,6 @@ set(O0MODELSRC
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS O0MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O1")
     e3sm_add_flags("${ITEM}" "-O0 -vector0")
   endforeach()
 

cime_config/machines/Depends.crayclanggpu.cmake

@@ -17,12 +17,9 @@ set(O2MODELSRC
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS O2MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O3")
     e3sm_add_flags("${ITEM}" "-O2")
   endforeach()
   foreach(ITEM IN LISTS O1MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O3")
-    e3sm_remove_flags("${ITEM}" "-O2")
     e3sm_add_flags("${ITEM}" "-O1")
   endforeach()
 endif()

cime_config/machines/Depends.pgi.cmake

@@ -12,13 +12,10 @@ set(O0MODELSRC
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS O1MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O2")
     e3sm_add_flags("${ITEM}" "-O1 -Mnovect")
   endforeach()
 
   foreach(ITEM IN LISTS O0MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O2")
-    e3sm_remove_flags("${ITEM}" "-O1")
     e3sm_add_flags("${ITEM}" "-O0 -Mnovect")
   endforeach()
 endif()

cime_config/machines/Depends.pgigpu.cmake

@@ -19,13 +19,10 @@ set(O0MODELSRC
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS O1MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O2")
     e3sm_add_flags("${ITEM}" "-O1 -Mnovect")
   endforeach()
 
   foreach(ITEM IN LISTS O0MODELSRC)
-    e3sm_remove_flags("${ITEM}" "-O2")
-    e3sm_remove_flags("${ITEM}" "-O1")
     e3sm_add_flags("${ITEM}" "-O0 -Mnovect")
   endforeach()
 endif()

cime_config/machines/Depends.pm-cpu.alvarez.cmake

@@ -4,11 +4,6 @@ set(NOOPT
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS NOOPT)
-    e3sm_remove_flags("${ITEM}" "-O2")
-    e3sm_remove_flags("${ITEM}" "-O")
+    e3sm_deoptimize_file(${ITEM})
   endforeach()
 endif()
-
-
-
-

cime_config/machines/Depends.pm-cpu.gnu.cmake

@@ -4,8 +4,7 @@ set(NOOPT
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS NOOPT)
-    e3sm_remove_flags("${ITEM}" "-O2")
-    e3sm_remove_flags("${ITEM}" "-O")
+    e3sm_deoptimize_file("${ITEM}")
   endforeach()
 endif()
 

cime_config/machines/Depends.pm-cpu.intel.cmake

@@ -5,7 +5,6 @@
 
 #if (NOT DEBUG)
 #  foreach(ITEM IN LISTS NOOPT)
-#    e3sm_remove_flags("${ITEM}" "-O2")
 #    e3sm_add_flags("${ITEM}" "-O0")
 #  endforeach()
 #endif()

cime_config/machines/Depends.pm-cpu.nvidia.cmake

@@ -26,7 +26,6 @@ set(PERFOBJS
 
 if (NOT DEBUG)
   foreach(ITEM IN LISTS PERFOBJS)
-    e3sm_remove_flags("${ITEM}" "-O1")
     e3sm_add_flags("${ITEM}" "-O2")
   endforeach()
 endif()

cime_config/machines/cmake_macros/CMakeLists.txt

@@ -1,4 +1,4 @@
 cmake_policy(SET CMP0057 NEW)
 cmake_minimum_required(VERSION 3.5)
 project(cime LANGUAGES C Fortran)
-include(../Macros.cmake)
+include(${CMAKE_CURRENT_BINARY_DIR}/../Macros.cmake)

cime_config/machines/cmake_macros/Macros.cmake

@@ -12,13 +12,19 @@ set(UNIVERSAL_MACRO ${MACROS_DIR}/universal.cmake)
 set(COMPILER_MACRO ${MACROS_DIR}/${COMPILER}.cmake)
 set(MACHINE_MACRO ${MACROS_DIR}/${MACH}.cmake)
 set(COMPILER_MACHINE_MACRO ${MACROS_DIR}/${COMPILER}_${MACH}.cmake)
+set(POST_PROCESS_MACRO ${SRCROOT}/cime_config/machines/cmake_macros/post_process.cmake)
 
 if (CONVERT_TO_MAKE)
-  get_cmake_property(VARS_BEFORE_BUILD_INTERNAL_IGNORE VARIABLES)
+  get_cmake_property(E3SM_CMAKE_INTERNAL_VARS_BEFORE_BUILD_INTERNAL_IGNORE VARIABLES)
+  foreach (VAR_BEFORE IN LISTS E3SM_CMAKE_INTERNAL_VARS_BEFORE_BUILD_INTERNAL_IGNORE)
+    set("E3SM_CMAKE_INTERNAL_${VAR_BEFORE}" "${${VAR_BEFORE}}")
+  endforeach()
+  list(APPEND E3SM_CMAKE_INTERNAL_VARS_BEFORE_BUILD_INTERNAL_IGNORE "VAR_BEFORE")
+  list(APPEND E3SM_CMAKE_INTERNAL_VARS_BEFORE_BUILD_INTERNAL_IGNORE "MACRO_FILE")
 endif()
 
 # Include order defines precedence
-foreach (MACRO_FILE ${UNIVERSAL_MACRO} ${COMPILER_MACRO} ${MACHINE_MACRO} ${COMPILER_MACHINE_MACRO})
+foreach (MACRO_FILE ${UNIVERSAL_MACRO} ${COMPILER_MACRO} ${MACHINE_MACRO} ${COMPILER_MACHINE_MACRO} ${POST_PROCESS_MACRO})
   if (EXISTS ${MACRO_FILE})
     include(${MACRO_FILE})
   else()
@@ -30,8 +36,17 @@ if (CONVERT_TO_MAKE)
   get_cmake_property(VARS_AFTER VARIABLES)
 
   foreach (VAR_AFTER IN LISTS VARS_AFTER)
-    if (NOT VAR_AFTER IN_LIST VARS_BEFORE_BUILD_INTERNAL_IGNORE)
-      message("CIME_SET_MAKEFILE_VAR ${VAR_AFTER} := ${${VAR_AFTER}}")
+    if (VAR_AFTER MATCHES "^E3SM_CMAKE_INTERNAL_")
+      # skip
+    else()
+      if (NOT VAR_AFTER IN_LIST E3SM_CMAKE_INTERNAL_VARS_BEFORE_BUILD_INTERNAL_IGNORE)
+        message("CIME_SET_MAKEFILE_VAR ${VAR_AFTER} := ${${VAR_AFTER}}")
+        list(APPEND E3SM_CMAKE_INTERNAL_VARS_BEFORE_BUILD_INTERNAL_IGNORE "${VAR_AFTER}")
+        set("E3SM_CMAKE_INTERNAL_${VAR_AFTER}" "${${VAR_AFTER}}")
+      elseif (NOT "${${VAR_AFTER}}" STREQUAL "${E3SM_CMAKE_INTERNAL_${VAR_AFTER}}")
+        message("CIME_SET_MAKEFILE_VAR ${VAR_AFTER} := ${${VAR_AFTER}}")
+        set("E3SM_CMAKE_INTERNAL_${VAR_AFTER}" "${${VAR_AFTER}}")
+      endif()
     endif()
   endforeach()
 endif()

cime_config/machines/cmake_macros/amdclang.cmake

@@ -1,21 +1,14 @@
-if (DEBUG)
-  string(APPEND CFLAGS   " -O0 -g")
-  string(APPEND FFLAGS   " -O0 -g")
-  string(APPEND CXXFLAGS " -O0 -g")
-  string(APPEND CPPDEFS " -DYAKL_DEBUG")
-endif()
+string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
+string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG")
 if (compile_threaded)
-  string(APPEND FFLAGS   " -fopenmp")
-  string(APPEND CFLAGS   " -fopenmp")
-  string(APPEND CXXFLAGS " -fopenmp")
-  string(APPEND LDFLAGS  " -fopenmp")
+  string(APPEND CMAKE_Fortran_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_C_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_CXX_FLAGS " -fopenmp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS  " -fopenmp")
 endif()
 string(APPEND CPPDEFS " -DNO_R16 -DCPRAMD -DFORTRANUNDERSCORE")
-string(APPEND FFLAGS_NOOPT " -O0")
-set(HAS_F2008_CONTIGUOUS "FALSE")
 set(MPICC "cc")
 set(MPICXX "CC")
 set(MPIFC "ftn")
-set(SUPPORTS_CXX "TRUE")
-set(CXX_LINKER "FORTRAN")
-string(APPEND CXX_LIBS " -lstdc++")

cime_config/machines/cmake_macros/amdclang_alvarez.cmake

@@ -1,25 +1,18 @@
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
-if (NOT DEBUG)
-  string(APPEND CFLAGS " -O2 -g")
-  string(APPEND CXXFLAGS " -O2 -g")
-  string(APPEND FFLAGS " -O2 -g")
-endif()
+string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2 -g")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2 -g")
 #string(APPEND FFLAGS " -march=znver3")
 set(SCC "clang")
 set(SCXX "clang++")
 set(SFC "flang")
 
-if (COMP_NAME STREQUAL cism)
-  string(APPEND CMAKE_OPTS " -D CISM_GNU=ON")
-endif()
-
-string(APPEND FC_AUTO_R8 " -fdefault-real-8")
-string(APPEND FFLAGS " -Mflushz ")
-string(APPEND FIXEDFLAGS " -Mfixed")
-string(APPEND FREEFLAGS " -Mfreeform")
+string(APPEND CMAKE_Fortran_FLAGS " -Mflushz ")
+string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -Mfixed")
+string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -Mfreeform")
 if (compile_threaded)
-  string(APPEND FFLAGS " -mp")
-  string(APPEND LDFLAGS " -mp")
+  string(APPEND CMAKE_Fortran_FLAGS " -mp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -mp")
 endif()

cime_config/machines/cmake_macros/amdclang_chicoma-cpu.cmake

@@ -2,25 +2,18 @@ if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
 set(PIO_FILESYSTEM_HINTS "lustre")
-if (NOT DEBUG)
-  string(APPEND CFLAGS " -O2 -g")
-  string(APPEND CXXFLAGS " -O2 -g")
-  string(APPEND FFLAGS " -O2")
-endif()
+string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2 -g")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
 #string(APPEND FFLAGS " -march=znver3")
 set(SCC "clang")
 set(SCXX "clang++")
 set(SFC "flang")
 
-if (COMP_NAME STREQUAL cism)
-  string(APPEND CMAKE_OPTS " -D CISM_GNU=ON")
-endif()
-
-string(APPEND FC_AUTO_R8 " -fdefault-real-8")
-string(APPEND FFLAGS " -Mflushz ")
-string(APPEND FIXEDFLAGS " -Mfixed")
-string(APPEND FREEFLAGS " -Mfreeform")
+string(APPEND CMAKE_Fortran_FLAGS " -Mflushz ")
+string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -Mfixed")
+string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -Mfreeform")
 if (compile_threaded)
-  string(APPEND FFLAGS " -mp")
-  string(APPEND LDFLAGS " -mp")
+  string(APPEND CMAKE_Fortran_FLAGS " -mp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -mp")
 endif()

cime_config/machines/cmake_macros/amdclang_crusher.cmake

@@ -9,5 +9,4 @@ string(APPEND CPPDEFS " -DLINUX")
 if (COMP_NAME STREQUAL gptl)
     string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
 endif()
-string(APPEND SLIBS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
-set(CRAY_LIBSCI_PREFIX_DIR "$ENV{CRAY_LIBSCI_PREFIX_DIR}")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")

cime_config/machines/cmake_macros/amdclang_frontier.cmake

@@ -9,5 +9,4 @@ string(APPEND CPPDEFS " -DLINUX")
 if (COMP_NAME STREQUAL gptl)
     string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
 endif()
-string(APPEND SLIBS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
-set(CRAY_LIBSCI_PREFIX_DIR "$ENV{CRAY_LIBSCI_PREFIX_DIR}")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")

cime_config/machines/cmake_macros/amdclang_pm-cpu.cmake

@@ -1,25 +1,18 @@
 if (COMP_NAME STREQUAL gptl)
   string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY")
 endif()
-if (NOT DEBUG)
-  string(APPEND CFLAGS " -O2 -g")
-  string(APPEND CXXFLAGS " -O2 -g")
-  string(APPEND FFLAGS " -O2")
-endif()
+string(APPEND CMAKE_C_FLAGS_RELEASE " -O2 -g")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2 -g")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE " -O2")
 #string(APPEND FFLAGS " -march=znver3")
 set(SCC "clang")
 set(SCXX "clang++")
 set(SFC "flang")
 
-if (COMP_NAME STREQUAL cism)
-  string(APPEND CMAKE_OPTS " -D CISM_GNU=ON")
-endif()
-
-string(APPEND FC_AUTO_R8 " -fdefault-real-8")
-string(APPEND FFLAGS " -Mflushz ")
-string(APPEND FIXEDFLAGS " -Mfixed")
-string(APPEND FREEFLAGS " -Mfreeform")
+string(APPEND CMAKE_Fortran_FLAGS " -Mflushz ")
+string(APPEND CMAKE_Fortran_FORMAT_FIXED_FLAG " -Mfixed")
+string(APPEND CMAKE_Fortran_FORMAT_FREE_FLAG " -Mfreeform")
 if (compile_threaded)
-  string(APPEND FFLAGS " -mp")
-  string(APPEND LDFLAGS " -mp")
+  string(APPEND CMAKE_Fortran_FLAGS " -mp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -mp")
 endif()

cime_config/machines/cmake_macros/amdclanggpu.cmake

@@ -1,24 +1,17 @@
-if (DEBUG)
-  string(APPEND CFLAGS   " -O0 -g")
-  string(APPEND FFLAGS   " -O0 -g")
-  string(APPEND CXXFLAGS " -O0 -g")
-  string(APPEND CPPDEFS " -DYAKL_DEBUG")
-endif()
+string(APPEND CMAKE_C_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_Fortran_FLAGS_DEBUG   " -O0 -g")
+string(APPEND CMAKE_CXX_FLAGS_DEBUG " -O0 -g")
+string(APPEND CPPDEFS_DEBUG " -DYAKL_DEBUG")
 if (compile_threaded)
-  string(APPEND FFLAGS   " -fopenmp")
-  string(APPEND CFLAGS   " -fopenmp")
-  string(APPEND CXXFLAGS " -fopenmp")
-  string(APPEND LDFLAGS  " -fopenmp")
+  string(APPEND CMAKE_Fortran_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_C_FLAGS   " -fopenmp")
+  string(APPEND CMAKE_CXX_FLAGS " -fopenmp")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS  " -fopenmp")
 endif()
 string(APPEND CPPDEFS " -DNO_R16 -DCPRAMD -DFORTRANUNDERSCORE")
-string(APPEND FFLAGS_NOOPT " -O0")
-set(HAS_F2008_CONTIGUOUS "FALSE")
 set(MPICC "cc")
 set(MPICXX "CC")
 set(MPIFC "ftn")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
-set(SUPPORTS_CXX "TRUE")
-set(CXX_LINKER "FORTRAN")
-string(APPEND CXX_LIBS " -lstdc++")

cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake

@@ -9,20 +9,18 @@ string(APPEND CPPDEFS " -DLINUX")
 if (COMP_NAME STREQUAL gptl)
     string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
 endif()
-if (NOT DEBUG)
-  string(APPEND CFLAGS   " -O2")
-  string(APPEND CXXFLAGS " -O2")
-  string(APPEND FFLAGS   " -O2")
-endif()
+string(APPEND CMAKE_C_FLAGS_RELEASE   " -O2")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -O2")
+string(APPEND CMAKE_Fortran_FLAGS_RELEASE   " -O2")
 
-string(APPEND CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
-string(APPEND SLIBS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
+string(APPEND SPIO_CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
+string(APPEND CMAKE_EXE_LINKER_FLAGS " -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
 
 set(MPICXX "hipcc")
 set(SCXX "hipcc")
-string(APPEND CXXFLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
-string(APPEND SLIBS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
+string(APPEND CMAKE_CXX_FLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
+string(APPEND CMAKE_EXE_LINKER_FLAGS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
 
 set(USE_HIP "TRUE")
-string(APPEND HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip")
+string(APPEND CMAKE_HIP_FLAGS "${CXXFLAGS} -munsafe-fp-atomics -x hip")