From 4bedfbb702b81ef8bd2c9cc74c703ba281941f58 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Mon, 4 Nov 2024 23:26:43 -0600 Subject: [PATCH 1/4] Adding support for PIO_IOTYPE_ADIOSC Adding support for a new SCORPIO I/O type, adiosc, that supports compressing data (lossless compression) written out using ADIOS (BP file format) To enable data compression using ADIOS, users need to change (xmlchange) PIO_TYPENAME to "adiosc" --- cime_config/machines/config_machines.xml | 3 ++- driver-mct/cime_config/config_component.xml | 2 +- driver-moab/cime_config/config_component.xml | 2 +- share/build/buildlib.spio | 8 ++++++++ share/util/shr_pio_mod.F90 | 2 ++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml index 36381d781535..bd39d417fc1b 100644 --- a/cime_config/machines/config_machines.xml +++ b/cime_config/machines/config_machines.xml @@ -1987,7 +1987,8 @@ commented out until "*** No rule to make target '.../libadios2pio-nm-lib.a'" iss /nfs/gce/projects/climate/software/perl5/lib/perl5 - $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.9.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$ADIOS2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/adios2/2.10.1/mpich-4.1.2/gcc-12.1.0; else echo "$ADIOS2_ROOT"; fi} + $SHELL{if [ -z "$BLOSC2_ROOT" ]; then echo /nfs/gce/projects/climate/software/linux-ubuntu22.04-x86_64/c-blosc2/2.15.1/gcc-12.1.0; else echo "$BLOSC2_ROOT"; fi} diff --git a/driver-mct/cime_config/config_component.xml b/driver-mct/cime_config/config_component.xml index 8cddad0abf89..cdfca40befa4 100644 --- a/driver-mct/cime_config/config_component.xml +++ b/driver-mct/cime_config/config_component.xml @@ -2628,7 +2628,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/driver-moab/cime_config/config_component.xml b/driver-moab/cime_config/config_component.xml index d3685126c61e..ac9e46a476e9 100644 --- a/driver-moab/cime_config/config_component.xml +++ b/driver-moab/cime_config/config_component.xml @@ -2620,7 +2620,7 @@ char - netcdf,pnetcdf,netcdf4p,netcdf4c,adios,hdf5,default + netcdf,pnetcdf,netcdf4p,netcdf4c,adios,adiosc,hdf5,default run_pio env_run.xml pio io type diff --git a/share/build/buildlib.spio b/share/build/buildlib.spio index d17627c4323d..064fa751e4ae 100755 --- a/share/build/buildlib.spio +++ b/share/build/buildlib.spio @@ -78,6 +78,8 @@ def buildlib(bldroot, installpath, case): # ADIOS2_ROOT. This is a workaround if "ADIOS2_ROOT" in os.environ: os.environ["ADIOS2_DIR"] = os.environ["ADIOS2_ROOT"] + if "BLOSC2_ROOT" in os.environ: + os.environ["Blosc2_DIR"] = os.environ["BLOSC2_ROOT"] # If variable PIO_VERSION_MAJOR is defined in the environment then # we assume that PIO is installed on the system @@ -105,10 +107,14 @@ def buildlib(bldroot, installpath, case): else: cmake_opts += f"-DGENF90_PATH={scorpio_src_dir}/src/genf90 " + adiosc_found = False if "ADIOS2_ROOT" in os.environ: cmake_opts += "-DWITH_ADIOS2:BOOL=ON " if "FROM_CREATE_TEST" in os.environ and os.environ["FROM_CREATE_TEST"] == "True": cmake_opts += "-DADIOS_BP2NC_TEST:BOOL=ON " + if "BLOSC2_ROOT" in os.environ: + cmake_opts += "-DADIOS_USE_COMPRESSION:BOOL=ON " + adiosc_found = True if debug: cmake_opts += "-DPIO_ENABLE_LOGGING=ON " @@ -281,6 +287,8 @@ def buildlib(bldroot, installpath, case): valid_values += ",netcdf4p,netcdf4c" if adios_found: valid_values += ",adios" + if adiosc_found: + valid_values += ",adiosc" if hdf5_found: valid_values += ",hdf5" diff --git a/share/util/shr_pio_mod.F90 b/share/util/shr_pio_mod.F90 index d500cbc680a2..11cefd725b54 100644 --- a/share/util/shr_pio_mod.F90 +++ b/share/util/shr_pio_mod.F90 @@ -679,6 +679,8 @@ subroutine shr_pio_getiotypefromname(typename, iotype, defaulttype) #ifndef PIO1 else if ( typename .eq. 'ADIOS') then iotype = pio_iotype_adios + else if ( typename .eq. 'ADIOSC') then + iotype = pio_iotype_adiosc else if ( typename .eq. 'HDF5') then iotype = pio_iotype_hdf5 #endif From a1c1f07cdf407e63adc8247fb4be171124587eea Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 5 Nov 2024 09:23:07 -0600 Subject: [PATCH 2/4] Adding a testmod to force ADIOS BP compression Adding a testmod to use/force data compression for ADIOS output --- cime_config/testmods_dirs/io/force_adiosc/shell_commands | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 cime_config/testmods_dirs/io/force_adiosc/shell_commands diff --git a/cime_config/testmods_dirs/io/force_adiosc/shell_commands b/cime_config/testmods_dirs/io/force_adiosc/shell_commands new file mode 100644 index 000000000000..543d5dbfef07 --- /dev/null +++ b/cime_config/testmods_dirs/io/force_adiosc/shell_commands @@ -0,0 +1,2 @@ +#!/bin/bash +./xmlchange PIO_TYPENAME="adiosc" From d9691e6043c85fdf51a380eaa1e7b9939f994573 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Tue, 5 Nov 2024 15:47:32 -0600 Subject: [PATCH 3/4] Updating to SCORPIO v1.6.6 Updating SCORPIO version from 1.6.5 to 1.6.6 SCORPIO v1.6.6 includes the following fixes/enhancements, * Support for apps that use I/O type PIO_IOTYPE_ADIOSC (Build support only, ADIOS BP output with compression is not supported yet) --- externals/scorpio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externals/scorpio b/externals/scorpio index 804f87c48fc9..59601f625e34 160000 --- a/externals/scorpio +++ b/externals/scorpio @@ -1 +1 @@ -Subproject commit 804f87c48fc9013009307b806d1aff2afa3a0d7c +Subproject commit 59601f625e34019de0e1d3411d07bdc9e70d0054 From 9331f79677470fba68798eec35029856d436a3e8 Mon Sep 17 00:00:00 2001 From: jayeshkrishna Date: Mon, 11 Nov 2024 12:06:03 -0600 Subject: [PATCH 4/4] Adding support for ADIOS builds with BLOSC Adding support for ADIOS builds that include support for BLOSC (lossless compression) in FindPIO cmake module --- components/cmake/modules/FindPIO.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/cmake/modules/FindPIO.cmake b/components/cmake/modules/FindPIO.cmake index 0277918ac8e0..5589dff0ed04 100644 --- a/components/cmake/modules/FindPIO.cmake +++ b/components/cmake/modules/FindPIO.cmake @@ -42,6 +42,9 @@ endif() # we can assume that an MPI case with ADIOS2_ROOT set is probably # using adios. if (NOT MPILIB STREQUAL "mpi-serial" AND DEFINED ENV{ADIOS2_ROOT}) + if(DEFINED ENV{BLOSC2_ROOT}) + set(ENV{Blosc2_DIR} "$ENV{BLOSC2_ROOT}") + endif() find_package(MPI REQUIRED COMPONENTS C) find_package(ADIOS2 REQUIRED COMPONENTS C) list(APPEND PIOLIBS adios2::adios2)