Update the VOL for HDF5 2.0.0 changes. (#73) #173
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Stack VOLs | |
| on: | |
| push: | |
| branches: [ master, dev ] | |
| paths-ignore: | |
| - '**/*.md' | |
| - '**/*.txt' | |
| - '**/*.jpg' | |
| - '**/*.png' | |
| - 'docs/*' | |
| - 'case_studies/*' | |
| pull_request: | |
| branches: [ master, dev ] | |
| paths-ignore: | |
| - '**/*.md' | |
| - '**/*.txt' | |
| - '**/*.jpg' | |
| - '**/*.png' | |
| - 'docs/*' | |
| - 'case_studies/*' | |
| env: | |
| MPICH_VERSION: 4.1.2 | |
| ARGOBOTS_VERSION: 1.1 | |
| ASYNC_VOL_VERSION: 1.8.1 | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/[email protected] | |
| - name: Set up dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get -y install automake autoconf libtool libtool-bin m4 cmake | |
| # The MPICH installed on github actions is too slow | |
| # sudo apt-get install mpich | |
| # mpicc -v | |
| # zlib | |
| sudo apt-get -y install zlib1g-dev | |
| - name: Build MPICH | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| rm -rf ${GITHUB_WORKSPACE}/MPICH | |
| mkdir ${GITHUB_WORKSPACE}/MPICH | |
| cd ${GITHUB_WORKSPACE}/MPICH | |
| wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz | |
| gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf - | |
| cd mpich-${MPICH_VERSION} | |
| ./configure --prefix=${GITHUB_WORKSPACE}/MPICH \ | |
| --silent \ | |
| --enable-romio \ | |
| --with-file-system=ufs \ | |
| --with-device=ch3:sock \ | |
| --disable-fortran \ | |
| CC=gcc | |
| make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 | |
| make -s -j 8 distclean >> qout 2>&1 | |
| - name: Install HDF5 | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| rm -rf ${GITHUB_WORKSPACE}/HDF5 | |
| mkdir ${GITHUB_WORKSPACE}/HDF5 | |
| cd ${GITHUB_WORKSPACE}/HDF5 | |
| wget -cq https://github.com/HDFGroup/hdf5/releases/latest/download/hdf5.tar.gz | |
| tar -zxf hdf5.tar.gz | |
| cd hdf5-* | |
| ./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \ | |
| --silent \ | |
| --enable-parallel \ | |
| --enable-build-mode=production \ | |
| --enable-unsupported \ | |
| --enable-threadsafe \ | |
| --disable-doxygen-doc \ | |
| --disable-doxygen-man \ | |
| --disable-doxygen-html \ | |
| --disable-tests \ | |
| --disable-fortran \ | |
| --disable-cxx \ | |
| CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc | |
| make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 | |
| make -s -j 8 distclean >> qout 2>&1 | |
| - name: Install Argobots | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| rm -rf ${ABT_DIR} | |
| mkdir ${ABT_DIR} | |
| cd ${ABT_DIR} | |
| wget -qc https://github.com/pmodels/argobots/archive/refs/tags/v${ARGOBOTS_VERSION}.tar.gz | |
| tar -xf v${ARGOBOTS_VERSION}.tar.gz | |
| cd argobots-${ARGOBOTS_VERSION} | |
| ./autogen.sh | |
| ./configure --prefix=${ABT_DIR} \ | |
| --silent \ | |
| CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc \ | |
| CXX=${GITHUB_WORKSPACE}/MPICH/bin/mpicxx | |
| make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 | |
| make -s -j 8 distclean >> qout 2>&1 | |
| - name: Install Async VOL | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export HDF5_ROOT=${HDF5_DIR} | |
| rm -rf ${ASYNC_DIR} | |
| mkdir ${ASYNC_DIR} | |
| cd ${ASYNC_DIR} | |
| wget -qc https://github.com/HDFGroup/vol-async/archive/refs/tags/v${ASYNC_VOL_VERSION}.tar.gz | |
| tar -xf v${ASYNC_VOL_VERSION}.tar.gz | |
| cd vol-async-${ASYNC_VOL_VERSION} | |
| mkdir build | |
| cd build | |
| CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc \ | |
| CXX=${GITHUB_WORKSPACE}/MPICH/bin/mpicxx \ | |
| cmake .. -DCMAKE_INSTALL_PREFIX=${ASYNC_DIR} | |
| make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 | |
| - name: Install Cache VOL | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export CAHCE_DIR=${GITHUB_WORKSPACE}/Cache | |
| export HDF5_ROOT=${HDF5_DIR} | |
| rm -rf ${CAHCE_DIR} | |
| mkdir ${CAHCE_DIR} | |
| cd ${CAHCE_DIR} | |
| git clone https://github.com/HDFGroup/vol-cache.git | |
| cd vol-cache | |
| mkdir build | |
| cd build | |
| export LD_LIBRARY_PATH="$ABT_DIR/lib:$LD_LIBRARY_PATH" | |
| CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc \ | |
| CXX=${GITHUB_WORKSPACE}/MPICH/bin/mpicxx \ | |
| CFLAGS=-DNDEBUG \ | |
| HDF5_VOL_DIR=${ASYNC_DIR} \ | |
| cmake .. -DCMAKE_INSTALL_PREFIX=${CAHCE_DIR} | |
| make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1 | |
| - name: configure and compile Log VOL connectror | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| LOGVOL_DIR=${GITHUB_WORKSPACE}/log_vol_install | |
| autoreconf -i | |
| ./configure --prefix=${LOGVOL_DIR} \ | |
| --with-hdf5=${GITHUB_WORKSPACE}/HDF5 \ | |
| --with-mpi=${GITHUB_WORKSPACE}/MPICH | |
| # Do NOT test QMCPACK, as it requires FFTW which fails to build | |
| make -s LIBTOOLFLAGS=--silent V=1 -j 8 install | |
| make -s LIBTOOLFLAGS=--silent V=1 -j 8 tests | |
| - name: Print config.log if error | |
| if: ${{ failure() }} | |
| run: | | |
| cat ${GITHUB_WORKSPACE}/config.log | |
| - name: Serial-run test - make check | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| make -s check | |
| - name: Print log | |
| if: ${{ always() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| fname=`find tests utils examples -type f -name "*.log"` | |
| for f in $fname ; do \ | |
| bname=`basename $f` ; \ | |
| if test "x$bname" != xconfig.log ; then \ | |
| echo "-------- dump $f ----------------------------" ; \ | |
| cat $f ; \ | |
| fi ; \ | |
| done | |
| - name: Parallel-run test - make ptest | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| make -s ptest | |
| - name: Test Cache and Async VOL - make check | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export CACHE_DIR=${GITHUB_WORKSPACE}/Cache | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export HDF5_ROOT=${HDF5_DIR} | |
| export HDF5_PLUGIN_PATH=${CACHE_DIR}/lib:${ASYNC_DIR}/lib | |
| export LD_LIBRARY_PATH=${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH} | |
| export HDF5_VOL_CONNECTOR="cache_ext config=${GITHUB_WORKSPACE}/cache.cfg;under_vol=512;under_info={under_vol=0;under_info={}}" | |
| export MPICH_MAX_THREAD_SAFETY=multiple | |
| export HDF5_USE_FILE_LOCKING=FALSE | |
| export HDF5_ASYNC_DISABLE_DSET_GET=0 | |
| # Start async execution at file close time | |
| export HDF5_ASYNC_EXE_FCLOSE=1 | |
| # Start async execution at group close time | |
| export HDF5_ASYNC_EXE_GCLOSE=1 | |
| # Start async execution at dataset close time | |
| export HDF5_ASYNC_EXE_DCLOSE=1 | |
| export TEST_NATIVE_VOL_ONLY=1 | |
| make -s check | |
| unset TEST_NATIVE_VOL_ONLY | |
| - name: Print log | |
| if: ${{ always() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| fname=`find tests utils examples -type f -name "*.log"` | |
| for f in $fname ; do \ | |
| bname=`basename $f` ; \ | |
| if test "x$bname" != xconfig.log ; then \ | |
| echo "-------- dump $f ----------------------------" ; \ | |
| cat $f ; \ | |
| fi ; \ | |
| done | |
| - name: Test Cache and Async VOL - make ptest | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export CACHE_DIR=${GITHUB_WORKSPACE}/Cache | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export HDF5_ROOT=${HDF5_DIR} | |
| export HDF5_PLUGIN_PATH=${CACHE_DIR}/lib:${ASYNC_DIR}/lib | |
| export LD_LIBRARY_PATH=${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH} | |
| export HDF5_VOL_CONNECTOR="cache_ext config=${GITHUB_WORKSPACE}/cache.cfg;under_vol=512;under_info={under_vol=0;under_info={}}" | |
| export MPICH_MAX_THREAD_SAFETY=multiple | |
| export HDF5_USE_FILE_LOCKING=FALSE | |
| export HDF5_ASYNC_DISABLE_DSET_GET=0 | |
| # Start async execution at file close time | |
| export HDF5_ASYNC_EXE_FCLOSE=1 | |
| # Start async execution at group close time | |
| export HDF5_ASYNC_EXE_GCLOSE=1 | |
| # Start async execution at dataset close time | |
| export HDF5_ASYNC_EXE_DCLOSE=1 | |
| export TEST_NATIVE_VOL_ONLY=1 | |
| make -s ptest | |
| unset TEST_NATIVE_VOL_ONLY | |
| - name: Test stacking Log VOL on top of Cache VOL only - make check | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export CACHE_DIR=${GITHUB_WORKSPACE}/Cache | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export LOGVOL_DIR=${GITHUB_WORKSPACE}/log_vol_install | |
| export HDF5_ROOT=${HDF5_DIR} | |
| export HDF5_PLUGIN_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib | |
| export LD_LIBRARY_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH} | |
| export HDF5_VOL_CONNECTOR="LOG under_vol=513;under_info={config=${GITHUB_WORKSPACE}/cache.cfg;under_vol=0;under_info={}}" | |
| export MPICH_MAX_THREAD_SAFETY=multiple | |
| export HDF5_USE_FILE_LOCKING=FALSE | |
| export HDF5_ASYNC_DISABLE_DSET_GET=0 | |
| # Start async execution at file close time | |
| export HDF5_ASYNC_EXE_FCLOSE=1 | |
| # Start async execution at group close time | |
| export HDF5_ASYNC_EXE_GCLOSE=1 | |
| # Start async execution at dataset close time | |
| export HDF5_ASYNC_EXE_DCLOSE=1 | |
| make -s check | |
| - name: Print log | |
| if: ${{ always() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| fname=`find tests utils examples -type f -name "*.log"` | |
| for f in $fname ; do \ | |
| bname=`basename $f` ; \ | |
| if test "x$bname" != xconfig.log ; then \ | |
| echo "-------- dump $f ----------------------------" ; \ | |
| cat $f ; \ | |
| fi ; \ | |
| done | |
| - name: Test stacking Log VOL on top of Cache VOL only - make ptest | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export CACHE_DIR=${GITHUB_WORKSPACE}/Cache | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export LOGVOL_DIR=${GITHUB_WORKSPACE}/log_vol_install | |
| export HDF5_ROOT=${HDF5_DIR} | |
| export HDF5_PLUGIN_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib | |
| export LD_LIBRARY_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH} | |
| export HDF5_VOL_CONNECTOR="LOG under_vol=513;under_info={config=${GITHUB_WORKSPACE}/cache.cfg;under_vol=0;under_info={}}" | |
| export MPICH_MAX_THREAD_SAFETY=multiple | |
| export HDF5_USE_FILE_LOCKING=FALSE | |
| export HDF5_ASYNC_DISABLE_DSET_GET=0 | |
| # Start async execution at file close time | |
| export HDF5_ASYNC_EXE_FCLOSE=1 | |
| # Start async execution at group close time | |
| export HDF5_ASYNC_EXE_GCLOSE=1 | |
| # Start async execution at dataset close time | |
| export HDF5_ASYNC_EXE_DCLOSE=1 | |
| make -s ptest | |
| - name: Test stacking Log VOL on top of Cache and Async VOL - make check | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export CACHE_DIR=${GITHUB_WORKSPACE}/Cache | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export LOGVOL_DIR=${GITHUB_WORKSPACE}/log_vol_install | |
| export HDF5_ROOT=${HDF5_DIR} | |
| export HDF5_PLUGIN_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib:${ASYNC_DIR}/lib | |
| export LD_LIBRARY_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH} | |
| export HDF5_VOL_CONNECTOR="LOG under_vol=513;under_info={config=${GITHUB_WORKSPACE}/cache.cfg;under_vol=512;under_info={under_vol=0;under_info={}}}" | |
| export MPICH_MAX_THREAD_SAFETY=multiple | |
| export HDF5_USE_FILE_LOCKING=FALSE | |
| export HDF5_ASYNC_DISABLE_DSET_GET=0 | |
| # Start async execution at file close time | |
| export HDF5_ASYNC_EXE_FCLOSE=1 | |
| # Start async execution at group close time | |
| export HDF5_ASYNC_EXE_GCLOSE=1 | |
| # Start async execution at dataset close time | |
| export HDF5_ASYNC_EXE_DCLOSE=1 | |
| make -s check | |
| - name: Print log | |
| if: ${{ always() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| fname=`find tests utils examples -type f -name "*.log"` | |
| for f in $fname ; do \ | |
| bname=`basename $f` ; \ | |
| if test "x$bname" != xconfig.log ; then \ | |
| echo "-------- dump $f ----------------------------" ; \ | |
| cat $f ; \ | |
| fi ; \ | |
| done | |
| - name: Test stacking Log VOL on top of Cache and Async VOL - make ptest | |
| if: ${{ success() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| export ABT_DIR=${GITHUB_WORKSPACE}/Argobots | |
| export ASYNC_DIR=${GITHUB_WORKSPACE}/Async | |
| export CACHE_DIR=${GITHUB_WORKSPACE}/Cache | |
| export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5 | |
| export LOGVOL_DIR=${GITHUB_WORKSPACE}/log_vol_install | |
| export HDF5_ROOT=${HDF5_DIR} | |
| export HDF5_PLUGIN_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib:${ASYNC_DIR}/lib | |
| export LD_LIBRARY_PATH=${LOGVOL_DIR}/lib:${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH} | |
| export HDF5_VOL_CONNECTOR="LOG under_vol=513;under_info={config=${GITHUB_WORKSPACE}/cache.cfg;under_vol=512;under_info={under_vol=0;under_info={}}}" | |
| export MPICH_MAX_THREAD_SAFETY=multiple | |
| export HDF5_USE_FILE_LOCKING=FALSE | |
| export HDF5_ASYNC_DISABLE_DSET_GET=0 | |
| # Start async execution at file close time | |
| export HDF5_ASYNC_EXE_FCLOSE=1 | |
| # Start async execution at group close time | |
| export HDF5_ASYNC_EXE_GCLOSE=1 | |
| # Start async execution at dataset close time | |
| export HDF5_ASYNC_EXE_DCLOSE=1 | |
| make -s ptest | |
| - name: make distclean | |
| if: ${{ always() }} | |
| run: | | |
| cd ${GITHUB_WORKSPACE} | |
| make -s distclean | |