From 28bde91bc3ec81003bab793bb5c5a2018742caf9 Mon Sep 17 00:00:00 2001 From: mesnardo Date: Fri, 7 Feb 2020 16:05:32 -0500 Subject: [PATCH 1/2] Bugfix: output IS (natural ordering) to re-arrange sub-data Change(s) in API: None. Change(s) in source: * Add new attribute `isNatural` (class `ProbeVolume`) that stores the index set for the sub-volume that contains natural index of points to track in the domain. * Add methods `writeIS`, `writeIS_HDF5`, `writeIS_ASCII`, in the class `ProbeVolume`, to write the Index Set (natural ordering) into file. * Rename attribute `is` (class `ProbeVolume`) into `isPetsc`; it contains the index (PETSc ordering) of the point to track. * Rename few methods in the class `ProbeVolume`. Change(s) in output files: * Index Set (containing the natural index of the points tracked in the volume) is saved into the output file for the probe. The indices can be used to re-order the sub-vector of data before visualization. --- include/petibm/probes.h | 48 ++++++++++++--- src/misc/probes.cpp | 131 ++++++++++++++++++++++++++++++---------- 2 files changed, 138 insertions(+), 41 deletions(-) diff --git a/include/petibm/probes.h b/include/petibm/probes.h index 9f449118..07df1fab 100644 --- a/include/petibm/probes.h +++ b/include/petibm/probes.h @@ -144,8 +144,11 @@ class ProbeVolume : public ProbeBase /** \brief Limits of the volume. */ type::RealVec2D box; - /** \brief Index set for the grid points to monitor. */ - IS is; + /** \brief Index set for the grid points to monitor (PETSc ordering). */ + IS isPetsc; + + /** \brief Index set for the grid points to monitor (Natural ordering). */ + IS isNatural; /** \brief Sub-vector of the region to monitor. */ Vec dvec; @@ -182,8 +185,8 @@ class ProbeVolume : public ProbeBase * \param box [in] Box area to monitor * \return PetscErrorCode */ - PetscErrorCode getSubMeshInfo(const type::Mesh &mesh, - const type::RealVec2D &box); + PetscErrorCode getInfo(const type::Mesh &mesh, + const type::RealVec2D &box); /** \brief Create the index set for the points to monitor. * @@ -197,28 +200,53 @@ class ProbeVolume : public ProbeBase * \param mesh [in] Cartesian mesh object * \return PetscErrorCode */ - PetscErrorCode createSubMesh(const type::Mesh &mesh); + PetscErrorCode createGrid(const type::Mesh &mesh); /** \brief Write the sub mesh grid points into a file. + * + * Supported formats are HDF5 and ASCII. * * \param filePath [in] Path of the file to write in * \return PetscErrorCode */ - PetscErrorCode writeSubMesh(const std::string &filePath); + PetscErrorCode writeGrid(const std::string &filePath); /** \brief Write the sub mesh into an ASCII file. * * \param filePath [in] Path of the file to write in * \return PetscErrorCode */ - PetscErrorCode writeSubMeshASCII(const std::string &filePath); + PetscErrorCode writeGrid_ASCII(const std::string &filePath); /** \brief Write the sub mesh into a HDF5 file. * * \param filePath [in] Path of the file to write in * \return PetscErrorCode */ - PetscErrorCode writeSubMeshHDF5(const std::string &filePath); + PetscErrorCode writeGrid_HDF5(const std::string &filePath); + + /** \brief Write index set (natural ordering) into a file. + * + * Supported formats are HDF5 and ASCII. + * + * \param filePath [in] Path of the file to write in + * \return PetscErrorCode + */ + PetscErrorCode writeIS(const std::string &filePath); + + /** \brief Write index set (natural ordering) into a HDF5 file. + * + * \param filePath [in] Path of the file to write in + * \return PetscErrorCode + */ + PetscErrorCode writeIS_HDF5(const std::string &filePath); + + /** \brief Write index set (natural ordering) into a HDF5 file. + * + * \param filePath [in] Path of the file to write in + * \return PetscErrorCode + */ + PetscErrorCode writeIS_ASCII(const std::string &filePath); /** \copydoc ProbeBase::monitorVec() */ PetscErrorCode monitorVec(const DM &da, @@ -242,7 +270,7 @@ class ProbeVolume : public ProbeBase * \param t [in] Time * \return PetscErrorCode */ - PetscErrorCode writeVecASCII(const Vec &vec, const PetscReal &t); + PetscErrorCode writeVec_ASCII(const Vec &vec, const PetscReal &t); /** \brief Output a PETSc Vec object to a HDF5 file. * @@ -250,7 +278,7 @@ class ProbeVolume : public ProbeBase * \param t [in] Time * \return PetscErrorCode */ - PetscErrorCode writeVecHDF5(const Vec &vec, const PetscReal &t); + PetscErrorCode writeVec_HDF5(const Vec &vec, const PetscReal &t); }; // ProbeVolume diff --git a/src/misc/probes.cpp b/src/misc/probes.cpp index 046be2d2..c675d655 100644 --- a/src/misc/probes.cpp +++ b/src/misc/probes.cpp @@ -182,7 +182,8 @@ PetscErrorCode ProbeVolume::init(const MPI_Comm &comm, // data are added together and we write the time averaged data n_sum = node["n_sum"].as(0); - is = PETSC_NULL; + isPetsc = PETSC_NULL; + isNatural = PETSC_NULL; dvec = PETSC_NULL; // store information about the sub-volume to monitor @@ -198,13 +199,11 @@ PetscErrorCode ProbeVolume::init(const MPI_Comm &comm, startIdxDir.resize(3, 0); // get information about the location of the sub-mesh - ierr = getSubMeshInfo(mesh, box); CHKERRQ(ierr); + ierr = getInfo(mesh, box); CHKERRQ(ierr); // create gridline coordinates for the sub-mesh - ierr = createSubMesh(mesh); CHKERRQ(ierr); + ierr = createGrid(mesh); CHKERRQ(ierr); // write the sub-mesh to file - ierr = writeSubMesh(path); CHKERRQ(ierr); - // create a PETSc Index Set object to easily grab a sub-vector - ierr = createIS(mesh); CHKERRQ(ierr); + ierr = writeGrid(path); CHKERRQ(ierr); // create a PETSc Viewer to output the data ierr = PetscViewerCreate(comm, &viewer); CHKERRQ(ierr); @@ -214,6 +213,11 @@ PetscErrorCode ProbeVolume::init(const MPI_Comm &comm, ierr = PetscViewerFileSetMode(viewer, FILE_MODE_APPEND); CHKERRQ(ierr); ierr = PetscViewerFileSetName(viewer, path.c_str()); CHKERRQ(ierr); + // create a PETSc Index Set objects to the sub-vector of interest + ierr = createIS(mesh); CHKERRQ(ierr); + // write Index Set (natural ordering) to file + ierr = writeIS(path); CHKERRQ(ierr); + PetscFunctionReturn(0); } // ProbeVolume::init @@ -225,15 +229,16 @@ PetscErrorCode ProbeVolume::destroy() PetscFunctionBeginUser; ierr = ProbeBase::destroy(); CHKERRQ(ierr); - if (is != PETSC_NULL) {ierr = ISDestroy(&is); CHKERRQ(ierr);} + if (isPetsc != PETSC_NULL) {ierr = ISDestroy(&isPetsc); CHKERRQ(ierr);} + if (isNatural != PETSC_NULL) {ierr = ISDestroy(&isNatural); CHKERRQ(ierr);} if (dvec != PETSC_NULL) {ierr = VecDestroy(&dvec); CHKERRQ(ierr);} PetscFunctionReturn(0); } // ProbeVolume::destroy // Get information about the sub-mesh area to monitor. -PetscErrorCode ProbeVolume::getSubMeshInfo(const type::Mesh &mesh, - const type::RealVec2D &box) +PetscErrorCode ProbeVolume::getInfo(const type::Mesh &mesh, + const type::RealVec2D &box) { std::vector::iterator low, up; @@ -256,9 +261,9 @@ PetscErrorCode ProbeVolume::getSubMeshInfo(const type::Mesh &mesh, 1, std::multiplies()); PetscFunctionReturn(0); -} // ProbeVolume::getSubMeshInfo +} // ProbeVolume::getInfo -// Create the index set for the points to monitor. +// Create the index sets (PETSc and natural ordering) of the points to monitor. PetscErrorCode ProbeVolume::createIS(const type::Mesh &mesh) { PetscErrorCode ierr; @@ -287,17 +292,25 @@ PetscErrorCode ProbeVolume::createIS(const type::Mesh &mesh) } indices.resize(count); + // AO will map `indices` from natural ordering to PETSc ordering. + // However, we need to keep the vector with natural-ordering indices + // so we can post-process the solution in the sub-volume. + std::vector indices2(indices); + AO ao; ierr = DMDAGetAO(mesh->da[field], &ao); CHKERRQ(ierr); ierr = AOApplicationToPetsc(ao, count, &indices[0]); CHKERRQ(ierr); - ierr = ISCreateGeneral( - comm, count, &indices[0], PETSC_COPY_VALUES, &is); CHKERRQ(ierr); + ierr = ISCreateGeneral(comm, count, &indices[0], + PETSC_COPY_VALUES, &isPetsc); CHKERRQ(ierr); + // Create IS containing the (natural )index of the points in the sub-volume. + ierr = ISCreateGeneral(comm, count, &indices2[0], + PETSC_COPY_VALUES, &isNatural); CHKERRQ(ierr); PetscFunctionReturn(0); } // ProbeVolume::createIS // Get the sub-mesh area to monitor. -PetscErrorCode ProbeVolume::createSubMesh(const type::Mesh &mesh) +PetscErrorCode ProbeVolume::createGrid(const type::Mesh &mesh) { PetscFunctionBeginUser; @@ -313,10 +326,10 @@ PetscErrorCode ProbeVolume::createSubMesh(const type::Mesh &mesh) } PetscFunctionReturn(0); -} // ProbeVolume::createSubMesh +} // ProbeVolume::createGrid // Write the sub-mesh grid points into a file. -PetscErrorCode ProbeVolume::writeSubMesh(const std::string &filePath) +PetscErrorCode ProbeVolume::writeGrid(const std::string &filePath) { PetscErrorCode ierr; @@ -325,11 +338,11 @@ PetscErrorCode ProbeVolume::writeSubMesh(const std::string &filePath) // only ASCII and HDF5 formats are currently supported if (std::strcmp(viewerType, "ascii") == 0) { - ierr = writeSubMeshASCII(filePath); CHKERRQ(ierr); + ierr = writeGrid_ASCII(filePath); CHKERRQ(ierr); } else if (std::strcmp(viewerType, "hdf5") == 0) { - ierr = writeSubMeshHDF5(filePath); CHKERRQ(ierr); + ierr = writeGrid_HDF5(filePath); CHKERRQ(ierr); } else SETERRQ(comm, PETSC_ERR_ARG_UNKNOWN_TYPE, @@ -337,10 +350,10 @@ PetscErrorCode ProbeVolume::writeSubMesh(const std::string &filePath) "\t PETSCVIEWERASCII and PETSCVIEWERHDF5"); PetscFunctionReturn(0); -} // ProbeVolume::writeSubMesh +} // ProbeVolume::writeGrid // Write the sub mesh into a HDF5 file. -PetscErrorCode ProbeVolume::writeSubMeshHDF5(const std::string &filePath) +PetscErrorCode ProbeVolume::writeGrid_HDF5(const std::string &filePath) { PetscErrorCode ierr; @@ -373,10 +386,10 @@ PetscErrorCode ProbeVolume::writeSubMeshHDF5(const std::string &filePath) } PetscFunctionReturn(0); -} // ProbeVolume::writeSubMeshHDF5 +} // ProbeVolume::writeGrid_HDF5 // Write the sub mesh into an ASCII file. -PetscErrorCode ProbeVolume::writeSubMeshASCII(const std::string &filePath) +PetscErrorCode ProbeVolume::writeGrid_ASCII(const std::string &filePath) { PetscErrorCode ierr; @@ -412,7 +425,63 @@ PetscErrorCode ProbeVolume::writeSubMeshASCII(const std::string &filePath) ierr = MPI_Barrier(comm); CHKERRQ(ierr); PetscFunctionReturn(0); -} // ProbeVolume::writeSubMeshASCII +} // ProbeVolume::writeGrid_ASCII + +// Write the index set (natural ordering) into a file. +PetscErrorCode ProbeVolume::writeIS(const std::string &filePath) +{ + PetscErrorCode ierr; + + PetscFunctionBeginUser; + + // only ASCII and HDF5 formats are currently supported + if (std::strcmp(viewerType, "ascii") == 0) + { + ierr = writeIS_ASCII(filePath); CHKERRQ(ierr); + } + else if (std::strcmp(viewerType, "hdf5") == 0) + { + ierr = writeIS_HDF5(filePath); CHKERRQ(ierr); + } + else + SETERRQ(comm, PETSC_ERR_ARG_UNKNOWN_TYPE, + "Unsupported PetscViewerType. Supported types are:\n" + "\t PETSCVIEWERASCII and PETSCVIEWERHDF5"); + + PetscFunctionReturn(0); +} // ProbeVolume::writeIS + +PetscErrorCode ProbeVolume::writeIS_HDF5(const std::string &filePath) +{ + PetscErrorCode ierr; + + PetscFunctionBeginUser; + + ierr = PetscViewerFileSetMode(viewer, FILE_MODE_APPEND); CHKERRQ(ierr); + ierr = PetscViewerFileSetName(viewer, filePath.c_str()); CHKERRQ(ierr); + + ierr = PetscViewerHDF5PushGroup(viewer, "mesh"); CHKERRQ(ierr); + ierr = PetscObjectSetName((PetscObject) isNatural, "IS"); CHKERRQ(ierr); + ierr = ISView(isNatural, viewer); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} // ProbeVolume::writeIS_HDF5 + +// Write index set (natural ordering) into HDF5 file. +PetscErrorCode ProbeVolume::writeIS_ASCII(const std::string &filePath) +{ + PetscErrorCode ierr; + + PetscFunctionBeginUser; + + ierr = PetscViewerPushFormat( + viewer, PETSC_VIEWER_ASCII_SYMMODU); CHKERRQ(ierr); + ierr = PetscObjectSetName((PetscObject) isNatural, "IS"); CHKERRQ(ierr); + ierr = ISView(isNatural, viewer); CHKERRQ(ierr); + ierr = PetscViewerPopFormat(viewer); CHKERRQ(ierr); + + PetscFunctionReturn(0); +} // ProbeVolume::writeIS_ASCII // Monitor a sub-region of the full-domain PETSc Vec object. PetscErrorCode ProbeVolume::monitorVec(const DM &da, @@ -426,7 +495,7 @@ PetscErrorCode ProbeVolume::monitorVec(const DM &da, // grab the part of the vector that corresponds to the sub-volume Vec svec; - ierr = VecGetSubVector(fvec, is, &svec); CHKERRQ(ierr); + ierr = VecGetSubVector(fvec, isPetsc, &svec); CHKERRQ(ierr); if (n_sum != 0) // we accumulate the data over the time-steps { if (dvec == PETSC_NULL) @@ -449,7 +518,7 @@ PetscErrorCode ProbeVolume::monitorVec(const DM &da, { ierr = writeVec(svec, t); CHKERRQ(ierr); } - ierr = VecRestoreSubVector(fvec, is, &svec); CHKERRQ(ierr); + ierr = VecRestoreSubVector(fvec, isPetsc, &svec); CHKERRQ(ierr); PetscFunctionReturn(0); } // ProbeVolume::monitorVec @@ -464,11 +533,11 @@ PetscErrorCode ProbeVolume::writeVec(const Vec &vec, const PetscReal &t) // only ASCII and HDF5 formats are currently supported if (std::strcmp(viewerType, "ascii") == 0) { - ierr = writeVecASCII(vec, t); CHKERRQ(ierr); + ierr = writeVec_ASCII(vec, t); CHKERRQ(ierr); } else if (std::strcmp(viewerType, "hdf5") == 0) { - ierr = writeVecHDF5(vec, t); CHKERRQ(ierr); + ierr = writeVec_HDF5(vec, t); CHKERRQ(ierr); } else SETERRQ(comm, PETSC_ERR_ARG_UNKNOWN_TYPE, @@ -479,7 +548,7 @@ PetscErrorCode ProbeVolume::writeVec(const Vec &vec, const PetscReal &t) } // ProbeVolume::writeVec // Write vector data data into a HDF5 file. -PetscErrorCode ProbeVolume::writeVecHDF5(const Vec &vec, const PetscReal &t) +PetscErrorCode ProbeVolume::writeVec_HDF5(const Vec &vec, const PetscReal &t) { PetscErrorCode ierr; @@ -498,10 +567,10 @@ PetscErrorCode ProbeVolume::writeVecHDF5(const Vec &vec, const PetscReal &t) } PetscFunctionReturn(0); -} // ProbeVolume::writeVecHDF5 +} // ProbeVolume::writeVec_HDF5 // Write vector data into an ASCII file. -PetscErrorCode ProbeVolume::writeVecASCII(const Vec &vec, const PetscReal &t) +PetscErrorCode ProbeVolume::writeVec_ASCII(const Vec &vec, const PetscReal &t) { PetscErrorCode ierr; @@ -517,7 +586,7 @@ PetscErrorCode ProbeVolume::writeVecASCII(const Vec &vec, const PetscReal &t) ierr = PetscViewerPopFormat(viewer); CHKERRQ(ierr); PetscFunctionReturn(0); -} // ProbeVolume::writeVecASCII +} // ProbeVolume::writeVec_ASCII //***************************************************************************// //************************* ProbePoint *************************// From cb3a15cc023cff5fa5c76b42de883f56e8e52413 Mon Sep 17 00:00:00 2001 From: mesnardo Date: Sat, 8 Feb 2020 18:55:25 -0500 Subject: [PATCH 2/2] Bump version to 0.5.1 [skip ci] --- CHANGELOG.md | 16 ++++++++++++++++ configure | 20 ++++++++++---------- configure.ac | 2 +- doc/Doxyfile | 2 +- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10762314..f36e1872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,22 @@ --- +## 0.5.1 + +--- + +### Added + +### Changed + +### Fixed + +* `ProbeVolume`: write a PETSc Index Set to the output file (HDF5 or ASCII) for the volume probe. The index set contains the natural index of the points located inside the volume being monitored. During post-processing stage, the index set can be used to re-arrange field values of the sub-volume and visualize the solution. Without this index set, the PETSc vector for the sub-volume (obtained with the PETSc routine `VecGetSubVector`) did not output the values in the natural ordering of the vector. `VecGetSubVector` simply concatenates the values in the parallel ordering of the vector. This problem only affected simulations running with multiple MPI processes where the window being monitored span over multiple process domains. + +### Removed + +--- + ## 0.5.0 --- diff --git a/configure b/configure index a78beeb5..954270ca 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for PetIBM 0.5. +# Generated by GNU Autoconf 2.69 for PetIBM 0.5.1. # # Report bugs to . # @@ -590,8 +590,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='PetIBM' PACKAGE_TARNAME='petibm' -PACKAGE_VERSION='0.5' -PACKAGE_STRING='PetIBM 0.5' +PACKAGE_VERSION='0.5.1' +PACKAGE_STRING='PetIBM 0.5.1' PACKAGE_BUGREPORT='mesnardo@gwu.edu, pychuang@gwu.edu' PACKAGE_URL='' @@ -1376,7 +1376,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures PetIBM 0.5 to adapt to many kinds of systems. +\`configure' configures PetIBM 0.5.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1447,7 +1447,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of PetIBM 0.5:";; + short | recursive ) echo "Configuration of PetIBM 0.5.1:";; esac cat <<\_ACEOF @@ -1589,7 +1589,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -PetIBM configure 0.5 +PetIBM configure 0.5.1 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2395,7 +2395,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by PetIBM $as_me 0.5, which was +It was created by PetIBM $as_me 0.5.1, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3262,7 +3262,7 @@ fi # Define the identity of the package. PACKAGE='petibm' - VERSION='0.5' + VERSION='0.5.1' cat >>confdefs.h <<_ACEOF @@ -20826,7 +20826,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by PetIBM $as_me 0.5, which was +This file was extended by PetIBM $as_me 0.5.1, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -20892,7 +20892,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -PetIBM config.status 0.5 +PetIBM config.status 0.5.1 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 6a08ee15..9493eb3e 100644 --- a/configure.ac +++ b/configure.ac @@ -4,7 +4,7 @@ # set-up AC_PREREQ([2.69]) -AC_INIT([PetIBM], [0.5], [mesnardo@gwu.edu, pychuang@gwu.edu]) +AC_INIT([PetIBM], [0.5.1], [mesnardo@gwu.edu, pychuang@gwu.edu]) AC_CONFIG_AUX_DIR([config]) AC_CONFIG_MACRO_DIR([m4]) AM_INIT_AUTOMAKE([foreign subdir-objects]) diff --git a/doc/Doxyfile b/doc/Doxyfile index 461146e9..e9060f0a 100644 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = PetIBM # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 0.5 +PROJECT_NUMBER = 0.5.1 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a