Write and read mesh using ADIOS2

cpp/demo/checkpointing/CMakeLists.txt

@@ -0,0 +1,40 @@
+# This file was generated by running
+#
+# python cmake/scripts/generate-cmakefiles.py from dolfinx/cpp
+#
+cmake_minimum_required(VERSION 3.19)
+
+set(PROJECT_NAME demo_checkpointing)
+project(${PROJECT_NAME} LANGUAGES C CXX)
+
+# Set C++20 standard
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(NOT TARGET dolfinx)
+  find_package(DOLFINX REQUIRED)
+endif()
+
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+
+add_executable(${PROJECT_NAME} main.cpp)
+target_link_libraries(${PROJECT_NAME} dolfinx)
+
+# Do not throw error for 'multi-line comments' (these are typical in rst which
+# includes LaTeX)
+include(CheckCXXCompilerFlag)
+check_cxx_compiler_flag("-Wno-comment" HAVE_NO_MULTLINE)
+set_source_files_properties(
+  main.cpp
+  PROPERTIES
+    COMPILE_FLAGS
+    "$<$<BOOL:${HAVE_NO_MULTLINE}>:-Wno-comment -Wall -Wextra -pedantic -Werror>"
+)
+
+# Test targets (used by DOLFINx testing system)
+set(TEST_PARAMETERS2 -np 2 ${MPIEXEC_PARAMS} "./${PROJECT_NAME}")
+set(TEST_PARAMETERS3 -np 3 ${MPIEXEC_PARAMS} "./${PROJECT_NAME}")
+add_test(NAME ${PROJECT_NAME}_mpi_2 COMMAND "mpirun" ${TEST_PARAMETERS2})
+add_test(NAME ${PROJECT_NAME}_mpi_3 COMMAND "mpirun" ${TEST_PARAMETERS3})
+add_test(NAME ${PROJECT_NAME}_serial COMMAND ${PROJECT_NAME})

cpp/demo/checkpointing/main.cpp

@@ -0,0 +1,123 @@
+// ```text
+// Copyright (C) 2024 Abdullah Mujahid
+// This file is part of DOLFINx (https://www.fenicsproject.org)
+// SPDX-License-Identifier:    LGPL-3.0-or-later
+// ```
+
+// # Checkpointing
+//
+
+#include <adios2.h>
+#include <dolfinx.h>
+#include <dolfinx/common/version.h>
+#include <dolfinx/io/ADIOS2_utils.h>
+#include <dolfinx/io/checkpointing.h>
+#include <mpi.h>
+#include <typeinfo>
+#include <variant>
+
+using namespace dolfinx;
+using namespace dolfinx::io;
+
+int main(int argc, char* argv[])
+{
+  dolfinx::init_logging(argc, argv);
+  MPI_Init(&argc, &argv);
+
+  // Create mesh and function space
+  auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet);
+  auto mesh = std::make_shared<mesh::Mesh<float>>(mesh::create_rectangle<float>(
+      MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4},
+      mesh::CellType::quadrilateral, part));
+
+  // Set up ADIOS2 IO and Engine
+  auto adios
+      = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-write", "BP5", "write");
+
+  auto io = adios.io();
+
+  // TODO: Need to move this inside the checkpointing module
+  io->DefineAttribute<std::string>("version", DOLFINX_VERSION_STRING);
+  io->DefineAttribute<std::string>("git_hash", DOLFINX_VERSION_GIT);
+
+  // io::checkpointing::write_mesh(io, engine, *mesh);
+  io::checkpointing::write_mesh(adios, *mesh);
+
+  adios.close();
+
+  // ----------------------------------------------------------------------
+  auto adios_query
+      = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-read", "BP5", "read");
+
+  auto adios_read
+      = ADIOS2Wrapper(mesh->comm(), "mesh.bp", "mesh-read", "BP5", "read");
+
+  // Can't use the same engine to query as well as read
+  // since, in that case BeginStep and EndStep will be called twice
+  // on a dataset written with a single Step
+  auto mesh_read_variant = io::checkpointing::read_mesh_variant(
+      adios_query, adios_read, mesh->comm());
+
+  // We cannot resolve easily the variant
+  // Hence the user can query the type and
+  // call the correct read_mesh
+  auto mesh_read = std::get<0>(mesh_read_variant);
+  // auto mesh_read = std::get<mesh::Mesh<float>>(mesh_read_variant);
+
+  auto adios_write
+      = ADIOS2Wrapper(mesh->comm(), "mesh2.bp", "mesh-rewrite", "BP5", "write");
+
+  io::checkpointing::write_mesh(adios_write, mesh_read);
+
+  // auto io_query = adios_query.io();
+  // auto engine_query = adios_query.engine();
+
+  // Following throws an error. engine_read->BeginStep() is needed to
+  // read the VariableType, but then EndStep() fails with message
+  // EndStep() called without a successful BeginStep()
+
+  // engine_query->BeginStep();
+  // std::string floating_point = io_query->VariableType("x");
+  // engine_query->EndStep();
+
+  // // std::string floating_point =
+  // dolfinx::io::checkpointing::query_type(adios_read); std::cout <<
+  // floating_point;
+
+  // // TODO: move type deduction inside checkpointing
+  // if ("float" == floating_point)
+  // {
+  //   using T = float;
+  //   mesh::Mesh<T> mesh_read
+  //       = io::checkpointing::read_mesh<T>(adios_read, mesh->comm());
+  //   adios_read.close();
+
+  //   auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp",
+  //   "mesh-rewrite",
+  //                                    "BP5", "write");
+
+  //   auto io_write = adios_write.io();
+
+  //   io::checkpointing::write_mesh(adios_write, mesh_read);
+  //   adios_write.close();
+  // }
+  // else if ("double" == floating_point)
+  // {
+  //   using T = double;
+  //   mesh::Mesh<T> mesh_read
+  //       = io::checkpointing::read_mesh<T>(adios_read, mesh->comm());
+  //   adios_read.close();
+
+  //   auto adios_write = ADIOS2Wrapper(mesh->comm(), "mesh2.bp",
+  //   "mesh-rewrite",
+  //                                    "BP5", "write");
+
+  //   auto io_write = adios_write.io();
+
+  //   io::checkpointing::write_mesh(adios_write, mesh_read);
+  //   adios_write.close();
+  // }
+
+  MPI_Finalize();
+  return 0;
+}

cpp/dolfinx/io/ADIOS2_utils.cpp

@@ -0,0 +1,45 @@
+// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells
+//
+// This file is part of DOLFINX (https://www.fenicsproject.org)
+//
+// SPDX-License-Identifier:    LGPL-3.0-or-later
+
+#ifdef HAS_ADIOS2
+
+#include "ADIOS2_utils.h"
+#include <adios2.h>
+#include <mpi.h>
+
+namespace
+{
+std::map<std::string, adios2::Mode> string_to_mode{
+    {"write", adios2::Mode::Write},
+    {"read", adios2::Mode::Read},
+};
+} // namespace
+
+using namespace dolfinx::io;
+
+//-----------------------------------------------------------------------------
+ADIOS2Wrapper::ADIOS2Wrapper(MPI_Comm comm, std::string filename,
+                             std::string tag, std::string engine_type,
+                             std::string mode)
+
+    : _adios(std::make_unique<adios2::ADIOS>(comm)),
+      _io(std::make_unique<adios2::IO>(_adios->DeclareIO(tag)))
+{
+  _io->SetEngine(engine_type);
+  _engine = std::make_unique<adios2::Engine>(
+      _io->Open(filename, string_to_mode[mode]));
+}
+//-----------------------------------------------------------------------------
+ADIOS2Wrapper::~ADIOS2Wrapper() { close(); }
+//-----------------------------------------------------------------------------
+void ADIOS2Wrapper::close()
+{
+  assert(_engine);
+  if (*_engine)
+    _engine->Close();
+}
+
+#endif

cpp/dolfinx/io/ADIOS2_utils.h

@@ -0,0 +1,68 @@
+// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells
+//
+// This file is part of DOLFINX (https://www.fenicsproject.org)
+//
+// SPDX-License-Identifier:    LGPL-3.0-or-later
+
+#pragma once
+
+#ifdef HAS_ADIOS2
+
+#include <adios2.h>
+#include <cassert>
+#include <filesystem>
+#include <mpi.h>
+
+/// @file ADIOS2_utils.h
+/// @brief Utils for ADIOS2
+
+namespace dolfinx::io
+{
+
+/// ADIOS2-based writers/readers
+class ADIOS2Wrapper
+{
+public:
+  /// @brief Create an ADIOS2-based engine writer/reader
+  /// @param[in] comm The MPI communicator
+  /// @param[in] filename Name of output file
+  /// @param[in] tag The ADIOS2 object name
+  /// @param[in] engine_type ADIOS2 engine type. See
+  /// https://adios2.readthedocs.io/en/latest/engines/engines.html.
+  /// @param[in] mode ADIOS2 mode, default is Write or Read
+  ADIOS2Wrapper(MPI_Comm comm, std::string filename, std::string tag,
+                std::string engine_type = "BP5", std::string mode = "write");
+
+  /// @brief Move constructor
+  ADIOS2Wrapper(ADIOS2Wrapper&& ADIOS2) = default;
+
+  /// @brief Copy constructor
+  ADIOS2Wrapper(const ADIOS2Wrapper&) = delete;
+
+  /// @brief Destructor
+  ~ADIOS2Wrapper();
+
+  /// @brief Move assignment
+  ADIOS2Wrapper& operator=(ADIOS2Wrapper&& ADIOS2) = default;
+
+  // Copy assignment
+  ADIOS2Wrapper& operator=(const ADIOS2Wrapper&) = delete;
+
+  /// @brief  Close the file
+  void close();
+
+  /// @brief  Get the IO object
+  std::shared_ptr<adios2::IO> io() { return _io; }
+
+  /// @brief  Close the Engine object
+  std::shared_ptr<adios2::Engine> engine() { return _engine; }
+
+protected:
+  std::shared_ptr<adios2::ADIOS> _adios;
+  std::shared_ptr<adios2::IO> _io;
+  std::shared_ptr<adios2::Engine> _engine;
+};
+
+} // namespace dolfinx::io
+
+#endif

cpp/dolfinx/io/CMakeLists.txt

@@ -1,7 +1,9 @@
 set(HEADERS_io
     ${CMAKE_CURRENT_SOURCE_DIR}/dolfinx_io.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.h
     ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.h
     ${CMAKE_CURRENT_SOURCE_DIR}/cells.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.h
     ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.h
     ${CMAKE_CURRENT_SOURCE_DIR}/vtk_utils.h
     ${CMAKE_CURRENT_SOURCE_DIR}/VTKFile.h
@@ -14,8 +16,10 @@ set(HEADERS_io
 
 target_sources(
   dolfinx
-  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp
+  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.cpp
+          ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/cells.cpp
+          ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/VTKFile.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/vtk_utils.cpp