Infrastructure for Python wrapper (#210)

* First version of infrastructure for Python wrapper
* Bug fix and suppress pip/uv/doxygen/sphinx outputs
* just sys.argv instead of argparse
* use the python wrapper in tests
* Update documentation

.gitignore

@@ -1,6 +1,9 @@
+# CMake generated
 /include/RMGConfig.hh
+/python/remage/cpp_utils.py
 /docs/Doxyfile
 /docs/conf.py
+
 /docs/_build
 /docs/_doxygen
 /docs/api
@@ -24,13 +27,74 @@ bookmarkFile
 
 compile_commands.json
 
-# junk
-*.DS_Store
+# editors
+*~
 *.swp
 *.swo
-.mypy_cache
-__pycache__
-*.dat
 
 # python
+
+# Environments
+.env
 .venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# setuptools_scm
+python/*/_version.py
+
+# ruff
+.ruff_cache/
+
+# OS specific stuff
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 project(
   remage
-  VERSION 0.6.2
+  VERSION 0.6.2 # TODO: get this dynamically
   DESCRIPTION "Simulation framework for HPGe-based experiments"
   LANGUAGES C CXX) # C is needed for Geant4's HDF5 support
 
@@ -175,6 +175,20 @@ message(STATUS "CMAKE_CXX_STANDARD is c++" ${CMAKE_CXX_STANDARD})
 
 add_subdirectory(src)
 
+# let's now look for python, needed for the remage python wrapper and tests
+find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+execute_process(
+  COMMAND "${Python3_EXECUTABLE}" -m venv --help
+  RESULT_VARIABLE VENV_AVAILABLE
+  OUTPUT_QUIET ERROR_QUIET)
+
+if(NOT VENV_AVAILABLE EQUAL 0)
+  message(FATAL_ERROR "Python3 is installed, but the 'venv' module is missing.")
+endif()
+
+add_subdirectory(python)
+
 option(RMG_BUILD_DOCS "Build remage documentation" OFF)
 if(RMG_BUILD_DOCS)
   add_subdirectory(docs)

cmake/cpp_config.py.in

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+REMAGE_CPP_EXE_PATH = "@REMAGE_CPP_EXE_PATH@"

docs/CMakeLists.txt

@@ -19,10 +19,9 @@ file(MAKE_DIRECTORY ${DOXYGEN_OUTPUT_DIR})
 # configure target that runs Doxygen
 add_custom_command(
   OUTPUT ${DOXYGEN_INDEX_FILE}
-  COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT}
+  COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT} > /dev/null
   DEPENDS ${DOXYFILE_IN} ${REMAGE_PUBLIC_HEADERS}
-  MAIN_DEPENDENCY ${DOXYFILE_OUT}
-  COMMENT "Running Doxygen")
+  MAIN_DEPENDENCY ${DOXYFILE_OUT})
 
 add_custom_target(
   doxygen ALL
@@ -60,16 +59,18 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${SPHINX_SOURCE}/conf.py @
 
 add_custom_command(
   OUTPUT ${SPHINX_INDEX_FILE}
-  COMMAND ${SPHINX_EXECUTABLE} -b html -Dbreathe_projects.remage=${DOXYGEN_OUTPUT_DIR}/xml
+  COMMAND ${SPHINX_EXECUTABLE} -q -b html -Dbreathe_projects.remage=${DOXYGEN_OUTPUT_DIR}/xml
           ${SPHINX_SOURCE} ${SPHINX_BUILD}
   WORKING_DIRECTORY ${SPHINX_SOURCE}
   DEPENDS ${SPHINX_SOURCES} ${SPHINX_IMAGES} ${DOXYGEN_INDEX_FILE}
   MAIN_DEPENDENCY ${SPHINX_SOURCE}/conf.py
-  ${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in
-  COMMENT "Generating Sphinx docs")
+  ${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in)
 
 # Nice named target so we can run the job easily
-add_custom_target(sphinx ALL DEPENDS ${SPHINX_INDEX_FILE})
+add_custom_target(
+  sphinx ALL
+  DEPENDS ${SPHINX_INDEX_FILE}
+  COMMENT "Generating Sphinx docs")
 
 # Add an install target to install the docs
 include(GNUInstallDirs)

docs/developer.md

@@ -17,6 +17,33 @@ your GitHub username):
 $ git clone [email protected]:yourusername/remage.git
 ```
 
+## That `remage` executable...
+
+To enhance _remage_'s capabilities without requiring complex C++ code, we
+implemented a Python wrapper. This wrapper handles input preprocessing, invokes
+the `remage-cpp` executable, and performs output postprocessing. While this
+approach slightly complicates the build system, it significantly reduces the
+amount of code to write and maintain.
+
+The C++ code resides in the `src` directory, with the `remage-cpp` executable
+built from `src/remage.cc`. The Python code is organized as a package under the
+`python` directory, where the `cli.py` module provides the _remage_ command-line
+interface.
+
+At build time, CMake compiles `remage-cpp` and installs the Python package in
+the build area. The Python package and its dependencies (see `pyproject.toml`)
+are installed into a virtual environment, ensuring an isolated environment with
+all required dependencies. The Python wrapper is configured to use the
+`remage-cpp` executable from the build area.
+
+This setup is replicated during installation, targeting the install prefix. A
+key advantage of this approach is enabling the use of the _remage_ executable in
+unit tests, which run on _remage_ from the build area.
+
+Information about the C++ part of _remage_ is forwarded to the Python wrapper
+via the `cmake/cpp_config.py.in` file, which is configured by CMake at build
+time and moved into the package source folder.
+
 ## Installing dependencies
 
 ```{include} _dependencies.md
@@ -35,6 +62,10 @@ $ cmake -DCMAKE_INSTALL_PREFIX=<optional prefix> ..
 $ make install
 ```
 
+```{tip}
+A list of available Make targets can be printed by running `make help`.
+```
+
 ## Code style
 
 A set of [pre-commit](https://pre-commit.com) hooks is configured to make sure

docs/index.md

@@ -6,9 +6,11 @@ _remage_ is a modern C++ simulation framework for germanium experiments.
 
 The installation process is documented in {doc}`install`.
 
-:::{warning} A proper user guide is not available yet. In the meanwhile, users
-can have a look at the {doc}`tutorial` or the provided
-[examples](https://github.com/legend-exp/remage/tree/main/examples). :::
+```{warning}
+ A proper user guide is not available yet. In the meanwhile, users can have a
+look at the {doc}`tutorial` or the provided
+[examples](https://github.com/legend-exp/remage/tree/main/examples).
+```
 
 In the simplest application, the user can simulate in an existing GDML geometry
 through the `remage` executable:

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["hatchling", "hatch-vcs"]
 build-backend = "hatchling.build"
 
 [project]
-name = "pyremage"
+name = "remage"
 authors = [
   { name = "Luigi Pertoldi", email = "[email protected]" },
 ]
@@ -45,6 +45,9 @@ dev = [
   "pytest-cov >=3",
 ]
 
+[project.scripts]
+remage = "remage.cli:remage_cli"
+
 [project.urls]
 Homepage = "https://github.com/legend-exp/remage"
 "Bug Tracker" = "https://github.com/legend-exp/remage/issues"
@@ -53,8 +56,13 @@ Changelog = "https://github.com/legend-exp/remage/releases"
 
 [tool.hatch]
 version.source = "vcs"
-build.hooks.vcs.version-file = "python/_version.py"
-metadata.path = "python/"
+build.hooks.vcs.version-file = "python/remage/_version.py"
+
+[tool.hatch.build.targets.wheel]
+packages = ["python/remage"]
+
+[tool.hatch.build.targets.wheel.force-include]
+"python/remage/cpp_config.py" = "remage/cpp_config.py"
 
 [tool.hatch.envs.default]
 features = ["test"]
@@ -106,8 +114,6 @@ ignore = [
   "ISC001",   # Conflicts with formatter
 ]
 isort.required-imports = ["from __future__ import annotations"]
-# Uncomment if using a _compat.typing backport
-# typing-modules = ["pyremage._compat.typing"]
 
 [tool.ruff.lint.per-file-ignores]
 "tests/**" = ["T20"]

python/CMakeLists.txt

@@ -0,0 +1,82 @@
+# List here manually all source files. Using GLOB is bad, see:
+# https://cmake.org/cmake/help/latest/command/file.html?highlight=Note#filesystem
+
+set(_r ${PROJECT_SOURCE_DIR})
+
+set(PYTHON_SOURCES ${_r}/cmake/cpp_config.py.in ${_r}/python/remage/__init__.py
+                   ${_r}/python/remage/cli.py ${_r}/pyproject.toml)
+
+# get the output name of the remage-cli target (set in src/CMakeLists.txt)
+get_target_property(REMAGE_CPP_OUTPUT_NAME remage-cli-cpp OUTPUT_NAME)
+
+# 1) construct the full path to the built executable
+set(REMAGE_CPP_EXE_PATH ${CMAKE_BINARY_DIR}/src/${REMAGE_CPP_OUTPUT_NAME})
+
+# configure cpp_config.py.in for the build area with the dynamically derived path
+configure_file(
+  ${PROJECT_SOURCE_DIR}/cmake/cpp_config.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.build.py # temporary location
+  @ONLY)
+
+# 2) construct the full path to the installed executable
+set(REMAGE_CPP_EXE_PATH ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/${REMAGE_CPP_OUTPUT_NAME})
+
+# configure cpp_config.py.in for the install area
+configure_file(
+  ${PROJECT_SOURCE_DIR}/cmake/cpp_config.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.install.py # temporary location
+  @ONLY)
+
+# create the virtual environment with python-venv
+# also install the uv package manager
+set(VENV_DIR ${CMAKE_BINARY_DIR}/python_venv)
+
+add_custom_command(
+  OUTPUT ${VENV_DIR}/bin/uv
+  COMMAND ${Python3_EXECUTABLE} -m venv ${VENV_DIR}
+  COMMAND ${VENV_DIR}/bin/python -m pip -q install --no-warn-script-location --upgrade pip
+  COMMAND ${VENV_DIR}/bin/python -m pip -q install --no-warn-script-location uv
+  COMMENT "Configuring Python virtual environment")
+
+add_custom_target(python-virtualenv DEPENDS ${VENV_DIR}/bin/uv)
+
+# install the remage wrapper package into the virtual environment with uv (build area)
+# NOTE: when uv/pip installs the package and creates the executable for the cli,
+# it hardcodes the path to the current python executable (e.g. the one of the
+# virtualenv) in the script's shebang
+add_custom_command(
+  OUTPUT ${VENV_DIR}/bin/remage
+  COMMAND
+    cp
+    ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.build.py # now we want to use the cpp_config for the build area
+    ${CMAKE_CURRENT_SOURCE_DIR}/remage/cpp_config.py
+  COMMAND ${VENV_DIR}/bin/python -m uv -q pip install --reinstall ${CMAKE_SOURCE_DIR}
+  DEPENDS python-virtualenv ${PYTHON_SOURCES}
+  COMMENT "Installing remage Python wrapper into the virtual environment")
+
+add_custom_target(remage-cli ALL DEPENDS ${VENV_DIR}/bin/remage)
+
+# store the path to the remage executable, needed later in tests (that must work in the build area)
+set_target_properties(remage-cli PROPERTIES PYEXE_PATH ${VENV_DIR}/bin/remage)
+
+# install section
+
+# install the package into the install prefix with the existing uv installation
+add_custom_command(
+  OUTPUT ${CMAKE_INSTALL_PREFIX}/bin/remage
+  COMMAND
+    cp
+    ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.install.py # now we want to use the cpp_config for the install area
+    ${CMAKE_CURRENT_SOURCE_DIR}/remage/cpp_config.py
+  COMMAND ${VENV_DIR}/bin/python -m uv -q pip install --reinstall --prefix ${CMAKE_INSTALL_PREFIX}
+          ${CMAKE_SOURCE_DIR})
+
+add_custom_target(
+  install-remage-cli
+  DEPENDS ${CMAKE_INSTALL_PREFIX}/bin/remage
+  COMMENT "Installing remage Python wrapper")
+
+# hack the install process to also install the remage wrapper
+install(
+  CODE "execute_process(COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install-remage-cli)"
+)

python/remage/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

python/remage/cli.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+
+from .cpp_config import REMAGE_CPP_EXE_PATH
+
+
+def remage_cli():
+    result = subprocess.run([REMAGE_CPP_EXE_PATH] + sys.argv[1:], check=False)
+    sys.exit(result.returncode)

python/remage/cpp_config.py

@@ -0,0 +1,3 @@
+from __future__ import annotations
+
+REMAGE_CPP_EXE_PATH = "/home/gipert/sw/src/legend/remage/build/src/remage-cpp"

src/CMakeLists.txt

@@ -146,9 +146,9 @@ set_property(
   PROPERTY COMPATIBLE_INTERFACE_STRING remage_MAJOR_VERSION)
 
 # executable for CLI
-add_executable(remage-cli ${_root}/src/remage.cc)
-target_link_libraries(remage-cli PUBLIC remage)
-set_target_properties(remage-cli PROPERTIES OUTPUT_NAME remage)
+add_executable(remage-cli-cpp ${_root}/src/remage.cc)
+target_link_libraries(remage-cli-cpp PUBLIC remage)
+set_target_properties(remage-cli-cpp PROPERTIES OUTPUT_NAME remage-cpp)
 
 # executable for dumping all docs
 add_executable(remage-doc-dump-cli EXCLUDE_FROM_ALL ${_root}/src/remage-doc-dump.cc)
@@ -188,11 +188,11 @@ install(
 install(
   DIRECTORY ../include/
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/remage
-  PATTERN "CLI11" EXCLUDE
-  PATTERN "EcoMug" EXCLUDE)
+  PATTERN CLI11 EXCLUDE
+  PATTERN EcoMug EXCLUDE)
 
 # install CLI binaries
-install(TARGETS remage-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS remage-cli-cpp RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 if(RMG_HAS_HDF5)
   install(TARGETS remage-to-lh5 RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif()