diff --git a/.gitignore b/.gitignore index 6383e5ce..c57cf708 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ +# CMake generated /include/RMGConfig.hh +/python/remage/cpp_utils.py /docs/Doxyfile /docs/conf.py + /docs/_build /docs/_doxygen /docs/api @@ -24,13 +27,74 @@ bookmarkFile compile_commands.json -# junk -*.DS_Store +# editors +*~ *.swp *.swo -.mypy_cache -__pycache__ -*.dat # python + +# Environments +.env .venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# setuptools_scm +python/*/_version.py + +# ruff +.ruff_cache/ + +# OS specific stuff +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db diff --git a/CMakeLists.txt b/CMakeLists.txt index 8df208ae..c58a5f74 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ cmake_minimum_required(VERSION 3.12 FATAL_ERROR) project( remage - VERSION 0.6.2 + VERSION 0.6.2 # TODO: get this dynamically DESCRIPTION "Simulation framework for HPGe-based experiments" LANGUAGES C CXX) # C is needed for Geant4's HDF5 support @@ -175,6 +175,20 @@ message(STATUS "CMAKE_CXX_STANDARD is c++" ${CMAKE_CXX_STANDARD}) add_subdirectory(src) +# let's now look for python, needed for the remage python wrapper and tests +find_package(Python3 REQUIRED COMPONENTS Interpreter) + +execute_process( + COMMAND "${Python3_EXECUTABLE}" -m venv --help + RESULT_VARIABLE VENV_AVAILABLE + OUTPUT_QUIET ERROR_QUIET) + +if(NOT VENV_AVAILABLE EQUAL 0) + message(FATAL_ERROR "Python3 is installed, but the 'venv' module is missing.") +endif() + +add_subdirectory(python) + option(RMG_BUILD_DOCS "Build remage documentation" OFF) if(RMG_BUILD_DOCS) add_subdirectory(docs) diff --git a/cmake/cpp_config.py.in b/cmake/cpp_config.py.in new file mode 100644 index 00000000..cd876599 --- /dev/null +++ b/cmake/cpp_config.py.in @@ -0,0 +1,3 @@ +from __future__ import annotations + +REMAGE_CPP_EXE_PATH = "@REMAGE_CPP_EXE_PATH@" diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index 803db3c9..d103cfd5 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -19,10 +19,9 @@ file(MAKE_DIRECTORY ${DOXYGEN_OUTPUT_DIR}) # configure target that runs Doxygen add_custom_command( OUTPUT ${DOXYGEN_INDEX_FILE} - COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT} + COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT} > /dev/null DEPENDS ${DOXYFILE_IN} ${REMAGE_PUBLIC_HEADERS} - MAIN_DEPENDENCY ${DOXYFILE_OUT} - COMMENT "Running Doxygen") + MAIN_DEPENDENCY ${DOXYFILE_OUT}) add_custom_target( doxygen ALL @@ -60,16 +59,18 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in ${SPHINX_SOURCE}/conf.py @ add_custom_command( OUTPUT ${SPHINX_INDEX_FILE} - COMMAND ${SPHINX_EXECUTABLE} -b html -Dbreathe_projects.remage=${DOXYGEN_OUTPUT_DIR}/xml + COMMAND ${SPHINX_EXECUTABLE} -q -b html -Dbreathe_projects.remage=${DOXYGEN_OUTPUT_DIR}/xml ${SPHINX_SOURCE} ${SPHINX_BUILD} WORKING_DIRECTORY ${SPHINX_SOURCE} DEPENDS ${SPHINX_SOURCES} ${SPHINX_IMAGES} ${DOXYGEN_INDEX_FILE} MAIN_DEPENDENCY ${SPHINX_SOURCE}/conf.py - ${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in - COMMENT "Generating Sphinx docs") + ${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in) # Nice named target so we can run the job easily -add_custom_target(sphinx ALL DEPENDS ${SPHINX_INDEX_FILE}) +add_custom_target( + sphinx ALL + DEPENDS ${SPHINX_INDEX_FILE} + COMMENT "Generating Sphinx docs") # Add an install target to install the docs include(GNUInstallDirs) diff --git a/docs/developer.md b/docs/developer.md index e45558df..4cfc42ab 100644 --- a/docs/developer.md +++ b/docs/developer.md @@ -17,6 +17,33 @@ your GitHub username): $ git clone git@github.com:yourusername/remage.git ``` +## That `remage` executable... + +To enhance _remage_'s capabilities without requiring complex C++ code, we +implemented a Python wrapper. This wrapper handles input preprocessing, invokes +the `remage-cpp` executable, and performs output postprocessing. While this +approach slightly complicates the build system, it significantly reduces the +amount of code to write and maintain. + +The C++ code resides in the `src` directory, with the `remage-cpp` executable +built from `src/remage.cc`. The Python code is organized as a package under the +`python` directory, where the `cli.py` module provides the _remage_ command-line +interface. + +At build time, CMake compiles `remage-cpp` and installs the Python package in +the build area. The Python package and its dependencies (see `pyproject.toml`) +are installed into a virtual environment, ensuring an isolated environment with +all required dependencies. The Python wrapper is configured to use the +`remage-cpp` executable from the build area. + +This setup is replicated during installation, targeting the install prefix. A +key advantage of this approach is enabling the use of the _remage_ executable in +unit tests, which run on _remage_ from the build area. + +Information about the C++ part of _remage_ is forwarded to the Python wrapper +via the `cmake/cpp_config.py.in` file, which is configured by CMake at build +time and moved into the package source folder. + ## Installing dependencies ```{include} _dependencies.md @@ -35,6 +62,10 @@ $ cmake -DCMAKE_INSTALL_PREFIX= .. $ make install ``` +```{tip} +A list of available Make targets can be printed by running `make help`. +``` + ## Code style A set of [pre-commit](https://pre-commit.com) hooks is configured to make sure diff --git a/docs/index.md b/docs/index.md index f97adb51..6b6b5e8b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,9 +6,11 @@ _remage_ is a modern C++ simulation framework for germanium experiments. The installation process is documented in {doc}`install`. -:::{warning} A proper user guide is not available yet. In the meanwhile, users -can have a look at the {doc}`tutorial` or the provided -[examples](https://github.com/legend-exp/remage/tree/main/examples). ::: +```{warning} + A proper user guide is not available yet. In the meanwhile, users can have a +look at the {doc}`tutorial` or the provided +[examples](https://github.com/legend-exp/remage/tree/main/examples). +``` In the simplest application, the user can simulate in an existing GDML geometry through the `remage` executable: diff --git a/pyproject.toml b/pyproject.toml index 7ab8b4f7..33633303 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["hatchling", "hatch-vcs"] build-backend = "hatchling.build" [project] -name = "pyremage" +name = "remage" authors = [ { name = "Luigi Pertoldi", email = "gipert@pm.me" }, ] @@ -45,6 +45,9 @@ dev = [ "pytest-cov >=3", ] +[project.scripts] +remage = "remage.cli:remage_cli" + [project.urls] Homepage = "https://github.com/legend-exp/remage" "Bug Tracker" = "https://github.com/legend-exp/remage/issues" @@ -53,8 +56,13 @@ Changelog = "https://github.com/legend-exp/remage/releases" [tool.hatch] version.source = "vcs" -build.hooks.vcs.version-file = "python/_version.py" -metadata.path = "python/" +build.hooks.vcs.version-file = "python/remage/_version.py" + +[tool.hatch.build.targets.wheel] +packages = ["python/remage"] + +[tool.hatch.build.targets.wheel.force-include] +"python/remage/cpp_config.py" = "remage/cpp_config.py" [tool.hatch.envs.default] features = ["test"] @@ -106,8 +114,6 @@ ignore = [ "ISC001", # Conflicts with formatter ] isort.required-imports = ["from __future__ import annotations"] -# Uncomment if using a _compat.typing backport -# typing-modules = ["pyremage._compat.typing"] [tool.ruff.lint.per-file-ignores] "tests/**" = ["T20"] diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 00000000..111fff8e --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,82 @@ +# List here manually all source files. Using GLOB is bad, see: +# https://cmake.org/cmake/help/latest/command/file.html?highlight=Note#filesystem + +set(_r ${PROJECT_SOURCE_DIR}) + +set(PYTHON_SOURCES ${_r}/cmake/cpp_config.py.in ${_r}/python/remage/__init__.py + ${_r}/python/remage/cli.py ${_r}/pyproject.toml) + +# get the output name of the remage-cli target (set in src/CMakeLists.txt) +get_target_property(REMAGE_CPP_OUTPUT_NAME remage-cli-cpp OUTPUT_NAME) + +# 1) construct the full path to the built executable +set(REMAGE_CPP_EXE_PATH ${CMAKE_BINARY_DIR}/src/${REMAGE_CPP_OUTPUT_NAME}) + +# configure cpp_config.py.in for the build area with the dynamically derived path +configure_file( + ${PROJECT_SOURCE_DIR}/cmake/cpp_config.py.in + ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.build.py # temporary location + @ONLY) + +# 2) construct the full path to the installed executable +set(REMAGE_CPP_EXE_PATH ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/${REMAGE_CPP_OUTPUT_NAME}) + +# configure cpp_config.py.in for the install area +configure_file( + ${PROJECT_SOURCE_DIR}/cmake/cpp_config.py.in + ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.install.py # temporary location + @ONLY) + +# create the virtual environment with python-venv +# also install the uv package manager +set(VENV_DIR ${CMAKE_BINARY_DIR}/python_venv) + +add_custom_command( + OUTPUT ${VENV_DIR}/bin/uv + COMMAND ${Python3_EXECUTABLE} -m venv ${VENV_DIR} + COMMAND ${VENV_DIR}/bin/python -m pip -q install --no-warn-script-location --upgrade pip + COMMAND ${VENV_DIR}/bin/python -m pip -q install --no-warn-script-location uv + COMMENT "Configuring Python virtual environment") + +add_custom_target(python-virtualenv DEPENDS ${VENV_DIR}/bin/uv) + +# install the remage wrapper package into the virtual environment with uv (build area) +# NOTE: when uv/pip installs the package and creates the executable for the cli, +# it hardcodes the path to the current python executable (e.g. the one of the +# virtualenv) in the script's shebang +add_custom_command( + OUTPUT ${VENV_DIR}/bin/remage + COMMAND + cp + ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.build.py # now we want to use the cpp_config for the build area + ${CMAKE_CURRENT_SOURCE_DIR}/remage/cpp_config.py + COMMAND ${VENV_DIR}/bin/python -m uv -q pip install --reinstall ${CMAKE_SOURCE_DIR} + DEPENDS python-virtualenv ${PYTHON_SOURCES} + COMMENT "Installing remage Python wrapper into the virtual environment") + +add_custom_target(remage-cli ALL DEPENDS ${VENV_DIR}/bin/remage) + +# store the path to the remage executable, needed later in tests (that must work in the build area) +set_target_properties(remage-cli PROPERTIES PYEXE_PATH ${VENV_DIR}/bin/remage) + +# install section + +# install the package into the install prefix with the existing uv installation +add_custom_command( + OUTPUT ${CMAKE_INSTALL_PREFIX}/bin/remage + COMMAND + cp + ${CMAKE_CURRENT_BINARY_DIR}/cpp_config.install.py # now we want to use the cpp_config for the install area + ${CMAKE_CURRENT_SOURCE_DIR}/remage/cpp_config.py + COMMAND ${VENV_DIR}/bin/python -m uv -q pip install --reinstall --prefix ${CMAKE_INSTALL_PREFIX} + ${CMAKE_SOURCE_DIR}) + +add_custom_target( + install-remage-cli + DEPENDS ${CMAKE_INSTALL_PREFIX}/bin/remage + COMMENT "Installing remage Python wrapper") + +# hack the install process to also install the remage wrapper +install( + CODE "execute_process(COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target install-remage-cli)" +) diff --git a/python/remage/__init__.py b/python/remage/__init__.py new file mode 100644 index 00000000..9d48db4f --- /dev/null +++ b/python/remage/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/python/remage/cli.py b/python/remage/cli.py new file mode 100644 index 00000000..5d884e85 --- /dev/null +++ b/python/remage/cli.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import subprocess +import sys + +from .cpp_config import REMAGE_CPP_EXE_PATH + + +def remage_cli(): + result = subprocess.run([REMAGE_CPP_EXE_PATH] + sys.argv[1:], check=False) + sys.exit(result.returncode) diff --git a/python/remage/cpp_config.py b/python/remage/cpp_config.py new file mode 100644 index 00000000..fe754a7d --- /dev/null +++ b/python/remage/cpp_config.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +REMAGE_CPP_EXE_PATH = "/home/gipert/sw/src/legend/remage/build/src/remage-cpp" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 65e08985..2e0ba000 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -146,9 +146,9 @@ set_property( PROPERTY COMPATIBLE_INTERFACE_STRING remage_MAJOR_VERSION) # executable for CLI -add_executable(remage-cli ${_root}/src/remage.cc) -target_link_libraries(remage-cli PUBLIC remage) -set_target_properties(remage-cli PROPERTIES OUTPUT_NAME remage) +add_executable(remage-cli-cpp ${_root}/src/remage.cc) +target_link_libraries(remage-cli-cpp PUBLIC remage) +set_target_properties(remage-cli-cpp PROPERTIES OUTPUT_NAME remage-cpp) # executable for dumping all docs add_executable(remage-doc-dump-cli EXCLUDE_FROM_ALL ${_root}/src/remage-doc-dump.cc) @@ -188,11 +188,11 @@ install( install( DIRECTORY ../include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/remage - PATTERN "CLI11" EXCLUDE - PATTERN "EcoMug" EXCLUDE) + PATTERN CLI11 EXCLUDE + PATTERN EcoMug EXCLUDE) # install CLI binaries -install(TARGETS remage-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS remage-cli-cpp RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) if(RMG_HAS_HDF5) install(TARGETS remage-to-lh5 RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) endif() diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b591366..be851279 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,8 +1,14 @@ cmake_minimum_required(VERSION 3.8) project(remage-tests) +# this can be used as the path to the remage executable in the tests +# in case the remage-cpp executable is needed, one can just use the +# "remage-cli-cpp" target name (see add_test() docs) +get_target_property(REMAGE_PYEXE remage-cli PYEXE_PATH) + add_subdirectory(basics) add_subdirectory(confinement) -add_subdirectory(output) add_subdirectory(internals) +add_subdirectory(output) +add_subdirectory(python) add_subdirectory(vertex) diff --git a/tests/basics/CMakeLists.txt b/tests/basics/CMakeLists.txt index eb15743b..f30b7f8d 100644 --- a/tests/basics/CMakeLists.txt +++ b/tests/basics/CMakeLists.txt @@ -14,11 +14,11 @@ set(_macros_extra run-2nbb.mac) set(_macros_vis vis-co60.mac vis-2nbb.mac) foreach(_mac ${_macros} ${_macros_extra} ${_macros_vis}) - add_test(NAME basics/${_mac} COMMAND remage-cli -g gdml/main.gdml -- macros/${_mac}) + add_test(NAME basics/${_mac} COMMAND ${REMAGE_PYEXE} -g gdml/main.gdml -- macros/${_mac}) endforeach() foreach(_mac ${_macros}) - add_test(NAME basics-mt/${_mac} COMMAND remage-cli -g gdml/main.gdml -t 2 macros/${_mac}) + add_test(NAME basics-mt/${_mac} COMMAND ${REMAGE_PYEXE} -g gdml/main.gdml -t 2 macros/${_mac}) set_tests_properties(basics-mt/${_mac} PROPERTIES LABELS mt) endforeach() @@ -29,7 +29,7 @@ endif() if(BxDecay0_THREADSAFE) foreach(_mac ${_macros_extra}) - add_test(NAME basics-mt/${_mac} COMMAND remage-cli -g gdml/main.gdml -t 2 macros/${_mac}) + add_test(NAME basics-mt/${_mac} COMMAND ${REMAGE_PYEXE} -g gdml/main.gdml -t 2 macros/${_mac}) set_tests_properties(basics-mt/${_mac} PROPERTIES LABELS "mt extra") endforeach() @@ -49,6 +49,6 @@ set_tests_properties(${_macros_vis} PROPERTIES SKIP_REGULAR_EXPRESSION "couldn't # further specific tests. # expect two overlaps from this prepared geometry. -add_test(NAME basics/overlaps.mac COMMAND remage-cli -- macros/overlaps.mac) +add_test(NAME basics/overlaps.mac COMMAND ${REMAGE_PYEXE} -- macros/overlaps.mac) set_tests_properties(basics/overlaps.mac PROPERTIES PASS_REGULAR_EXPRESSION "GeomVol1002.*GeomVol1002") diff --git a/tests/confinement/CMakeLists.txt b/tests/confinement/CMakeLists.txt index 180a4b39..2107bbe2 100644 --- a/tests/confinement/CMakeLists.txt +++ b/tests/confinement/CMakeLists.txt @@ -13,15 +13,15 @@ set(_macros complex-volume.mac geometrical.mac native-surface.mac geometrical-an geometrical-or-physical.mac native-volume.mac) foreach(_mac ${_macros}) - add_test(NAME confinement/${_mac} COMMAND remage-cli -g gdml/geometry.gdml -o test-out.root -- - macros/${_mac}) + add_test(NAME confinement/${_mac} COMMAND ${REMAGE_PYEXE} -g gdml/geometry.gdml -o test-out.root + -- macros/${_mac}) - add_test(NAME confinement-mt/${_mac} COMMAND remage-cli -g gdml/geometry.gdml -t 2 -o + add_test(NAME confinement-mt/${_mac} COMMAND ${REMAGE_PYEXE} -g gdml/geometry.gdml -t 2 -o test-out.root -- macros/${_mac}) set_tests_properties(confinement-mt/${_mac} PROPERTIES LABELS mt) add_test(NAME confinement-vis/${_mac} - COMMAND remage-cli -g gdml/geometry.gdml -o test-out.root -- macros/_vis.mac + COMMAND ${REMAGE_PYEXE} -g gdml/geometry.gdml -o test-out.root -- macros/_vis.mac macros/${_mac} macros/_vis-export.mac) set_tests_properties(confinement-vis/${_mac} PROPERTIES LABELS vis) set_tests_properties(confinement-vis/${_mac} PROPERTIES SKIP_REGULAR_EXPRESSION diff --git a/tests/output/CMakeLists.txt b/tests/output/CMakeLists.txt index b6e0c67b..6a3bb02e 100644 --- a/tests/output/CMakeLists.txt +++ b/tests/output/CMakeLists.txt @@ -28,16 +28,17 @@ endforeach() set(_macros ntuple-per-det.mac ntuple-per-det-vol.mac ntuple-flat.mac) foreach(_mac ${_macros}) - add_test(NAME output/hdf5-${_mac} COMMAND ./run-test-hdf5.sh $ + add_test(NAME output/hdf5-${_mac} COMMAND ./run-test-hdf5.sh $ ${_visit_hdf5} ${_remage_to_lh5} ${_mac}) - add_test(NAME output/root-${_mac} COMMAND ./run-test-root.sh $ ${_mac}) + add_test(NAME output/root-${_mac} COMMAND ./run-test-root.sh $ + ${_mac}) endforeach() list(TRANSFORM _macros PREPEND "output/hdf5-" OUTPUT_VARIABLE _macros_hdf5) set_tests_properties(${_macros_hdf5} PROPERTIES LABELS extra DEPENDS output/build-test-visit-hdf5) # SPECIAL TESTS -add_test(NAME output/th228-chain COMMAND ./run-test-th228-chain.py $) +add_test(NAME output/th228-chain COMMAND ./run-test-th228-chain.py $) # Geant4 <= 11.0.3 is deleting non-empty HDF5 files after a successful run, so disable the tests. if(Geant4_VERSION VERSION_LESS "11.0.4" OR NOT RMG_HAS_HDF5) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt new file mode 100644 index 00000000..a6a18ad4 --- /dev/null +++ b/tests/python/CMakeLists.txt @@ -0,0 +1 @@ +add_test(NAME python/cli COMMAND "${REMAGE_PYEXE}" -q --help) diff --git a/tests/vertex/CMakeLists.txt b/tests/vertex/CMakeLists.txt index e266d458..4b16e003 100644 --- a/tests/vertex/CMakeLists.txt +++ b/tests/vertex/CMakeLists.txt @@ -12,8 +12,9 @@ endforeach() set(_macros vert-hdf5.mac vert-lh5.mac) foreach(_mac ${_macros}) - add_test(NAME vertex/${_mac} COMMAND remage-cli -g gdml/geometry.gdml -- macros/${_mac}) + add_test(NAME vertex/${_mac} COMMAND ${REMAGE_PYEXE} -g gdml/geometry.gdml -- macros/${_mac}) set_tests_properties(vertex/${_mac} PROPERTIES LABELS extra) - add_test(NAME vertex-mt/${_mac} COMMAND remage-cli -g gdml/geometry.gdml -t 2 macros/${_mac}) + add_test(NAME vertex-mt/${_mac} COMMAND ${REMAGE_PYEXE} -g gdml/geometry.gdml -t 2 + macros/${_mac}) set_tests_properties(vertex-mt/${_mac} PROPERTIES LABELS "mt extra") endforeach()