From 3dcb2683cbd6bfd5b5f903661203f1dcfdb62a16 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 17:34:46 +0100 Subject: [PATCH 001/209] WIP on using pyproject.toml --- CMakeLists.txt | 4 +- nmodl/__init__.py | 10 ++- packaging/build_requirements.txt | 6 -- pyproject.toml | 28 ++++++++ requirements.txt | 8 --- setup.py | 113 ------------------------------- src/pybind/pynmodl.cpp | 1 - 7 files changed, 40 insertions(+), 130 deletions(-) delete mode 100644 packaging/build_requirements.txt create mode 100644 pyproject.toml delete mode 100644 requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index c4bc770421..9bc9c66a60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,7 +163,9 @@ find_package(PythonInterp 3.8 REQUIRED) cpp_cc_strip_python_shims(EXECUTABLE "${PYTHON_EXECUTABLE}" OUTPUT PYTHON_EXECUTABLE) include(cmake/hpc-coding-conventions/cpp/cmake/bbp-find-python-module.cmake) cpp_cc_find_python_module(jinja2 2.9.3 REQUIRED) -cpp_cc_find_python_module(pytest 3.3.0 REQUIRED) +if (NMODL_ENABLE_TESTS) + cpp_cc_find_python_module(pytest 3.3.0 REQUIRED) +endif() cpp_cc_find_python_module(sympy 1.3 REQUIRED) cpp_cc_find_python_module(textwrap 0.9 REQUIRED) cpp_cc_find_python_module(yaml 3.12 REQUIRED) diff --git a/nmodl/__init__.py b/nmodl/__init__.py index d0dfe8d4ad..5d5498b60b 100644 --- a/nmodl/__init__.py +++ b/nmodl/__init__.py @@ -1,7 +1,15 @@ try: # Try importing but catch exception in case bindings are not available from ._nmodl import NmodlDriver, to_json, to_nmodl # noqa - from ._nmodl import __version__ + + from importlib.metadata import version, PackageNotFoundError + + try: + __version__ = version("nmodl") + except PackageNotFoundError: + # package is not installed + pass + __all__ = ["NmodlDriver", "to_json", "to_nmodl"] except ImportError: print("[NMODL] [warning] :: Python bindings are not available") diff --git a/packaging/build_requirements.txt b/packaging/build_requirements.txt deleted file mode 100644 index 9f6f031d0b..0000000000 --- a/packaging/build_requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -Jinja2>=2.9.3 -PyYAML>=3.13 -pytest -sympy>=1.3 -find_libpython -scikit-build diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..39b7bd6e7b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "NMODL" +authors = [ + {name = "Blue Brain Project", email = "bbp-ou-hpc@groupes.epfl.ch"}, +] +description = "NEURON Modeling Language Source-to-Source Compiler Framework" +license = {file = "LICENSE"} +readme = {file = "README.rst", content-type = "text/x-rst"} +dynamic = ["version"] +dependencies = [ + "sympy>=1.3", + "Jinja2>=2.9.3", + "PyYAML>=3.13", +] + +[build-system] +requires = [ + "setuptools", + "find_libpython", + "scikit-build", + "sympy>=1.3", + "Jinja2>=2.9.3", + "PyYAML>=3.13", + "setuptools-scm>=8.0", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8816849e01..0000000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -Jinja2>=2.9.3 -PyYAML>=3.13 -pytest -pytest-cov -sympy -numpy -find_libpython -scikit-build diff --git a/setup.py b/setup.py index 5059f51502..9e54fcf2e6 100644 --- a/setup.py +++ b/setup.py @@ -2,133 +2,20 @@ # See the top-level LICENSE file for details. # # SPDX-License-Identifier: Apache-2.0 - -import inspect -import os -import subprocess import sys -from setuptools import Command from skbuild import setup -""" -A generic wrapper to access nmodl binaries from a python installation -Please create a softlink with the binary name to be called. -""" -import stat -from pkg_resources import working_set -from find_libpython import find_libpython - - -# Main source of the version. Dont rename, used by Cmake -try: - v = ( - subprocess.run(["git", "describe", "--tags"], stdout=subprocess.PIPE) - .stdout.strip() - .decode() - ) - __version__ = v[: v.rfind("-")].replace("-", ".") if "-" in v else v - # allow to override version during development/testing - if "NMODL_WHEEL_VERSION" in os.environ: - __version__ = os.environ['NMODL_WHEEL_VERSION'] -except Exception as e: - raise RuntimeError("Could not get version from Git repo") from e - - -class lazy_dict(dict): - """When the value associated to a key is a function, then returns - the function call instead of the function. - """ - - def __getitem__(self, item): - value = dict.__getitem__(self, item) - if inspect.isfunction(value): - return value() - return value - - -def get_sphinx_command(): - """Lazy load of Sphinx distutils command class - """ - # If nbconvert is installed to .eggs on the fly when running setup.py then - # templates from it will not be found. This is a workaround. - if 'JUPYTER_PATH' not in os.environ: - import nbconvert - os.environ['JUPYTER_PATH'] = os.path.realpath(os.path.join(os.path.dirname(nbconvert.__file__), '..', 'share', 'jupyter')) - print("Setting JUPYTER_PATH={}".format(os.environ['JUPYTER_PATH'])) - - from sphinx.setup_command import BuildDoc - - return BuildDoc - - -class Docs(Command): - description = "Generate & optionally upload documentation to docs server" - user_options = [] - finalize_options = lambda self: None - initialize_options = lambda self: None - - def run(self, *args, **kwargs): - self.run_command("doctest") - self.run_command("buildhtml") - - -install_requirements = [ - "PyYAML>=3.13", - "sympy>=1.3", - "find_libpython" -] - cmake_args = ["-DPYTHON_EXECUTABLE=" + sys.executable] if "bdist_wheel" in sys.argv: cmake_args.append("-DLINK_AGAINST_PYTHON=FALSE") cmake_args.append("-DNMODL_ENABLE_TESTS=FALSE") -# For CI, we want to build separate wheel -package_name = "NMODL" -if "NMODL_NIGHTLY_TAG" in os.environ: - package_name += os.environ["NMODL_NIGHTLY_TAG"] - -# Parse long description from README.rst -with open('README.rst', 'r', encoding='utf-8') as f: - long_description = f.read() - setup( - name=package_name, - version=__version__, - author="Blue Brain Project", - author_email="bbp-ou-hpc@groupes.epfl.ch", - description="NEURON Modeling Language Source-to-Source Compiler Framework", - long_description=long_description, - long_description_content_type='text/markdown', packages=["nmodl"], scripts=["pywheel/shim/nmodl"], include_package_data=True, cmake_minimum_required_version="3.15.0", cmake_args=cmake_args, - cmdclass=lazy_dict( - docs=Docs, doctest=get_sphinx_command, buildhtml=get_sphinx_command, - ), - zip_safe=False, - # myst_parser 2.0.0 want sphinx >= 6 but it leads to incompatibily with sphinxcontrib-applehelp and - # sphinxcontrib-htmlhelp that want PEP-517 support instead of setup.py (name clash with '-' and '.') - # So we pin low versions of packages - setup_requires=[ - "jinja2>=2.9.3", - "jupyter-client", - "jupyter", - "myst_parser<2.0.0", - "mistune<3", - "nbconvert", - "nbsphinx>=0.3.2", - "pytest>=3.7.2", - "sphinxcontrib-applehelp<1.0.3", # After this version it needs a toml file to work, no more setup.py - "sphinxcontrib-htmlhelp<=2.0.0", # After this version it needs a toml file to work, no more setup.py - "sphinx<6", - "sphinx-rtd-theme", # needs sphinx < 7 - "docutils<0.20", # needed by sphinx - ] - + install_requirements, - install_requires=install_requirements, ) diff --git a/src/pybind/pynmodl.cpp b/src/pybind/pynmodl.cpp index bb58f3fc77..b4a8b80a0e 100644 --- a/src/pybind/pynmodl.cpp +++ b/src/pybind/pynmodl.cpp @@ -138,7 +138,6 @@ void init_symtab_module(py::module& m); PYBIND11_MODULE(_nmodl, m_nmodl) { m_nmodl.doc() = "NMODL : Source-to-Source Code Generation Framework"; - m_nmodl.attr("__version__") = nmodl::Version::NMODL_VERSION; py::class_ _{m_nmodl, "nmodl::parser::NmodlDriver"}; py::class_ nmodl_driver( From b2939d268ab15e1c614e69711eb34e932ccc3e34 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 17:37:30 +0100 Subject: [PATCH 002/209] Use `pip wheel` instead of `setup.py bdist_wheel` --- packaging/build_wheels.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 961033f3ff..749eb9d915 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -54,7 +54,7 @@ build_wheel_linux() { rm -rf dist _skbuild # Workaround for https://github.com/pypa/manylinux/issues/1309 git config --global --add safe.directory "*" - python setup.py bdist_wheel + python -m pip wheel --dist-dir dist/ -vvv . echo " - Repairing..." auditwheel repair dist/*.whl @@ -74,7 +74,7 @@ build_wheel_osx() { echo " - Building..." rm -rf dist _skbuild - python setup.py bdist_wheel + python -m pip wheel --dist-dir dist/ -vvv . echo " - Repairing..." delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel From 66da08d7749932dc257f1ce82309a9afbce89079 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 17:42:19 +0100 Subject: [PATCH 003/209] Fix formatting --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bc9c66a60..b4b6be91c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,7 +163,7 @@ find_package(PythonInterp 3.8 REQUIRED) cpp_cc_strip_python_shims(EXECUTABLE "${PYTHON_EXECUTABLE}" OUTPUT PYTHON_EXECUTABLE) include(cmake/hpc-coding-conventions/cpp/cmake/bbp-find-python-module.cmake) cpp_cc_find_python_module(jinja2 2.9.3 REQUIRED) -if (NMODL_ENABLE_TESTS) +if(NMODL_ENABLE_TESTS) cpp_cc_find_python_module(pytest 3.3.0 REQUIRED) endif() cpp_cc_find_python_module(sympy 1.3 REQUIRED) From 8ac280e0cf75965e091857e56d162cb44aef9b35 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 17:51:40 +0100 Subject: [PATCH 004/209] Do not use `-r reqs` when building wheel --- packaging/build_wheels.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 749eb9d915..d9345a61af 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -48,7 +48,6 @@ build_wheel_linux() { echo " - Installing build requirements" pip install pip auditwheel setuptools - pip install -r packaging/build_requirements.txt echo " - Building..." rm -rf dist _skbuild @@ -70,7 +69,7 @@ build_wheel_osx() { (( $skip )) && return 0 echo " - Installing build requirements" - pip install --upgrade delocate -r packaging/build_requirements.txt + pip install --upgrade delocate echo " - Building..." rm -rf dist _skbuild From 5246907e0bf21067dd692ee11a5bfed36d0b5724 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 17:57:10 +0100 Subject: [PATCH 005/209] Wrong flag for `pip wheel` --- packaging/build_wheels.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index d9345a61af..86e63efaa6 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -53,7 +53,7 @@ build_wheel_linux() { rm -rf dist _skbuild # Workaround for https://github.com/pypa/manylinux/issues/1309 git config --global --add safe.directory "*" - python -m pip wheel --dist-dir dist/ -vvv . + python -m pip wheel --wheel-dir dist/ -vvv . echo " - Repairing..." auditwheel repair dist/*.whl @@ -73,7 +73,7 @@ build_wheel_osx() { echo " - Building..." rm -rf dist _skbuild - python -m pip wheel --dist-dir dist/ -vvv . + python -m pip wheel --wheel-dir dist/ -vvv . echo " - Repairing..." delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel From dc494a2a3f929cb4f9d3aa8ff047b6f74fb4ddc9 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 18:15:02 +0100 Subject: [PATCH 006/209] Fix tabs --- pyproject.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39b7bd6e7b..a83984df3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,18 +8,18 @@ license = {file = "LICENSE"} readme = {file = "README.rst", content-type = "text/x-rst"} dynamic = ["version"] dependencies = [ - "sympy>=1.3", - "Jinja2>=2.9.3", + "sympy>=1.3", + "Jinja2>=2.9.3", "PyYAML>=3.13", ] [build-system] requires = [ - "setuptools", - "find_libpython", - "scikit-build", - "sympy>=1.3", - "Jinja2>=2.9.3", + "setuptools", + "find_libpython", + "scikit-build", + "sympy>=1.3", + "Jinja2>=2.9.3", "PyYAML>=3.13", "setuptools-scm>=8.0", ] From 390490540bfb6411e18ff29bd43e1f19b396f1c5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 18:18:46 +0100 Subject: [PATCH 007/209] Specify exact wheel we will use in CI --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 02f96583f4..6f7236bf08 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -239,7 +239,7 @@ stages: sudo apt-add-repository -y ppa:deadsnakes/ppa sudo apt-get update sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv - packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl + packaging/test_wheel.bash python$(python.version) wheelhouse/NMODL*.whl condition: succeeded() displayName: 'Test ManyLinux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml @@ -304,7 +304,7 @@ stages: condition: succeeded() displayName: 'Publish wheel as build artifact' - script: | - packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl + packaging/test_wheel.bash python$(python.version) wheelhouse/NMODL*.whl condition: succeeded() displayName: 'Test macos Wheel with Python $(python.version)' - template: ci/upload-wheels.yml From e2ba8bdf602d067226ce74287ede16ff7c54d0f5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 18:31:28 +0100 Subject: [PATCH 008/209] Update runtime dependencies --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a83984df3e..cd0596f24c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ license = {file = "LICENSE"} readme = {file = "README.rst", content-type = "text/x-rst"} dynamic = ["version"] dependencies = [ + "find_libpython", "sympy>=1.3", "Jinja2>=2.9.3", "PyYAML>=3.13", From 6e4c626bdc38c336accfe2001a21bb657a2de5ad Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 5 Feb 2024 19:09:34 +0100 Subject: [PATCH 009/209] Only repair the NMODL wheel --- packaging/build_wheels.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 86e63efaa6..5ba6370374 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -56,7 +56,7 @@ build_wheel_linux() { python -m pip wheel --wheel-dir dist/ -vvv . echo " - Repairing..." - auditwheel repair dist/*.whl + auditwheel repair dist/NMODL*.whl deactivate } @@ -76,7 +76,7 @@ build_wheel_osx() { python -m pip wheel --wheel-dir dist/ -vvv . echo " - Repairing..." - delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel + delocate-wheel -w wheelhouse -v dist/NMODL*.whl # we started clean, there's a single wheel deactivate } From 58be003e92f1fa1b3e2551b46d08baea0d66c1ef Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 6 Feb 2024 10:27:35 +0100 Subject: [PATCH 010/209] Revert "Specify exact wheel we will use in CI" This reverts commit 9761a16d7495b9adf899b8a8aac2dcf1d8a03953. --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6f7236bf08..02f96583f4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -239,7 +239,7 @@ stages: sudo apt-add-repository -y ppa:deadsnakes/ppa sudo apt-get update sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv - packaging/test_wheel.bash python$(python.version) wheelhouse/NMODL*.whl + packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl condition: succeeded() displayName: 'Test ManyLinux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml @@ -304,7 +304,7 @@ stages: condition: succeeded() displayName: 'Publish wheel as build artifact' - script: | - packaging/test_wheel.bash python$(python.version) wheelhouse/NMODL*.whl + packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl condition: succeeded() displayName: 'Test macos Wheel with Python $(python.version)' - template: ci/upload-wheels.yml From f373d31360f1e7339c1de668f99cc21dde75bc76 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 6 Feb 2024 10:28:48 +0100 Subject: [PATCH 011/209] Use `python -m build` instead of `pip wheel` --- packaging/build_wheels.bash | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 5ba6370374..2338aeceec 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -47,16 +47,16 @@ build_wheel_linux() { (( $skip )) && return 0 echo " - Installing build requirements" - pip install pip auditwheel setuptools + pip install pip auditwheel setuptools build echo " - Building..." rm -rf dist _skbuild # Workaround for https://github.com/pypa/manylinux/issues/1309 git config --global --add safe.directory "*" - python -m pip wheel --wheel-dir dist/ -vvv . + python -m build --wheel -o dist/ echo " - Repairing..." - auditwheel repair dist/NMODL*.whl + auditwheel repair dist/*.whl deactivate } @@ -69,14 +69,14 @@ build_wheel_osx() { (( $skip )) && return 0 echo " - Installing build requirements" - pip install --upgrade delocate + pip install --upgrade delocate build echo " - Building..." rm -rf dist _skbuild - python -m pip wheel --wheel-dir dist/ -vvv . + python -m build --wheel -o dist/ echo " - Repairing..." - delocate-wheel -w wheelhouse -v dist/NMODL*.whl # we started clean, there's a single wheel + delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel deactivate } From 97774cb05bf63d4a6f3eac4f4ccdc061bd30b372 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 6 Feb 2024 14:41:00 +0100 Subject: [PATCH 012/209] Add workaround for PEP 621 not allowing a dynamic `name` PEP 621 does not allow `project.name` to be dynamic, and requires any build backend to fail if one declares it as such. Since we only need to change the name of the package when creating a nightly release, this adds a `change_name.py` script, which only runs in the CI, that can change the `project.name` entry in-place. Note that, since `tomllib` has only become a part of the standard Python library in version 3.11, we use the `tomli` (for reading) and `tomli-w` (for writing) packages for parsing the `pyproject.toml` file. --- azure-pipelines.yml | 8 ++++++ packaging/change_name.py | 53 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 packaging/change_name.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 02f96583f4..d68952e4ae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -222,6 +222,10 @@ stages: else export TAG="" fi + # the following 2 lines are a workaround for PEP 621 not allowing a + # dynamic `name` in `pyproject.toml` + python3 -m pip install --user tomli tomli-w + python3 packaging/change_name.py pyproject.toml "NMODL${TAG}" docker run --rm \ -w /root/nmodl \ -v $PWD:/root/nmodl \ @@ -295,6 +299,10 @@ stages: else export NMODL_NIGHTLY_TAG="" fi + # the following 2 lines are a workaround for PEP 621 not allowing a + # dynamic `name` in `pyproject.toml` + python3 -m pip install tomli tomli-w + python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}" packaging/build_wheels.bash osx $(python.version) condition: succeeded() displayName: 'Build macos Wheel' diff --git a/packaging/change_name.py b/packaging/change_name.py new file mode 100644 index 0000000000..f3c453dc56 --- /dev/null +++ b/packaging/change_name.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +""" +Barebones utility for changing the name of the package in `pyproject.toml`. + +Usage +----- +[SCRIPT] [PYPROJECT_FILE] [NAME] + +Notes +----- +Should only be used by the CI +""" +import re +from argparse import ArgumentParser +from pathlib import Path + +import tomli +import tomli_w + + +def main(): + parser = ArgumentParser() + parser.add_argument("pyproject_file", help="The path to the pyproject.toml file") + parser.add_argument("name", help="The new name of the project") + args = parser.parse_args() + + if not Path(args.pyproject_file).exists(): + raise FileNotFoundError(f"File {args.pyproject_file} not found") + + with open(args.pyproject_file, "rb") as f: + toml_dict = tomli.load(f) + + # check name is conforms to the naming convention + # see: + # https://packaging.python.org/en/latest/specifications/name-normalization/ + + if not re.match( + r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", + args.name, + flags=re.IGNORECASE, + ): + raise ValueError( + f"The package name {args.name} does not conform to the standard Python package naming convention" + ) + + toml_dict["project"]["name"] = args.name + + with open(args.pyproject_file, "wb") as f: + tomli_w.dump(toml_dict, f) + + +if __name__ == "__main__": + main() From 3595c04f9a20b02f77d95efb6b0f5c746ec74533 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 7 Feb 2024 13:30:30 +0100 Subject: [PATCH 013/209] Use `scikit-build-core` as the build backend --- pyproject.toml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd0596f24c..0c4b0111c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,14 +16,21 @@ dependencies = [ [build-system] requires = [ - "setuptools", + "scikit-build-core", + "setuptools-scm>=8.0", "find_libpython", - "scikit-build", "sympy>=1.3", "Jinja2>=2.9.3", "PyYAML>=3.13", - "setuptools-scm>=8.0", ] -build-backend = "setuptools.build_meta" +build-backend = "scikit_build_core.build" + +[tool.scikit-build] +cmake.args = [ + "-DLINK_AGAINST_PYTHON=FALSE", + "-DNMODL_ENABLE_TESTS=FALSE", +] +cmake.verbose = true +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" [tool.setuptools_scm] From 5dfab36acb2d5d2e85fb33e045aaebaa068c291b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 7 Feb 2024 15:46:45 +0100 Subject: [PATCH 014/209] Add missing packages to wheel --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0c4b0111c9..dc761fdb9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,5 +32,6 @@ cmake.args = [ ] cmake.verbose = true metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" +wheel.packages = ["nmodl"] [tool.setuptools_scm] From 3e62103da7a6ce6affd2b1c70b9d91ded8e1c9bf Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 7 Feb 2024 16:24:23 +0100 Subject: [PATCH 015/209] Use cmake.define key instead of cmake.args --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dc761fdb9f..2159ae815c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,12 +26,12 @@ requires = [ build-backend = "scikit_build_core.build" [tool.scikit-build] -cmake.args = [ - "-DLINK_AGAINST_PYTHON=FALSE", - "-DNMODL_ENABLE_TESTS=FALSE", -] cmake.verbose = true metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" wheel.packages = ["nmodl"] +[tool.scikit-build.cmake.define] +LINK_AGAINST_PYTHON = "FALSE" +NMODL_ENABLE_TESTS = "FALSE" + [tool.setuptools_scm] From 8c45d572cf4a5e145ee3c88468cfa2ef6a24cc47 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 7 Feb 2024 17:48:50 +0100 Subject: [PATCH 016/209] Refactor the nmodl wrapper --- {pywheel/shim => nmodl}/_binwrapper.py | 9 ++------- pyproject.toml | 1 + pywheel/shim/nmodl | 1 - 3 files changed, 3 insertions(+), 8 deletions(-) rename {pywheel/shim => nmodl}/_binwrapper.py (86%) delete mode 120000 pywheel/shim/nmodl diff --git a/pywheel/shim/_binwrapper.py b/nmodl/_binwrapper.py similarity index 86% rename from pywheel/shim/_binwrapper.py rename to nmodl/_binwrapper.py index c495e037f3..e1971a2b84 100755 --- a/pywheel/shim/_binwrapper.py +++ b/nmodl/_binwrapper.py @@ -11,7 +11,7 @@ from find_libpython import find_libpython -def _config_exe(exe_name): +def main(): """Sets the environment to run the real executable (returned)""" package_name = "nmodl" @@ -37,12 +37,7 @@ def _config_exe(exe_name): # set PYTHONPATH for embedded python to properly find the nmodl module os.environ["PYTHONPATH"] = working_set.by_key[package_name].location + ':' + os.environ.get("PYTHONPATH", "") - return os.path.join(NMODL_BIN, exe_name) - - -if __name__ == "__main__": - """Set the pointed file as executable""" - exe = _config_exe(os.path.basename(sys.argv[0])) + exe = os.path.join(NMODL_BIN, os.path.basename(sys.argv[0])) st = os.stat(exe) os.chmod(exe, st.st_mode | stat.S_IEXEC) os.execv(exe, sys.argv) diff --git a/pyproject.toml b/pyproject.toml index 2159ae815c..a8bba56007 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ dependencies = [ "Jinja2>=2.9.3", "PyYAML>=3.13", ] +scripts = {nmodl = "nmodl._binwrapper:main"} [build-system] requires = [ diff --git a/pywheel/shim/nmodl b/pywheel/shim/nmodl deleted file mode 120000 index cf9ff4fba4..0000000000 --- a/pywheel/shim/nmodl +++ /dev/null @@ -1 +0,0 @@ -_binwrapper.py \ No newline at end of file From 2a221a19734188276b4e9a18a826b55110b9a4f0 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 7 Feb 2024 17:55:07 +0100 Subject: [PATCH 017/209] Added optional dependencies --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a8bba56007..afcf022059 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,21 @@ dependencies = [ ] scripts = {nmodl = "nmodl._binwrapper:main"} +optional-dependencies.test = ["pytest>=3.3.0", "pytest-cov"] +optional-dependencies.docs = [ + "jupyter-client", + "jupyter", + "myst_parser<2.0.0", + "mistune<3", + "nbconvert", + "nbsphinx>=0.3.2", + "sphinxcontrib-applehelp<1.0.3", # After this version it needs a toml file to work, no more setup.py + "sphinxcontrib-htmlhelp<=2.0.0", # After this version it needs a toml file to work, no more setup.py + "sphinx<6", + "sphinx-rtd-theme", # needs sphinx < 7 + "docutils<0.20", # needed by sphinx +] + [build-system] requires = [ "scikit-build-core", From c558e80a14c8452148c363196d2b4626c45cae2e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 8 Feb 2024 14:08:14 +0100 Subject: [PATCH 018/209] Remove unused script in setup.py --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 9e54fcf2e6..dde4e0bc6d 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ setup( packages=["nmodl"], - scripts=["pywheel/shim/nmodl"], include_package_data=True, cmake_minimum_required_version="3.15.0", cmake_args=cmake_args, From 6870fe0770a5a8f7202999916074c89e98db7d22 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 8 Feb 2024 17:49:40 +0100 Subject: [PATCH 019/209] Workaround for wheel build failure on MacOS On MacOS there seems to be an issue with `scikit-build-core` when `build-dir` is unset, so now we explicitly set it when building the wheel. --- packaging/build_wheels.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 2338aeceec..dd62686f5d 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -73,7 +73,7 @@ build_wheel_osx() { echo " - Building..." rm -rf dist _skbuild - python -m build --wheel -o dist/ + python -m build --wheel -o dist/ --config-setting='build-dir=_build' echo " - Repairing..." delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel From 60d845d89cfcb36eed78514ce12377bb87bbaeaa Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 11 Feb 2024 21:25:35 +0100 Subject: [PATCH 020/209] Change mode of `change_name.py` --- packaging/change_name.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 packaging/change_name.py diff --git a/packaging/change_name.py b/packaging/change_name.py old mode 100644 new mode 100755 From 4c9cae74b9af8a03db50a1deb0c78ccaf2a0f5cb Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 11 Feb 2024 21:29:13 +0100 Subject: [PATCH 021/209] Add file for showing dependencies --- packaging/show_deps.py | 112 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100755 packaging/show_deps.py diff --git a/packaging/show_deps.py b/packaging/show_deps.py new file mode 100755 index 0000000000..3a24bc0714 --- /dev/null +++ b/packaging/show_deps.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +""" +Small script that can parse TOML files without any dependencies +""" +import sys +from argparse import ArgumentParser +from pathlib import Path + +if sys.version_info >= (3, 11): + # use the standard library module + from tomllib import load + +else: + import pip + + _earliest_pip_version = (21, 2, 0) + _current_pip_version = tuple(int(_) for _ in pip.__version__.split(".")) + if _current_pip_version >= _earliest_pip_version: + # use the vendored version in pip + from pip._vendor.tomli import load + + else: + # as a last resort, try to use tomli if it's installed + try: + from tomli import load + except ModuleNotFoundError as err: + raise ModuleNotFoundError( + f"pip version {pip.__version__} found, " + f"{'.'.join(map(str,_earliest_pip_version))} or above " + f"or the `tomli` package required\n" + "You can either install a newer version of pip (using `pip install -U pip`), " + "or install the `tomli` package (using `pip install tomli`)" + ) from err + + +def main(): + """ + Main module + """ + parser = ArgumentParser() + parser.add_argument( + "dir", + help="the directory containing the `pyproject.toml` file", + ) + parser.add_argument( + "--runtime", + "-r", + action="store_true", + help="show only runtime dependencies", + ) + parser.add_argument( + "--build", + "-b", + action="store_true", + help="show only build dependencies", + ) + parser.add_argument( + "--type", + "-t", + help="the type of optional dependency to show (default: show packages from all types)", + ) + parser.add_argument( + "--list", + "-l", + action="store_true", + help="list all of the optional dependency types. " + "Note that this overrides all other options", + ) + args = parser.parse_args() + + if not (Path(args.dir) / "pyproject.toml").exists(): + raise FileNotFoundError(f"No `pyproject.toml` file found in {args.dir}") + + with open(Path(args.dir) / "pyproject.toml", "rb") as f: + content = load(f) + + top_key, opt_key = "project", "optional-dependencies" + + if args.list: + for key in content[top_key].get(opt_key): + print(key) + return + + if args.build: + print("# build time dependencies") + for package in content["build-system"]["requires"]: + print(package) + + if args.runtime: + if content[top_key].get("dependencies"): + print("# runtime dependencies") + for package in content[top_key].get("dependencies"): + print(package) + + if args.type: + if args.type not in content[top_key].get(opt_key): + raise ValueError( + f"The value {args.type} is not a type of optional dependency or `build-system`", + ) + print(f"# dependencies for {args.type}") + for item in content[top_key][opt_key][args.type]: + print(item) + + elif content[top_key].get(opt_key): + for key in content[top_key].get(opt_key): + print(f"# dependencies for {key}") + for item in content[top_key][opt_key][key]: + print(item) + + +if __name__ == "__main__": + main() From aa533accd2b3614c692af12e33f50f7cef09fa30 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 11 Feb 2024 21:35:45 +0100 Subject: [PATCH 022/209] Use `show_deps.py` to get requirements in CI --- .github/workflows/coverage.yml | 2 +- .github/workflows/nmodl-ci.yml | 2 +- .github/workflows/nmodl-doc.yml | 2 +- .github/workflows/sonarsource.yml | 2 +- azure-pipelines.yml | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6d43825ccd..390ada396e 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -46,7 +46,7 @@ jobs: working-directory: ${{runner.workspace}}/nmodl run: | pip3 install -U pip setuptools - pip3 install --user -r requirements.txt + pip3 install --user -r <(packaging/show_deps.py -b -r .) - name: Restore compiler cache uses: actions/cache@v4 with: diff --git a/.github/workflows/nmodl-ci.yml b/.github/workflows/nmodl-ci.yml index 0341cedef1..9c50aab456 100644 --- a/.github/workflows/nmodl-ci.yml +++ b/.github/workflows/nmodl-ci.yml @@ -83,7 +83,7 @@ jobs: working-directory: ${{runner.workspace}}/nmodl run: | python3 -m pip install -U pip setuptools - python3 -m pip install --user -r requirements.txt + python3 -m pip install --user -r <(packaging/show_deps.py -b -r .) - name: Install neuron-nightly if: ${{matrix.config.enable_usecases == 'On'}} diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index 22ab1192b3..5507b9ac3a 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -59,7 +59,7 @@ jobs: working-directory: ${{runner.workspace}}/nmodl run: | pip3 install -U pip setuptools - pip3 install --user -r requirements.txt + pip3 install --user -r $(packaging/show_deps.py -r -b .) # This step will set up an SSH connection on tmate.io for live debugging. # To trigger it, simply add 'live-debug-docs' to your last pushed commit message. diff --git a/.github/workflows/sonarsource.yml b/.github/workflows/sonarsource.yml index 45b09e7457..1efff66043 100644 --- a/.github/workflows/sonarsource.yml +++ b/.github/workflows/sonarsource.yml @@ -32,7 +32,7 @@ jobs: working-directory: ${{runner.workspace}}/nmodl run: | pip3 install -U pip setuptools - pip3 install --user -r requirements.txt + pip3 install --user -r <(packaging/show_deps.py -r -b .) - name: Install sonar-scanner and build-wrapper uses: SonarSource/sonarcloud-github-c-cpp@v2 - name: Configure project diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d68952e4ae..bdc331d1b7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,7 +88,7 @@ stages: sudo apt-get install -y python3.8 python3.8-dev python3.8-venv ninja-build sudo apt-get remove -y python3-importlib-metadata python3.8 -m pip install --upgrade pip setuptools - python3.8 -m pip install --user -r $(Build.Repository.LocalPath)/requirements.txt + python3.8 -m pip install --user -r <($(Build.Repository.LocalPath)/packaging/show_deps.py -r -b $(Build.Repository.LocalPath)) # we manually get version 3.15.0 to make sure that changes in the cmake # files do not require unsupported versions of cmake in our package. wget --quiet --output-document=- "https://github.com/Kitware/CMake/releases/download/$CMAKE_VER/$CMAKE_PKG.tar.gz" | tar xzpf - @@ -149,7 +149,7 @@ stages: - script: | brew install flex bison cmake python@3 python3 -m pip install --upgrade pip setuptools - python3 -m pip install --user -r $(Build.Repository.LocalPath)/requirements.txt + python3 -m pip install --user -r <($(Build.Repository.LocalPath)/packaging/show_deps.py -r -b $(Build.Repository.LocalPath)) displayName: 'Install Dependencies' - script: | export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH; From 8c0181ff9607cb3e8f6f98fc21bcc37f56ed980b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 11 Feb 2024 22:22:26 +0100 Subject: [PATCH 023/209] Fix typo in docs CI --- .github/workflows/nmodl-doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index 5507b9ac3a..c0959fa15d 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -59,7 +59,7 @@ jobs: working-directory: ${{runner.workspace}}/nmodl run: | pip3 install -U pip setuptools - pip3 install --user -r $(packaging/show_deps.py -r -b .) + pip3 install --user -r <(packaging/show_deps.py -r -b .) # This step will set up an SSH connection on tmate.io for live debugging. # To trigger it, simply add 'live-debug-docs' to your last pushed commit message. From 3686a04189417a4bfcd9314c1c90678ab2a78feb Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 09:42:54 +0100 Subject: [PATCH 024/209] Add numpy to test deps --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index afcf022059..f8d48135c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ ] scripts = {nmodl = "nmodl._binwrapper:main"} -optional-dependencies.test = ["pytest>=3.3.0", "pytest-cov"] +optional-dependencies.test = ["pytest>=3.3.0", "pytest-cov", "numpy"] optional-dependencies.docs = [ "jupyter-client", "jupyter", From 0c72f231d36b9b21b3a874ac4a25bc26a08de64e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 11:45:25 +0100 Subject: [PATCH 025/209] Remove setup.py --- packaging/build_wheels.bash | 4 ++-- setup.py | 20 -------------------- 2 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 setup.py diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index dd62686f5d..fd38464067 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -12,8 +12,8 @@ set -xe # - python (>=3.8) # - C/C++ compiler -if [ ! -f setup.py ]; then - echo "Error: setup.py not found. Please launch $0 from the nmodl root dir" +if ! [ -f setup.py ] || ! [ -f pyproject.toml ]; then + echo "Error: setup.py or pyproject.toml not found. Please launch $0 from the nmodl root dir" exit 1 fi diff --git a/setup.py b/setup.py deleted file mode 100644 index dde4e0bc6d..0000000000 --- a/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 Blue Brain Project, EPFL. -# See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: Apache-2.0 -import sys - -from skbuild import setup - - -cmake_args = ["-DPYTHON_EXECUTABLE=" + sys.executable] -if "bdist_wheel" in sys.argv: - cmake_args.append("-DLINK_AGAINST_PYTHON=FALSE") - cmake_args.append("-DNMODL_ENABLE_TESTS=FALSE") - -setup( - packages=["nmodl"], - include_package_data=True, - cmake_minimum_required_version="3.15.0", - cmake_args=cmake_args, -) From 5af535493bdd832d6d8dcbccf7290411edb583a8 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 11:46:20 +0100 Subject: [PATCH 026/209] Add min cmake version --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f8d48135c3..ce7688a6c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ build-backend = "scikit_build_core.build" [tool.scikit-build] cmake.verbose = true +cmake.version = ">=3.15.0" metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" wheel.packages = ["nmodl"] From f7e08da440a92785e15546ad1516c852d672619c Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 14:34:44 +0100 Subject: [PATCH 027/209] Reorder items in pyproject.toml --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ce7688a6c8..30884dfec8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,11 +42,13 @@ requires = [ build-backend = "scikit_build_core.build" [tool.scikit-build] -cmake.verbose = true -cmake.version = ">=3.15.0" metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" wheel.packages = ["nmodl"] +[tool.scikit-build.cmake] +verbose = true +version = ">=3.15.0" + [tool.scikit-build.cmake.define] LINK_AGAINST_PYTHON = "FALSE" NMODL_ENABLE_TESTS = "FALSE" From 1b3562e251c6922d8750ae4ccf839c01a0f6d25f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 16:39:42 +0100 Subject: [PATCH 028/209] Enable debug logging when building wheel --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 30884dfec8..fdc7830499 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ build-backend = "scikit_build_core.build" [tool.scikit-build] metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" wheel.packages = ["nmodl"] +logging.level = "DEBUG" [tool.scikit-build.cmake] verbose = true From a6f3d57d8968716a2b92a3d90bc479ad0251ff78 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 16:40:24 +0100 Subject: [PATCH 029/209] Remove explicit build dir on MacOS --- packaging/build_wheels.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index fd38464067..75f248f961 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -73,7 +73,7 @@ build_wheel_osx() { echo " - Building..." rm -rf dist _skbuild - python -m build --wheel -o dist/ --config-setting='build-dir=_build' + python -m build --wheel -o dist/ echo " - Repairing..." delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel From 991d3e5bbd03d59d530d1c6c7e011ca87c6957f9 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 16:45:55 +0100 Subject: [PATCH 030/209] Fix condition in packaging script --- packaging/build_wheels.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 75f248f961..c87523487b 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -12,7 +12,7 @@ set -xe # - python (>=3.8) # - C/C++ compiler -if ! [ -f setup.py ] || ! [ -f pyproject.toml ]; then +if ! [ -f setup.py ] && ! [ -f pyproject.toml ]; then echo "Error: setup.py or pyproject.toml not found. Please launch $0 from the nmodl root dir" exit 1 fi From 9b34ba1777e2f3e81f4dc39538130e855baa09ac Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 17:09:57 +0100 Subject: [PATCH 031/209] Fix script for testing wheels --- packaging/test_wheel.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 0552fd3f0e..25f7eb8f78 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -2,7 +2,7 @@ # A simple set of tests checking if a wheel is working correctly set -xe -if [ ! -f setup.py ]; then +if ! [ -f setup.py ] && ! [ -f pyproject.toml ]; then echo "Error: Please launch $0 from the root dir" exit 1 fi From 0301ef7e601bc0cbdd847d5618030b3e1ee27a3b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 12 Feb 2024 19:15:31 +0100 Subject: [PATCH 032/209] Add workaround for symlinked /tmp on MacOS --- packaging/build_wheels.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index c87523487b..3dd900f81e 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -73,7 +73,9 @@ build_wheel_osx() { echo " - Building..." rm -rf dist _skbuild - python -m build --wheel -o dist/ + # the custom `build-dir` is a workaround for this issue: + # https://gitlab.kitware.com/cmake/cmake/-/issues/20107 + python -m build --wheel -o dist/ -C build-dir=_build echo " - Repairing..." delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel From c22b23dec541861113e43b482d270aab57a23fde Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 10:01:35 +0100 Subject: [PATCH 033/209] Relax build and runtime reqs --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fdc7830499..f338121f66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,8 +10,6 @@ dynamic = ["version"] dependencies = [ "find_libpython", "sympy>=1.3", - "Jinja2>=2.9.3", - "PyYAML>=3.13", ] scripts = {nmodl = "nmodl._binwrapper:main"} @@ -34,8 +32,6 @@ optional-dependencies.docs = [ requires = [ "scikit-build-core", "setuptools-scm>=8.0", - "find_libpython", - "sympy>=1.3", "Jinja2>=2.9.3", "PyYAML>=3.13", ] From 7cfc27cd773ee9ab21f36aa82bd846fb2b09ff80 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 10:03:24 +0100 Subject: [PATCH 034/209] Remove textwrap dependency It is part of the Python stdlib --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b4b6be91c6..6f6c009922 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,7 +167,6 @@ if(NMODL_ENABLE_TESTS) cpp_cc_find_python_module(pytest 3.3.0 REQUIRED) endif() cpp_cc_find_python_module(sympy 1.3 REQUIRED) -cpp_cc_find_python_module(textwrap 0.9 REQUIRED) cpp_cc_find_python_module(yaml 3.12 REQUIRED) # ============================================================================= From e18e851d7acb02d41d2ffc0708b8e7f4cd461bba Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 10:13:06 +0100 Subject: [PATCH 035/209] Rearrange cmake options - added `NMODL_BUILD_WHEEL` option, which automatically turns off `NMODL_ENABLE_TESTS` and `LINK_AGAINST_PYTHON` - made sympy only required if we are not building a wheel --- CMakeLists.txt | 10 +++++++++- pyproject.toml | 3 +-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6f6c009922..c78e1f8974 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,12 @@ set(NMODL_EXTRA_CXX_FLAGS "" CACHE STRING "Add extra compile flags for NMODL sources") separate_arguments(NMODL_EXTRA_CXX_FLAGS) +option(LINK_AGAINST_PYTHON "Should the Python library be linked or not" ON) +option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) +if(NMODL_BUILD_WHEEL) + set(LINK_AGAINST_PYTHON OFF) + set(NMODL_ENABLE_TESTS OFF) +endif() # ============================================================================= # Settings to enable project as submodule @@ -166,7 +172,9 @@ cpp_cc_find_python_module(jinja2 2.9.3 REQUIRED) if(NMODL_ENABLE_TESTS) cpp_cc_find_python_module(pytest 3.3.0 REQUIRED) endif() -cpp_cc_find_python_module(sympy 1.3 REQUIRED) +if(NOT NMODL_BUILD_WHEEL) + cpp_cc_find_python_module(sympy 1.3 REQUIRED) +endif() cpp_cc_find_python_module(yaml 3.12 REQUIRED) # ============================================================================= diff --git a/pyproject.toml b/pyproject.toml index f338121f66..b0362a4eef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,6 @@ verbose = true version = ">=3.15.0" [tool.scikit-build.cmake.define] -LINK_AGAINST_PYTHON = "FALSE" -NMODL_ENABLE_TESTS = "FALSE" +NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] From 5e51d2a78d41779fbfd5ea3ea060be86d58f3011 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 11:51:48 +0100 Subject: [PATCH 036/209] Be explicit about Python version in pyproject --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b0362a4eef..42875415f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "sympy>=1.3", ] scripts = {nmodl = "nmodl._binwrapper:main"} +requires-python = ">=3.8" optional-dependencies.test = ["pytest>=3.3.0", "pytest-cov", "numpy"] optional-dependencies.docs = [ From bcabef3d881171ab4ba7cbfefaf9995196727add Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 11:41:32 +0100 Subject: [PATCH 037/209] Add GHA for building wheels using cibuildwheel --- .github/workflows/nmodl-wheel.yml | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/nmodl-wheel.yml diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml new file mode 100644 index 0000000000..b29b7a38cc --- /dev/null +++ b/.github/workflows/nmodl-wheel.yml @@ -0,0 +1,35 @@ +name: Build wheels + +on: + push: + branches: + - master + pull_request: + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-13, macos-14] + + steps: + - uses: actions/checkout@v4 + + # Used to host cibuildwheel + - uses: actions/setup-python@v3 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.16.5 + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + # to supply options, put them in 'env', like: + # env: + # CIBW_SOME_OPTION: value + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl From f2986920038488abc7ae06b57fe6649e6a1e61d5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 11:59:06 +0100 Subject: [PATCH 038/209] Add flex and bison to CI --- .github/workflows/nmodl-wheel.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index b29b7a38cc..fc7960bb06 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -15,6 +15,17 @@ jobs: os: [ubuntu-latest, windows-latest, macos-13, macos-14] steps: + - name: Install external dependencies + run: | + if [[ "$RUNNER_OS" == "Linux" ]]; then + sudo apt-get update + sudo apt-get install -y flex bison + elif [[ "$RUNNER_OS" == "Windows" ]]; then + choco install flex bison -y + elif [[ "$RUNNER_OS" == "macOS" ]]; then + brew install flex bison + fi + - uses: actions/checkout@v4 # Used to host cibuildwheel From f0cac17adbc8a9281ff8ec5ac4cd0baadf482c93 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:01:13 +0100 Subject: [PATCH 039/209] Use GHA if to be platform independent --- .github/workflows/nmodl-wheel.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index fc7960bb06..781be798b8 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -15,16 +15,21 @@ jobs: os: [ubuntu-latest, windows-latest, macos-13, macos-14] steps: - - name: Install external dependencies + - name: Setup Flex and Bison + if: runner.os == 'Linux' run: | - if [[ "$RUNNER_OS" == "Linux" ]]; then - sudo apt-get update - sudo apt-get install -y flex bison - elif [[ "$RUNNER_OS" == "Windows" ]]; then - choco install flex bison -y - elif [[ "$RUNNER_OS" == "macOS" ]]; then - brew install flex bison - fi + sudo apt-get update + sudo apt-get install -y flex bison + + - name: Setup Flex and Bison + if: runner.os == 'Windows' + run: | + choco install flex bison -y + + - name: Setup Flex and Bison + if: runner.os == 'macOS' + run: | + brew install flex bison - uses: actions/checkout@v4 From de1ce19d307a2b36639e3c742a3612f847975569 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:03:53 +0100 Subject: [PATCH 040/209] Use winflexbison3 on Windows --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 781be798b8..c26e58198d 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -24,7 +24,7 @@ jobs: - name: Setup Flex and Bison if: runner.os == 'Windows' run: | - choco install flex bison -y + choco install winflexbison3 -y - name: Setup Flex and Bison if: runner.os == 'macOS' From 9c84c5e75fb3af30539abc3272b8341db5701fc7 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:12:04 +0100 Subject: [PATCH 041/209] Skip PyPy builds --- .github/workflows/nmodl-wheel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index c26e58198d..f8ab56b243 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -42,8 +42,8 @@ jobs: - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse # to supply options, put them in 'env', like: - # env: - # CIBW_SOME_OPTION: value + env: + CIBW_SKIP: pp* - uses: actions/upload-artifact@v4 with: From d126b6207cdd36f51855ed1df7f66edcc86a0bed Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:12:13 +0100 Subject: [PATCH 042/209] Don't build on ubuntu just yet --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index f8ab56b243..69575e3f7d 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-13, macos-14] + os: [windows-latest, macos-13, macos-14] steps: - name: Setup Flex and Bison From 944993e2081b9c84e1a374c2551ba4351294e526 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:18:25 +0100 Subject: [PATCH 043/209] Fix homebrewed bison and flex paths --- .github/workflows/nmodl-wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 69575e3f7d..77f8ca9a14 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -44,6 +44,7 @@ jobs: # to supply options, put them in 'env', like: env: CIBW_SKIP: pp* + CIBW_ENVIRONMENT_MACOS: PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" - uses: actions/upload-artifact@v4 with: From f515a75fc9731ef4a467fd30d1fe7f363fee286f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:22:41 +0100 Subject: [PATCH 044/209] Workaround for that stupid cmake symlink bug --- .github/workflows/nmodl-wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 77f8ca9a14..010c827ce7 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -45,6 +45,7 @@ jobs: env: CIBW_SKIP: pp* CIBW_ENVIRONMENT_MACOS: PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" + CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" - uses: actions/upload-artifact@v4 with: From c9e5379b6377a3c95e8884b1c63ae484bda1933e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:34:42 +0100 Subject: [PATCH 045/209] Workaround for filesystem path on MacOS For more info, see: https://github.com/pypa/cibuildwheel/discussions/1636 --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 010c827ce7..bcc4cda349 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -44,7 +44,7 @@ jobs: # to supply options, put them in 'env', like: env: CIBW_SKIP: pp* - CIBW_ENVIRONMENT_MACOS: PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" + CIBW_ENVIRONMENT_MACOS: PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" - uses: actions/upload-artifact@v4 From 12888dc5d6519bd2137a7318d1835ff22c740925 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:40:25 +0100 Subject: [PATCH 046/209] Workaround for homebrew path --- .github/workflows/nmodl-wheel.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index bcc4cda349..8442ba269e 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -44,7 +44,9 @@ jobs: # to supply options, put them in 'env', like: env: CIBW_SKIP: pp* - CIBW_ENVIRONMENT_MACOS: PATH="/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' + CIBW_ENVIRONMENT_MACOS: > + PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" + MACOSX_DEPLOYMENT_TARGET='10.15' CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" - uses: actions/upload-artifact@v4 From c7b1e3582ed0ef31003194fe4b9fe596c4322114 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 12:55:21 +0100 Subject: [PATCH 047/209] Remove POSIX-specific headers --- src/utils/common_utils.cpp | 1 - src/utils/common_utils.hpp | 2 -- src/utils/file_library.cpp | 2 -- 3 files changed, 5 deletions(-) diff --git a/src/utils/common_utils.cpp b/src/utils/common_utils.cpp index 9bcdeeb8f5..b9f79923a3 100644 --- a/src/utils/common_utils.cpp +++ b/src/utils/common_utils.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #if defined(WIN32) || defined(_WIN32) || defined(__WIN32) && !defined(__CYGWIN__) #define IS_WINDOWS diff --git a/src/utils/common_utils.hpp b/src/utils/common_utils.hpp index 147d46099e..539046416d 100644 --- a/src/utils/common_utils.hpp +++ b/src/utils/common_utils.hpp @@ -10,8 +10,6 @@ #include #include #include -#include -#include #include diff --git a/src/utils/file_library.cpp b/src/utils/file_library.cpp index 9293ec442b..9fb6b27170 100644 --- a/src/utils/file_library.cpp +++ b/src/utils/file_library.cpp @@ -9,8 +9,6 @@ #include #include -#include -#include #include "utils/common_utils.hpp" #include "utils/string_utils.hpp" From 3f4edd9c7d2c0eb5062e65b33a9ee7cb09e5036d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 13:23:14 +0100 Subject: [PATCH 048/209] Rename `parser.py` to `language_parser.py` This is to avoid confusion with the built-in `parser` module (removed since Python 3.10). --- src/language/code_generator.cmake | 2 +- src/language/code_generator.py | 2 +- src/language/{parser.py => language_parser.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/language/{parser.py => language_parser.py} (100%) diff --git a/src/language/code_generator.cmake b/src/language/code_generator.cmake index 90705ac3df..0c6ccae181 100644 --- a/src/language/code_generator.cmake +++ b/src/language/code_generator.cmake @@ -36,7 +36,7 @@ set(CODE_GENERATOR_PY_FILES ${PROJECT_SOURCE_DIR}/src/language/code_generator.py ${PROJECT_SOURCE_DIR}/src/language/node_info.py ${PROJECT_SOURCE_DIR}/src/language/nodes.py - ${PROJECT_SOURCE_DIR}/src/language/parser.py + ${PROJECT_SOURCE_DIR}/src/language/language_parser.py ${PROJECT_SOURCE_DIR}/src/language/utils.py ) diff --git a/src/language/code_generator.py b/src/language/code_generator.py index 63fa9743dd..f2d09ae49a 100644 --- a/src/language/code_generator.py +++ b/src/language/code_generator.py @@ -18,7 +18,7 @@ import jinja2 -from parser import LanguageParser +from language_parser import LanguageParser import node_info import utils diff --git a/src/language/parser.py b/src/language/language_parser.py similarity index 100% rename from src/language/parser.py rename to src/language/language_parser.py From f7479b48c14b6bf56620732da64ec2153d869790 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 14:10:51 +0100 Subject: [PATCH 049/209] Disable Windows build for now --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 8442ba269e..e404a31533 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, macos-13, macos-14] + os: [macos-13, macos-14] steps: - name: Setup Flex and Bison From 7a0066c28999ebfded07a8ce569ad9082387f63c Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 14:39:56 +0100 Subject: [PATCH 050/209] Use custom manylinux image for x86 and arm64 --- .github/workflows/nmodl-wheel.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index e404a31533..eb84a508c4 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -12,15 +12,9 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-13, macos-14] + os: [ubuntu-latest] steps: - - name: Setup Flex and Bison - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y flex bison - - name: Setup Flex and Bison if: runner.os == 'Windows' run: | @@ -48,6 +42,9 @@ jobs: PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" + CIBW_MANYLINUX_X86_64_IMAGE: docker.io/bluebrain/nmodl:wheel + # use custom image for now + CIBW_MANYLINUX_AARCH64_IMAGE: docker.io/jcgoran/nmodl:wheel-aarch64 - uses: actions/upload-artifact@v4 with: From e4e3be1ad69b5c7a1854fc89f0f3c4a02ca0222b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 14:51:50 +0100 Subject: [PATCH 051/209] Use `CIBW_BEFORE_BUILD_LINUX` to install bison/flex --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index eb84a508c4..f848d7f893 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -42,9 +42,9 @@ jobs: PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" - CIBW_MANYLINUX_X86_64_IMAGE: docker.io/bluebrain/nmodl:wheel # use custom image for now CIBW_MANYLINUX_AARCH64_IMAGE: docker.io/jcgoran/nmodl:wheel-aarch64 + CIBW_BEFORE_BUILD_LINUX: yum install -y bison flex - uses: actions/upload-artifact@v4 with: From 79835d99b5607ef8d49b7849a839342b8f678f80 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 14:58:32 +0100 Subject: [PATCH 052/209] Add custom deps on Linux CI --- .github/workflows/nmodl-wheel.yml | 4 +++- packaging/install_dependencies.sh | 37 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 packaging/install_dependencies.sh diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index f848d7f893..08d8016526 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -42,9 +42,11 @@ jobs: PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" + CIBW_ENVIRONMENT_LINUX: > + PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH # use custom image for now CIBW_MANYLINUX_AARCH64_IMAGE: docker.io/jcgoran/nmodl:wheel-aarch64 - CIBW_BEFORE_BUILD_LINUX: yum install -y bison flex + CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh - uses: actions/upload-artifact@v4 with: diff --git a/packaging/install_dependencies.sh b/packaging/install_dependencies.sh new file mode 100644 index 0000000000..3c98d18b4c --- /dev/null +++ b/packaging/install_dependencies.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# TODO put all of this inside of a Docker image to save time + +set -xe + +yum -y install \ + git \ + wget \ + make \ + vim \ + curl \ + unzip \ + autoconf \ + automake \ + make \ + openssh-server \ + libtool \ + && yum -y clean all \ + && rm -rf /var/cache + +curl -L -o flex-2.6.4.tar.gz https://github.com/westes/flex/files/981163/flex-2.6.4.tar.gz \ + && tar -xvzf flex-2.6.4.tar.gz \ + && cd flex-2.6.4 \ + && ./configure --prefix=/nmodlwheel/flex \ + && make -j 3 install \ + && cd .. \ + && rm -rf flex-2.6.4.tar.gz flex-2.6.4 +curl -L -o bison-3.7.3.tar.gz https://ftp.gnu.org/gnu/bison/bison-3.7.3.tar.gz \ + && tar -xvzf bison-3.7.3.tar.gz \ + && cd bison-3.7.3 \ + && ./configure --prefix=/nmodlwheel/bison \ + && make -j 3 install \ + && cd .. \ + && rm -rf bison-3.7.3.tar.gz bison-3.7.3 + +set +xe From 0ff22e9a2f2a6e7ad62d3155075e7b846d016c73 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 15:38:52 +0100 Subject: [PATCH 053/209] Skip 32-bit builds for now --- .github/workflows/nmodl-wheel.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 08d8016526..e0a0c39029 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -37,7 +37,8 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse # to supply options, put them in 'env', like: env: - CIBW_SKIP: pp* + # we skip PyPy (for now) and 32-bit builds + CIBW_SKIP: "pp* *-win32 *-manylinux_i686" CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' From 544d5e0a8445e4fd51cfe7cebc763d11fb39fc7e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 15:41:23 +0100 Subject: [PATCH 054/209] TEMP remove x86 so arm can actually try to build --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index e0a0c39029..327f63510f 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -38,7 +38,7 @@ jobs: # to supply options, put them in 'env', like: env: # we skip PyPy (for now) and 32-bit builds - CIBW_SKIP: "pp* *-win32 *-manylinux_i686" + CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64" CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' From 8dc35b930a58e0cbd37d8e0dd49bbc48d64035f9 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 15:57:15 +0100 Subject: [PATCH 055/209] TEMP also remove musllinux stuff so we can actually build arm --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 327f63510f..876f68f8da 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -38,7 +38,7 @@ jobs: # to supply options, put them in 'env', like: env: # we skip PyPy (for now) and 32-bit builds - CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64" + CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64 *-musllinux_i686 *-musllinux-x86_64" CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' From c4a1ffe7240fc28d9f3a67d650a05ef472e4bd6e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 16:01:11 +0100 Subject: [PATCH 056/209] TEMP typo --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 876f68f8da..463bbfec59 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -38,7 +38,7 @@ jobs: # to supply options, put them in 'env', like: env: # we skip PyPy (for now) and 32-bit builds - CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64 *-musllinux_i686 *-musllinux-x86_64" + CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64 *-musllinux_i686 *-musllinux_x86_64" CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' From 33ff2e84a09339e413141064fde5724066b435fe Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 16:07:37 +0100 Subject: [PATCH 057/209] TEMP Only build arm? --- .github/workflows/nmodl-wheel.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 463bbfec59..43adedef00 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -38,7 +38,8 @@ jobs: # to supply options, put them in 'env', like: env: # we skip PyPy (for now) and 32-bit builds - CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64 *-musllinux_i686 *-musllinux_x86_64" + #CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64 *-musllinux_i686 *-musllinux_x86_64" + CIBW_BUILD: cp39-manylinux_aarch64 CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' @@ -46,7 +47,7 @@ jobs: CIBW_ENVIRONMENT_LINUX: > PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH # use custom image for now - CIBW_MANYLINUX_AARCH64_IMAGE: docker.io/jcgoran/nmodl:wheel-aarch64 + #CIBW_MANYLINUX_AARCH64_IMAGE: docker.io/jcgoran/nmodl:wheel-aarch64 CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh - uses: actions/upload-artifact@v4 From 4c0ec4ec96bf3ed1234438510579371910909cf3 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 16:12:35 +0100 Subject: [PATCH 058/209] TEMP maybe it works now? Used this guide here: https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation --- .github/workflows/nmodl-wheel.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 43adedef00..4db92f949e 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -25,6 +25,13 @@ jobs: run: | brew install flex bison + # for multi-arch builds + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + - uses: actions/checkout@v4 # Used to host cibuildwheel @@ -40,6 +47,7 @@ jobs: # we skip PyPy (for now) and 32-bit builds #CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64 *-musllinux_i686 *-musllinux_x86_64" CIBW_BUILD: cp39-manylinux_aarch64 + CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' From 237ea1277b5178a69334a26567503a5640e4f088 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 16:52:12 +0100 Subject: [PATCH 059/209] The one that more or less works (minus Windows) --- .github/workflows/nmodl-wheel.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 4db92f949e..6956e5404f 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] + os: [ubuntu-latest, windows-latest, macos-13, macos-14] steps: - name: Setup Flex and Bison @@ -44,18 +44,17 @@ jobs: run: python -m cibuildwheel --output-dir wheelhouse # to supply options, put them in 'env', like: env: - # we skip PyPy (for now) and 32-bit builds - #CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-manylinux_x86_64 *-musllinux_i686 *-musllinux_x86_64" - CIBW_BUILD: cp39-manylinux_aarch64 - CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x + # we skip PyPy, 32-bit builds, and musllinux builds + CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-musllinux_i686 *-musllinux_x86_64" + # MacOS-specific setup CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" MACOSX_DEPLOYMENT_TARGET='10.15' CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" + # Linux-specific setup CIBW_ENVIRONMENT_LINUX: > PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH - # use custom image for now - #CIBW_MANYLINUX_AARCH64_IMAGE: docker.io/jcgoran/nmodl:wheel-aarch64 + # TODO: replace this with a custom Docker image to speed-up builds CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh - uses: actions/upload-artifact@v4 From d4bf54a5f91ea04e9cca9abc25730574608ce5e5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 16:52:39 +0100 Subject: [PATCH 060/209] TEMP windows debug --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 6956e5404f..1d21837104 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-13, macos-14] + os: [windows-latest] steps: - name: Setup Flex and Bison From 740931979f806fdb86bc1a60689d6966a6dcfeac Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 16:53:09 +0100 Subject: [PATCH 061/209] Add circleCI for aarch64 Linux builds --- .circleci/config.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..2fc301e4e5 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,25 @@ +version: 2 + +jobs: + linux-aarch64-wheels: + working_directory: ~/linux-aarch64-wheels + machine: + image: ubuntu-2004:2022.04.1 + # resource_class is what tells CircleCI to use an ARM worker for native arm builds + # https://circleci.com/product/features/resource-classes/ + resource_class: arm.medium + steps: + - checkout + - run: + name: Build the Linux aarch64 wheels. + command: | + python3 -m pip install --user cibuildwheel==2.16.5 + python3 -m cibuildwheel --output-dir wheelhouse + - store_artifacts: + path: wheelhouse/ + +workflows: + version: 2 + all-tests: + jobs: + - linux-aarch64-wheels From f71b07c5ba80906a758214648b6c9f47a52c4414 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 17:08:17 +0100 Subject: [PATCH 062/209] Add installation of flex and bison before --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fc301e4e5..c280e1b360 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,6 +12,8 @@ jobs: - checkout - run: name: Build the Linux aarch64 wheels. + environment: + CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh command: | python3 -m pip install --user cibuildwheel==2.16.5 python3 -m cibuildwheel --output-dir wheelhouse From e9893785dc9156849b7f10832049b0bea3f32462 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 17:12:10 +0100 Subject: [PATCH 063/209] Fix custom paths to flex and bison --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c280e1b360..2c7ef43513 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,8 @@ jobs: name: Build the Linux aarch64 wheels. environment: CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh + CIBW_ENVIRONMENT_LINUX: > + PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH command: | python3 -m pip install --user cibuildwheel==2.16.5 python3 -m cibuildwheel --output-dir wheelhouse From c593efcb4ab16b7c110e6b80a734b1de102df6b6 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 18:00:38 +0100 Subject: [PATCH 064/209] Try a fix for jinja and paths --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42875415f8..2c18d5c225 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ optional-dependencies.docs = [ requires = [ "scikit-build-core", "setuptools-scm>=8.0", - "Jinja2>=2.9.3", + "Jinja2 @ git+https://github.com/KapJI/jinja@unc-paths", "PyYAML>=3.13", ] build-backend = "scikit_build_core.build" From ee3a0959d931dde760965ee6c2729fccf7b05060 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 18:06:36 +0100 Subject: [PATCH 065/209] Revert "Try a fix for jinja and paths" This reverts commit c593efcb4ab16b7c110e6b80a734b1de102df6b6. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2c18d5c225..42875415f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ optional-dependencies.docs = [ requires = [ "scikit-build-core", "setuptools-scm>=8.0", - "Jinja2 @ git+https://github.com/KapJI/jinja@unc-paths", + "Jinja2>=2.9.3", "PyYAML>=3.13", ] build-backend = "scikit_build_core.build" From 2ac01526f9868aec0d88e47e68c0caf6d57d5f7f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 18:07:39 +0100 Subject: [PATCH 066/209] Skip pypy wheels on circleCI --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c7ef43513..5146e6b1f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,7 @@ jobs: - run: name: Build the Linux aarch64 wheels. environment: + CIBW_SKIP: "pp*" CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh CIBW_ENVIRONMENT_LINUX: > PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH From 290a6085d7b47a59498dee78ab64d292943d2abc Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 18:09:39 +0100 Subject: [PATCH 067/209] Also don't build musllinux for now --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5146e6b1f8..e1aa8f6f2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: - run: name: Build the Linux aarch64 wheels. environment: - CIBW_SKIP: "pp*" + CIBW_SKIP: "pp* *-musllinux_aarch64" CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh CIBW_ENVIRONMENT_LINUX: > PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH From ce6f64505ea9143034e6c6c36ad3ab3abf2699fc Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 15 Feb 2024 18:24:51 +0100 Subject: [PATCH 068/209] Be more strict about resolving paths maybe? --- src/language/code_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/language/code_generator.py b/src/language/code_generator.py index f2d09ae49a..52563b9ab1 100644 --- a/src/language/code_generator.py +++ b/src/language/code_generator.py @@ -59,7 +59,7 @@ class CodeGenerator( """ def __new__(cls, base_dir, clang_format=None): - this_dir = Path(__file__).parent.resolve() + this_dir = Path(__file__).parent.resolve(strict=True) jinja_templates_dir = this_dir / "templates" py_files = [Path(p).relative_to(this_dir) for p in this_dir.glob("*.py")] yaml_files = [Path(p).relative_to(this_dir) for p in this_dir.glob("*.yaml")] @@ -358,9 +358,9 @@ def parse_args(args=None): # destination directory to render templates args.base_dir = ( - Path(args.base_dir).resolve() + Path(args.base_dir).resolve(strict=True) if args.base_dir - else Path(__file__).resolve().parent.parent + else Path(__file__).resolve(strict=True).parent.parent ) return args From 32a67bae49ac33448a169067c0e3e47ec9e0147a Mon Sep 17 00:00:00 2001 From: JCGoran Date: Thu, 15 Feb 2024 21:50:09 +0100 Subject: [PATCH 069/209] Update .circleci/config.yml Use non-deprecated base image Co-authored-by: Nicolas Cornu --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1aa8f6f2d..2498a00f0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ jobs: linux-aarch64-wheels: working_directory: ~/linux-aarch64-wheels machine: - image: ubuntu-2004:2022.04.1 + image: default # resource_class is what tells CircleCI to use an ARM worker for native arm builds # https://circleci.com/product/features/resource-classes/ resource_class: arm.medium From 5225ea403f4b1239e0423fb9dfb2528c2556e18a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 14:22:33 +0100 Subject: [PATCH 070/209] TEMP Try the main jinja branch? --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 42875415f8..7a4cf0cf79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ optional-dependencies.docs = [ requires = [ "scikit-build-core", "setuptools-scm>=8.0", - "Jinja2>=2.9.3", + "Jinja2 @ git+https://github.com/pallets/jinja@main", "PyYAML>=3.13", ] build-backend = "scikit_build_core.build" From 6258e3466251a6959364b1f805e6f42b5949ef65 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 15:01:12 +0100 Subject: [PATCH 071/209] Revert "TEMP Try the main jinja branch?" This reverts commit 5225ea403f4b1239e0423fb9dfb2528c2556e18a. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7a4cf0cf79..42875415f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ optional-dependencies.docs = [ requires = [ "scikit-build-core", "setuptools-scm>=8.0", - "Jinja2 @ git+https://github.com/pallets/jinja@main", + "Jinja2>=2.9.3", "PyYAML>=3.13", ] build-backend = "scikit_build_core.build" From 16c50766568f58ee1bfef7c25be7fead66145055 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 15:02:31 +0100 Subject: [PATCH 072/209] TEMP only build on Python 3.11 on Windows --- .github/workflows/nmodl-wheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 1d21837104..c97dbe4ed2 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -46,6 +46,7 @@ jobs: env: # we skip PyPy, 32-bit builds, and musllinux builds CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-musllinux_i686 *-musllinux_x86_64" + CIBW_BUILD: "cp311-win_amd64" # MacOS-specific setup CIBW_ENVIRONMENT_MACOS: > PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" From 17aa9d1ec396168033d128c44c76a42eee59ec33 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 15:46:18 +0100 Subject: [PATCH 073/209] Fix for fmt installation --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c78e1f8974..05bbb138b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,10 @@ option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) set(NMODL_ENABLE_TESTS OFF) + # this is a sort of a workaround for the behavior described here: + # https://github.com/fmtlib/fmt/pull/3264 + # basically fmt will now not install itself in `/lib` and similar directories + set(FMT_INSTALL OFF) endif() # ============================================================================= From 2bd0b5c733da691fecc6adc32bf4dc23b2c094f0 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 15:46:18 +0100 Subject: [PATCH 074/209] Fix for fmt installation --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index c78e1f8974..05bbb138b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,10 @@ option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) set(NMODL_ENABLE_TESTS OFF) + # this is a sort of a workaround for the behavior described here: + # https://github.com/fmtlib/fmt/pull/3264 + # basically fmt will now not install itself in `/lib` and similar directories + set(FMT_INSTALL OFF) endif() # ============================================================================= From 1807b9736b829b3493530ac2770cab5607d80b6b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 16:50:02 +0100 Subject: [PATCH 075/209] Make formatter happt --- CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 05bbb138b7..55e3ce3e15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,9 +30,6 @@ option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) set(NMODL_ENABLE_TESTS OFF) - # this is a sort of a workaround for the behavior described here: - # https://github.com/fmtlib/fmt/pull/3264 - # basically fmt will now not install itself in `/lib` and similar directories set(FMT_INSTALL OFF) endif() From af7b123fc2333ca2941679a26fe776379a34a7d9 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 16:50:25 +0100 Subject: [PATCH 076/209] Make formatter happy --- CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 05bbb138b7..55e3ce3e15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,9 +30,6 @@ option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) set(NMODL_ENABLE_TESTS OFF) - # this is a sort of a workaround for the behavior described here: - # https://github.com/fmtlib/fmt/pull/3264 - # basically fmt will now not install itself in `/lib` and similar directories set(FMT_INSTALL OFF) endif() From 74052a576950bcfc6c43eeb5e53db911709acb92 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 17:15:59 +0100 Subject: [PATCH 077/209] Move custom cibuildwheel config to pyproject.toml This allows easier maintenance if using different CIs --- .circleci/config.yml | 3 --- .github/workflows/nmodl-wheel.yml | 10 ---------- pyproject.toml | 8 ++++++++ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2498a00f0a..390325239b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,9 +14,6 @@ jobs: name: Build the Linux aarch64 wheels. environment: CIBW_SKIP: "pp* *-musllinux_aarch64" - CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh - CIBW_ENVIRONMENT_LINUX: > - PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH command: | python3 -m pip install --user cibuildwheel==2.16.5 python3 -m cibuildwheel --output-dir wheelhouse diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index c97dbe4ed2..c0db305f9c 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -47,16 +47,6 @@ jobs: # we skip PyPy, 32-bit builds, and musllinux builds CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-musllinux_i686 *-musllinux_x86_64" CIBW_BUILD: "cp311-win_amd64" - # MacOS-specific setup - CIBW_ENVIRONMENT_MACOS: > - PATH="/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH" - MACOSX_DEPLOYMENT_TARGET='10.15' - CIBW_CONFIG_SETTINGS_MACOS: "build-dir=_build" - # Linux-specific setup - CIBW_ENVIRONMENT_LINUX: > - PATH=/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH - # TODO: replace this with a custom Docker image to speed-up builds - CIBW_BEFORE_BUILD_LINUX: bash packaging/install_dependencies.sh - uses: actions/upload-artifact@v4 with: diff --git a/pyproject.toml b/pyproject.toml index 42875415f8..d9e9d68951 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,3 +51,11 @@ version = ">=3.15.0" NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] + +[tool.cibuildwheel.macos] +environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } +config-settings = {build-dir = "_build"} + +[tool.cibuildwheel.linux] +before-build = "bash packaging/install_dependencies.sh" +environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } From 868376274c5d8bab3db9bc3be708c43c5eabf1c7 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 16 Feb 2024 17:20:20 +0100 Subject: [PATCH 078/209] Use `before-all` instead of `before-build` for ext For flex and bison installs, we only need to do this once (period), instead of once per wheel. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d9e9d68951..ae4a107141 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,5 +57,5 @@ environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/ config-settings = {build-dir = "_build"} [tool.cibuildwheel.linux] -before-build = "bash packaging/install_dependencies.sh" +before-all = "bash packaging/install_dependencies.sh" environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } From 5b3dbfd994439882684d8618bce4c40cb735b6ba Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 09:39:26 +0100 Subject: [PATCH 079/209] Add testing using cibuildwheel --- packaging/test_wheel.bash | 67 +++++++-------------------------------- pyproject.toml | 3 ++ 2 files changed, 15 insertions(+), 55 deletions(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 25f7eb8f78..595f9310e0 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -2,69 +2,26 @@ # A simple set of tests checking if a wheel is working correctly set -xe -if ! [ -f setup.py ] && ! [ -f pyproject.toml ]; then - echo "Error: Please launch $0 from the root dir" - exit 1 -fi -if [ "$#" -lt 2 ]; then - echo "Usage: $(basename $0) python_exe python_wheel [use_virtual_env]" +if [ -z "${VIRTUAL_ENV}" ] +then + echo "ERROR: you do not appear to be in a virtual environment, please use one before running tests" exit 1 fi -# cli parameters -python_exe=$1 -python_wheel=$2 -use_venv=$3 #if $3 is not "false" then use virtual environment - -python_ver=$("$python_exe" -c "import sys; print('%d%d' % tuple(sys.version_info)[:2])") - - test_wheel () { + # the path to the root directory + rootdir="$1" # sample mod file for nrnivmodl check - local TEST_DIR="test_dir" - mkdir -p $TEST_DIR - cp ../nmodl/ext/example/*.mod $TEST_DIR/ - cp ../test/integration/mod/cabpump.mod ../test/integration/mod/var_init.inc $TEST_DIR/ - cd $TEST_DIR - for mod in *.mod + test_dir="$(mktemp -d)" + cp "${rootdir}"/nmodl/ext/example/*.mod "${test_dir}" + cp "${rootdir}/test/integration/mod/cabpump.mod" "${rootdir}/test/integration/mod/var_init.inc" "${test_dir}" + for mod in "${test_dir}"/*.mod do - nmodl $mod sympy --analytic + nmodl "$mod" sympy --analytic done - $python_exe -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('hh.mod')" - cd .. - #clean-up - rm -rf $TEST_DIR + python -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('${test_dir}/hh.mod')" } -echo "== Testing $python_wheel using $python_exe ($python_ver) ==" - -mkdir testwheel && cd testwheel - -# creat python virtual environment and use `python` as binary name -# because it will be correct one from venv. -if [[ "$use_venv" != "false" ]]; then - echo " == Creating virtual environment == " - venv_name="nmodl_test_venv_${python_ver}" - $python_exe -m venv $venv_name - . $venv_name/bin/activate - python_exe=`which python` -else - echo " == Using global install == " -fi - -# install nmodl -$python_exe -m pip install -U pip -$python_exe -m pip install ../$python_wheel -$python_exe -m pip show nmodl || $python_exe -m pip show nmodl-nightly - # run tests -test_wheel $(which python) - -# cleanup -if [[ "$use_venv" != "false" ]]; then - deactivate -fi - -rm -rf $venv_name -echo "Removed $venv_name" +test_wheel "$@" diff --git a/pyproject.toml b/pyproject.toml index ae4a107141..69b371e806 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] +[tool.cibuildwheel] +test-command = "bash {package}/packaging/test_wheel.bash {package}" + [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } config-settings = {build-dir = "_build"} From b12a86202efc8c977705fc9bf230035973219525 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 09:51:23 +0100 Subject: [PATCH 080/209] Remove script for building wheels We do everything using cibuildwheel now --- packaging/build_wheels.bash | 117 ------------------------------------ 1 file changed, 117 deletions(-) delete mode 100755 packaging/build_wheels.bash diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash deleted file mode 100755 index 3dd900f81e..0000000000 --- a/packaging/build_wheels.bash +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -set -xe -# A script to loop over the available pythons installed -# on Linux/OSX and build wheels -# -# Note: It should be invoked from nmodl directory -# -# PREREQUESITES: -# - cmake (>=3.15) -# - flex (>= 2.6) -# - bison (>=3.0) -# - python (>=3.8) -# - C/C++ compiler - -if ! [ -f setup.py ] && ! [ -f pyproject.toml ]; then - echo "Error: setup.py or pyproject.toml not found. Please launch $0 from the nmodl root dir" - exit 1 -fi - -setup_venv() { - local py_bin="$1" - local py_ver=$("$py_bin" -c "import sys; print('%d%d' % tuple(sys.version_info)[:2])") - local venv_dir="nmodl_build_venv$py_ver" - - if [ "$py_ver" -lt 37 ]; then - echo "[SKIP] Python $py_ver not supported" - skip=1 - return 0 - fi - - echo " - Creating $venv_dir: $py_bin -m venv $venv_dir" - "$py_bin" -m venv "$venv_dir" - . "$venv_dir/bin/activate" - - if ! pip install --upgrade pip setuptools wheel; then - curl https://bootstrap.pypa.io/get-pip.py | python - pip install --upgrade setuptools wheel - fi - -} - - -build_wheel_linux() { - echo "[BUILD WHEEL] Building with interpreter $1" - local skip= - setup_venv "$1" - (( $skip )) && return 0 - - echo " - Installing build requirements" - pip install pip auditwheel setuptools build - - echo " - Building..." - rm -rf dist _skbuild - # Workaround for https://github.com/pypa/manylinux/issues/1309 - git config --global --add safe.directory "*" - python -m build --wheel -o dist/ - - echo " - Repairing..." - auditwheel repair dist/*.whl - - deactivate -} - - -build_wheel_osx() { - echo "[BUILD WHEEL] Building with interpreter $1" - local skip= - setup_venv "$1" - (( $skip )) && return 0 - - echo " - Installing build requirements" - pip install --upgrade delocate build - - echo " - Building..." - rm -rf dist _skbuild - # the custom `build-dir` is a workaround for this issue: - # https://gitlab.kitware.com/cmake/cmake/-/issues/20107 - python -m build --wheel -o dist/ -C build-dir=_build - - echo " - Repairing..." - delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel - - deactivate -} - -# platform for which wheel to be build -platform=$1 - -# python version for which wheel to be built; 3* (default) means all python 3 versions -python_wheel_version=3* -if [ "$2" ]; then - python_wheel_version=$2 -fi - -# MAIN - -case "$1" in - - linux) - python_wheel_version=${python_wheel_version//[-._]/} - for py_bin in /opt/python/cp${python_wheel_version}*/bin/python; do - build_wheel_linux "$py_bin" - done - ;; - - osx) - for py_bin in /Library/Frameworks/Python.framework/Versions/${python_wheel_version}*/bin/python3; do - build_wheel_osx "$py_bin" - done - ;; - - *) - echo "Usage: $(basename $0) [version]" - exit 1 - ;; - -esac From 4900d1653d4be67e43fb8a6f69a62d8ed660aff0 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 09:52:24 +0100 Subject: [PATCH 081/209] Remove wheel building on Azure We are using GHA for building x86_64 wheel for Linux, MacOS, Windows, as well as MacOS arm64 wheels. --- azure-pipelines.yml | 122 -------------------------------------------- 1 file changed, 122 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bdc331d1b7..914163bc1c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -194,125 +194,3 @@ stages: env: SHELL: 'bash' displayName: 'Build Neuron and Run Integration Tests' - - job: 'manylinux_wheels' - timeoutInMinutes: 45 - pool: - vmImage: 'ubuntu-20.04' - strategy: - matrix: - ${{ if eq(variables.buildWheel, True) }}: - Python38: - python.version: '3.8' - Python39: - python.version: '3.9' - Python310: - python.version: '3.10' - Python311: - python.version: '3.11' - ${{ if eq(variables.buildWheel, False) }}: - Python311: - python.version: '3.11' - steps: - - checkout: self - submodules: True - condition: succeeded() - - script: | - if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then - export TAG="-nightly" - else - export TAG="" - fi - # the following 2 lines are a workaround for PEP 621 not allowing a - # dynamic `name` in `pyproject.toml` - python3 -m pip install --user tomli tomli-w - python3 packaging/change_name.py pyproject.toml "NMODL${TAG}" - docker run --rm \ - -w /root/nmodl \ - -v $PWD:/root/nmodl \ - -e NMODL_NIGHTLY_TAG=$TAG \ - 'bluebrain/nmodl:wheel' \ - packaging/build_wheels.bash linux $(python.version) - condition: succeeded() - displayName: 'Building ManyLinux Wheel' - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' - condition: succeeded() - displayName: 'Publish wheel as build artifact' - - script: | - sudo apt-add-repository -y ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv - packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl - condition: succeeded() - displayName: 'Test ManyLinux Wheel with Python $(python.version)' - - template: ci/upload-wheels.yml - - job: 'macos_wheels' - timeoutInMinutes: 45 - pool: - vmImage: 'macOS-11' - strategy: - matrix: - ${{ if eq(variables.buildWheel, True) }}: - Python38: - python.version: '3.8' - python.org.version: '3.8.10' - python.installer.name: 'macosx10.9.pkg' - Python39: - python.version: '3.9' - python.org.version: '3.9.13' - python.installer.name: 'macosx10.9.pkg' - Python310: - python.version: '3.10' - python.org.version: '3.10.5' - python.installer.name: 'macos11.pkg' - Python311: - python.version: '3.11' - python.org.version: '3.11.1' - python.installer.name: 'macos11.pkg' - ${{ if eq(variables.buildWheel, False) }}: - Python311: - python.version: '3.11' - python.org.version: '3.11.1' - python.installer.name: 'macos11.pkg' - steps: - - checkout: self - submodules: True - condition: succeeded() - - script: | - brew install flex bison cmake ninja - export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH; - condition: succeeded() - displayName: 'Install Dependencies' - - script: | - installer=python-$(python.org.version)-$(python.installer.name) - url=https://www.python.org/ftp/python/$(python.org.version)/$installer - curl $url -o $installer - sudo installer -pkg $installer -target / - condition: succeeded() - displayName: 'Install Python from python.org' - - script: | - export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH - export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) - if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then - export NMODL_NIGHTLY_TAG="-nightly" - else - export NMODL_NIGHTLY_TAG="" - fi - # the following 2 lines are a workaround for PEP 621 not allowing a - # dynamic `name` in `pyproject.toml` - python3 -m pip install tomli tomli-w - python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}" - packaging/build_wheels.bash osx $(python.version) - condition: succeeded() - displayName: 'Build macos Wheel' - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' - condition: succeeded() - displayName: 'Publish wheel as build artifact' - - script: | - packaging/test_wheel.bash python$(python.version) wheelhouse/*.whl - condition: succeeded() - displayName: 'Test macos Wheel with Python $(python.version)' - - template: ci/upload-wheels.yml From b0e4b35d25981454acbf717f3585599fd099116d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 09:55:58 +0100 Subject: [PATCH 082/209] Simplify tests even more --- packaging/test_wheel.bash | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 595f9310e0..e9fec5683f 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -2,13 +2,6 @@ # A simple set of tests checking if a wheel is working correctly set -xe - -if [ -z "${VIRTUAL_ENV}" ] -then - echo "ERROR: you do not appear to be in a virtual environment, please use one before running tests" - exit 1 -fi - test_wheel () { # the path to the root directory rootdir="$1" From 3da19ca386cd254f74a53df8d7cf80db29bee528 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 10:01:10 +0100 Subject: [PATCH 083/209] Add pytest tests --- pyproject.toml | 6 +++++- setup.cfg | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 69b371e806..c43d1028ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,8 @@ NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] [tool.cibuildwheel] -test-command = "bash {package}/packaging/test_wheel.bash {package}" +test-command = "bash {package}/packaging/test_wheel.bash {package} && pytest {package}/test/unit/pybind" +test-extras = ["test"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } @@ -62,3 +63,6 @@ config-settings = {build-dir = "_build"} [tool.cibuildwheel.linux] before-all = "bash packaging/install_dependencies.sh" environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } + +[tool.pytest] +testpaths = "test/unit/pybind" diff --git a/setup.cfg b/setup.cfg index 9de58dd72a..54a600626f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[tool:pytest] -testpaths = test/unit/pybind - [doctest] builder = doctest From 814a20302627a544a983a4a5cda74a736ed98bfb Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 14:22:24 +0100 Subject: [PATCH 084/209] TEMP try to install libpython? WIP on tests on manylinux --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index c43d1028ce..c21584eaef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ config-settings = {build-dir = "_build"} [tool.cibuildwheel.linux] before-all = "bash packaging/install_dependencies.sh" environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } +before-test = "yum -y install python$(\"import sys; print('%d%d' % tuple(sys.version_info)[:2])\")-devel" [tool.pytest] testpaths = "test/unit/pybind" From 8ccbbd09d594c369570c46e8d48b663bf8faff01 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 14:33:22 +0100 Subject: [PATCH 085/209] Remove tests for cli for now --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c21584eaef..7001a6a32c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] [tool.cibuildwheel] -test-command = "bash {package}/packaging/test_wheel.bash {package} && pytest {package}/test/unit/pybind" +test-command = "pytest {package}/test/unit/pybind" test-extras = ["test"] [tool.cibuildwheel.macos] @@ -63,7 +63,6 @@ config-settings = {build-dir = "_build"} [tool.cibuildwheel.linux] before-all = "bash packaging/install_dependencies.sh" environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } -before-test = "yum -y install python$(\"import sys; print('%d%d' % tuple(sys.version_info)[:2])\")-devel" [tool.pytest] testpaths = "test/unit/pybind" From f1f1f4a8d228676559d640a7e18602cf28efe480 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 15:42:16 +0100 Subject: [PATCH 086/209] Add setuptools to runtime deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 42875415f8..97ff041dd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dynamic = ["version"] dependencies = [ "find_libpython", "sympy>=1.3", + "setuptools", ] scripts = {nmodl = "nmodl._binwrapper:main"} requires-python = ">=3.8" From 5265c420aa5d2c0bbeeb3a8223bbf2252ceb047f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 17 Feb 2024 15:42:16 +0100 Subject: [PATCH 087/209] Add setuptools to runtime deps --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 7001a6a32c..7ed736da00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dynamic = ["version"] dependencies = [ "find_libpython", "sympy>=1.3", + "setuptools", ] scripts = {nmodl = "nmodl._binwrapper:main"} requires-python = ">=3.8" From 3e7cc6a728a4ef8e73201d1dbfd830c982129d2f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 18 Feb 2024 08:46:35 +0100 Subject: [PATCH 088/209] Move all setup.cfg settings to pyproject.toml --- pyproject.toml | 6 ++++++ setup.cfg | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 setup.cfg diff --git a/pyproject.toml b/pyproject.toml index 7ed736da00..24bf7ebe68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,3 +67,9 @@ environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } [tool.pytest] testpaths = "test/unit/pybind" + +[tool.doctest] +builder = "doctest" + +[tool.flake8] +max-line-length = 100 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 54a600626f..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[doctest] -builder = doctest - -[flake8] -max-line-length = 100 From eba6c6bf509e63dd65991f097354237bfcc0e5ff Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 18 Feb 2024 08:53:17 +0100 Subject: [PATCH 089/209] Working copy (without Windows) --- .github/workflows/nmodl-wheel.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index c0db305f9c..efdca66f28 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest] + os: [ubuntu-latest, macOS-13, macOS-14] steps: - name: Setup Flex and Bison @@ -25,13 +25,6 @@ jobs: run: | brew install flex bison - # for multi-arch builds - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - uses: actions/checkout@v4 # Used to host cibuildwheel From a54c558e22686f26a7d9c9fbe992e01f88f4612d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 18 Feb 2024 08:55:11 +0100 Subject: [PATCH 090/209] For real this time --- .github/workflows/nmodl-wheel.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index efdca66f28..4657fbfd75 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -39,7 +39,6 @@ jobs: env: # we skip PyPy, 32-bit builds, and musllinux builds CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-musllinux_i686 *-musllinux_x86_64" - CIBW_BUILD: "cp311-win_amd64" - uses: actions/upload-artifact@v4 with: From db9054ec4cffcb8053263ced446c3d57d5415957 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 18 Feb 2024 20:21:20 +0100 Subject: [PATCH 091/209] Greatly simplify the wheel testing --- packaging/test_wheel.bash | 19 +++---------------- pyproject.toml | 5 ++++- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index e9fec5683f..111d73a34e 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -2,19 +2,6 @@ # A simple set of tests checking if a wheel is working correctly set -xe -test_wheel () { - # the path to the root directory - rootdir="$1" - # sample mod file for nrnivmodl check - test_dir="$(mktemp -d)" - cp "${rootdir}"/nmodl/ext/example/*.mod "${test_dir}" - cp "${rootdir}/test/integration/mod/cabpump.mod" "${rootdir}/test/integration/mod/var_init.inc" "${test_dir}" - for mod in "${test_dir}"/*.mod - do - nmodl "$mod" sympy --analytic - done - python -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('${test_dir}/hh.mod')" -} - -# run tests -test_wheel "$@" +find "$1/test/" "$1/nmodl/ext/" \ + -name "*.mod" \ + -exec python -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('{}')" \; diff --git a/pyproject.toml b/pyproject.toml index 24bf7ebe68..171774896f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,10 @@ NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] [tool.cibuildwheel] -test-command = "pytest {package}/test/unit/pybind" +test-command = [ + "pytest {package}/test/unit/pybind", + "bash {package}/packaging/test_wheel.bash {package}", +] test-extras = ["test"] [tool.cibuildwheel.macos] From 5ff006d9af05575c4e1c51681978eb0ae80ab905 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 08:56:24 +0100 Subject: [PATCH 092/209] Revert "Be more strict about resolving paths maybe?" This reverts commit ce6f64505ea9143034e6c6c36ad3ab3abf2699fc. --- src/language/code_generator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/language/code_generator.py b/src/language/code_generator.py index 52563b9ab1..f2d09ae49a 100644 --- a/src/language/code_generator.py +++ b/src/language/code_generator.py @@ -59,7 +59,7 @@ class CodeGenerator( """ def __new__(cls, base_dir, clang_format=None): - this_dir = Path(__file__).parent.resolve(strict=True) + this_dir = Path(__file__).parent.resolve() jinja_templates_dir = this_dir / "templates" py_files = [Path(p).relative_to(this_dir) for p in this_dir.glob("*.py")] yaml_files = [Path(p).relative_to(this_dir) for p in this_dir.glob("*.yaml")] @@ -358,9 +358,9 @@ def parse_args(args=None): # destination directory to render templates args.base_dir = ( - Path(args.base_dir).resolve(strict=True) + Path(args.base_dir).resolve() if args.base_dir - else Path(__file__).resolve(strict=True).parent.parent + else Path(__file__).resolve().parent.parent ) return args From 51f761061018a00c4992889181c8204d6d2e1851 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 18 Feb 2024 08:46:35 +0100 Subject: [PATCH 093/209] Move some setup.cfg settings to pyproject.toml The two remaining options stay in setup.cfg for the following reasons: - flake8 does not support pyproject.toml yet: https://github.com/PyCQA/flake8/issues/234 - the doctest option is not completely transparent so I am leaving it there for now --- pyproject.toml | 3 +++ setup.cfg | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 97ff041dd3..a13024ea12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,3 +52,6 @@ version = ">=3.15.0" NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] + +[tool.pytest.ini_options] +testpaths = "test/unit/pybind" diff --git a/setup.cfg b/setup.cfg index 9de58dd72a..54a600626f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[tool:pytest] -testpaths = test/unit/pybind - [doctest] builder = doctest From 8e79c27e0fc8fd34feaa806cb750eab9a89f8d5a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sun, 18 Feb 2024 08:46:35 +0100 Subject: [PATCH 094/209] Move some setup.cfg settings to pyproject.toml The two remaining options stay in setup.cfg for the following reasons: - flake8 does not support pyproject.toml yet: https://github.com/PyCQA/flake8/issues/234 - the doctest option is not completely transparent so I am leaving it there for now --- pyproject.toml | 8 +------- setup.cfg | 5 +++++ 2 files changed, 6 insertions(+), 7 deletions(-) create mode 100644 setup.cfg diff --git a/pyproject.toml b/pyproject.toml index 171774896f..e4513f1e4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,11 +68,5 @@ config-settings = {build-dir = "_build"} before-all = "bash packaging/install_dependencies.sh" environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } -[tool.pytest] +[tool.pytest.ini_options] testpaths = "test/unit/pybind" - -[tool.doctest] -builder = "doctest" - -[tool.flake8] -max-line-length = 100 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..54a600626f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[doctest] +builder = doctest + +[flake8] +max-line-length = 100 From 8d67f69f20b49df0c7b00168774480c4969ce862 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 09:54:37 +0100 Subject: [PATCH 095/209] Remove `show_deps.py` in favor of `pip-tools` --- .github/workflows/coverage.yml | 8 +-- .github/workflows/nmodl-ci.yml | 6 +- .github/workflows/nmodl-doc.yml | 12 ++-- .github/workflows/sonarsource.yml | 4 +- azure-pipelines.yml | 6 +- packaging/show_deps.py | 112 ------------------------------ 6 files changed, 18 insertions(+), 130 deletions(-) delete mode 100755 packaging/show_deps.py diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 390ada396e..284702cd95 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -6,7 +6,7 @@ concurrency: on: push: - branches: + branches: - master - live-debug* - release/** @@ -41,12 +41,12 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - uses: actions/checkout@v4 with: - fetch-depth: 2 + fetch-depth: 2 - name: Install Python3 dependencies working-directory: ${{runner.workspace}}/nmodl run: | - pip3 install -U pip setuptools - pip3 install --user -r <(packaging/show_deps.py -b -r .) + pip3 install -U pip setuptools 'pip-tools>=7.4.0' + pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) - name: Restore compiler cache uses: actions/cache@v4 with: diff --git a/.github/workflows/nmodl-ci.yml b/.github/workflows/nmodl-ci.yml index 9c50aab456..fedf7fffbc 100644 --- a/.github/workflows/nmodl-ci.yml +++ b/.github/workflows/nmodl-ci.yml @@ -82,8 +82,8 @@ jobs: - name: Install Python3 dependencies working-directory: ${{runner.workspace}}/nmodl run: | - python3 -m pip install -U pip setuptools - python3 -m pip install --user -r <(packaging/show_deps.py -b -r .) + python3 -m pip install -U pip setuptools 'pip-tools>=7.4.0' + python3 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) - name: Install neuron-nightly if: ${{matrix.config.enable_usecases == 'On'}} @@ -190,7 +190,7 @@ jobs: path: ${{runner.workspace}}/nmodl/build/Testing/*/Test.xml # This step will set up an SSH connection on tmate.io for live debugging. - # To enable it, you have to: + # To enable it, you have to: # * add 'live-debug-tests' to your PR title # * push something to your PR branch (note that just re-running disregards the title update) - name: live debug session on failure (manual steps required, check `nmodl-ci.yml`) diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index c0959fa15d..5ec6d8b39c 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -6,7 +6,7 @@ concurrency: on: push: - branches: + branches: - master - release/** pull_request: @@ -37,7 +37,7 @@ jobs: uses: jwlawson/actions-setup-cmake@v1 with: cmake-version: ${{ env.DESIRED_CMAKE_VERSION }} - + - name: Install apt packages run: | sudo apt-get update @@ -58,8 +58,8 @@ jobs: - name: Install Python3 dependencies working-directory: ${{runner.workspace}}/nmodl run: | - pip3 install -U pip setuptools - pip3 install --user -r <(packaging/show_deps.py -r -b .) + pip3 install -U pip setuptools 'pip-tools>=7.4.0' + pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) # This step will set up an SSH connection on tmate.io for live debugging. # To trigger it, simply add 'live-debug-docs' to your last pushed commit message. @@ -77,7 +77,7 @@ jobs: docs-${{github.ref}}- docs- - - name: Documentation + - name: Documentation id: documentation working-directory: ${{runner.workspace}}/nmodl run: | @@ -93,7 +93,7 @@ jobs: echo "status=done" >> $GITHUB_OUTPUT env: CCACHE_DIR: ${{runner.workspace}}/ccache - + - name: Deploy 🚀 uses: JamesIves/github-pages-deploy-action@v4 if: steps.documentation.outputs.status == 'done' && startsWith(github.ref, 'refs/heads/master') diff --git a/.github/workflows/sonarsource.yml b/.github/workflows/sonarsource.yml index 1efff66043..27ed84cc08 100644 --- a/.github/workflows/sonarsource.yml +++ b/.github/workflows/sonarsource.yml @@ -31,8 +31,8 @@ jobs: - name: Install Python3 dependencies working-directory: ${{runner.workspace}}/nmodl run: | - pip3 install -U pip setuptools - pip3 install --user -r <(packaging/show_deps.py -r -b .) + pip3 install -U pip setuptools 'pip-tools>=7.4.0' + pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) - name: Install sonar-scanner and build-wrapper uses: SonarSource/sonarcloud-github-c-cpp@v2 - name: Configure project diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bdc331d1b7..2973ab173d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,8 +87,8 @@ stages: sudo apt-get install -y g++-9 flex bison libfl-dev cython libx11-dev libxcomposite-dev libncurses-dev mpich sudo apt-get install -y python3.8 python3.8-dev python3.8-venv ninja-build sudo apt-get remove -y python3-importlib-metadata - python3.8 -m pip install --upgrade pip setuptools - python3.8 -m pip install --user -r <($(Build.Repository.LocalPath)/packaging/show_deps.py -r -b $(Build.Repository.LocalPath)) + python3.8 -m pip install --upgrade pip setuptools 'pip-tools>=7.4.0' + python3.8 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) # we manually get version 3.15.0 to make sure that changes in the cmake # files do not require unsupported versions of cmake in our package. wget --quiet --output-document=- "https://github.com/Kitware/CMake/releases/download/$CMAKE_VER/$CMAKE_PKG.tar.gz" | tar xzpf - @@ -149,7 +149,7 @@ stages: - script: | brew install flex bison cmake python@3 python3 -m pip install --upgrade pip setuptools - python3 -m pip install --user -r <($(Build.Repository.LocalPath)/packaging/show_deps.py -r -b $(Build.Repository.LocalPath)) + python3 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) displayName: 'Install Dependencies' - script: | export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH; diff --git a/packaging/show_deps.py b/packaging/show_deps.py deleted file mode 100755 index 3a24bc0714..0000000000 --- a/packaging/show_deps.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -""" -Small script that can parse TOML files without any dependencies -""" -import sys -from argparse import ArgumentParser -from pathlib import Path - -if sys.version_info >= (3, 11): - # use the standard library module - from tomllib import load - -else: - import pip - - _earliest_pip_version = (21, 2, 0) - _current_pip_version = tuple(int(_) for _ in pip.__version__.split(".")) - if _current_pip_version >= _earliest_pip_version: - # use the vendored version in pip - from pip._vendor.tomli import load - - else: - # as a last resort, try to use tomli if it's installed - try: - from tomli import load - except ModuleNotFoundError as err: - raise ModuleNotFoundError( - f"pip version {pip.__version__} found, " - f"{'.'.join(map(str,_earliest_pip_version))} or above " - f"or the `tomli` package required\n" - "You can either install a newer version of pip (using `pip install -U pip`), " - "or install the `tomli` package (using `pip install tomli`)" - ) from err - - -def main(): - """ - Main module - """ - parser = ArgumentParser() - parser.add_argument( - "dir", - help="the directory containing the `pyproject.toml` file", - ) - parser.add_argument( - "--runtime", - "-r", - action="store_true", - help="show only runtime dependencies", - ) - parser.add_argument( - "--build", - "-b", - action="store_true", - help="show only build dependencies", - ) - parser.add_argument( - "--type", - "-t", - help="the type of optional dependency to show (default: show packages from all types)", - ) - parser.add_argument( - "--list", - "-l", - action="store_true", - help="list all of the optional dependency types. " - "Note that this overrides all other options", - ) - args = parser.parse_args() - - if not (Path(args.dir) / "pyproject.toml").exists(): - raise FileNotFoundError(f"No `pyproject.toml` file found in {args.dir}") - - with open(Path(args.dir) / "pyproject.toml", "rb") as f: - content = load(f) - - top_key, opt_key = "project", "optional-dependencies" - - if args.list: - for key in content[top_key].get(opt_key): - print(key) - return - - if args.build: - print("# build time dependencies") - for package in content["build-system"]["requires"]: - print(package) - - if args.runtime: - if content[top_key].get("dependencies"): - print("# runtime dependencies") - for package in content[top_key].get("dependencies"): - print(package) - - if args.type: - if args.type not in content[top_key].get(opt_key): - raise ValueError( - f"The value {args.type} is not a type of optional dependency or `build-system`", - ) - print(f"# dependencies for {args.type}") - for item in content[top_key][opt_key][args.type]: - print(item) - - elif content[top_key].get(opt_key): - for key in content[top_key].get(opt_key): - print(f"# dependencies for {key}") - for item in content[top_key][opt_key][key]: - print(item) - - -if __name__ == "__main__": - main() From e36e0e75c063a301ee593acb4099e2aa58632a97 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 11:35:58 +0100 Subject: [PATCH 096/209] Replace `pkg_resources` with `importlib` No more dependency on setuptools! --- nmodl/_binwrapper.py | 40 ++++++++++++++++++++++------------------ pyproject.toml | 3 ++- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/nmodl/_binwrapper.py b/nmodl/_binwrapper.py index e1971a2b84..6108b8d84c 100755 --- a/nmodl/_binwrapper.py +++ b/nmodl/_binwrapper.py @@ -3,41 +3,45 @@ A generic wrapper to access nmodl binaries from a python installation Please create a softlink with the binary name to be called. """ - import os -import sys import stat -from pkg_resources import working_set +import sys + +if sys.version_info >= (3, 9): + from importlib.metadata import metadata + from importlib.resources import files +else: + from importlib_metadata import metadata + from importlib_resources import files + from find_libpython import find_libpython def main(): """Sets the environment to run the real executable (returned)""" - package_name = "nmodl" - - if package_name not in working_set.by_key: - print ("INFO : Using nmodl-nightly Package (Developer Version)") - package_name = 'nmodl-nightly' - - assert ( - package_name in working_set.by_key - ), "NMODL package not found! Verify PYTHONPATH" + try: + metadata("nmodl-nightly") + print("INFO : Using nmodl-nightly Package (Developer Version)") + except Exception: + pass - NMODL_PREFIX = os.path.join(working_set.by_key[package_name].location, "nmodl") - NMODL_HOME = os.path.join(NMODL_PREFIX, ".data") - NMODL_BIN = os.path.join(NMODL_HOME, "bin") + NMODL_PREFIX = files("nmodl") + NMODL_HOME = NMODL_PREFIX / ".data" + NMODL_BIN = NMODL_HOME / "bin" # add libpython*.so path to environment os.environ["NMODL_PYLIB"] = find_libpython() # add nmodl home to environment (i.e. necessary for nrnunits.lib) - os.environ["NMODLHOME"] = NMODL_HOME + os.environ["NMODLHOME"] = str(NMODL_HOME) # set PYTHONPATH for embedded python to properly find the nmodl module - os.environ["PYTHONPATH"] = working_set.by_key[package_name].location + ':' + os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = ( + str(NMODL_PREFIX) + ":" + os.environ.get("PYTHONPATH", "") + ) - exe = os.path.join(NMODL_BIN, os.path.basename(sys.argv[0])) + exe = NMODL_BIN / os.path.basename(sys.argv[0]) st = os.stat(exe) os.chmod(exe, st.st_mode | stat.S_IEXEC) os.execv(exe, sys.argv) diff --git a/pyproject.toml b/pyproject.toml index e4513f1e4e..98a28cf8d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,8 @@ dynamic = ["version"] dependencies = [ "find_libpython", "sympy>=1.3", - "setuptools", + "importlib-metadata;python_version<'3.9'", + "importlib-resources;python_version<'3.9'", ] scripts = {nmodl = "nmodl._binwrapper:main"} requires-python = ">=3.8" From 4873e76d72162fe9517c88838c4f27739cfe41bd Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 11:35:58 +0100 Subject: [PATCH 097/209] Replace `pkg_resources` with `importlib` No more dependency on setuptools! --- nmodl/_binwrapper.py | 40 ++++++++++++++++++++++------------------ pyproject.toml | 3 ++- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/nmodl/_binwrapper.py b/nmodl/_binwrapper.py index e1971a2b84..6108b8d84c 100755 --- a/nmodl/_binwrapper.py +++ b/nmodl/_binwrapper.py @@ -3,41 +3,45 @@ A generic wrapper to access nmodl binaries from a python installation Please create a softlink with the binary name to be called. """ - import os -import sys import stat -from pkg_resources import working_set +import sys + +if sys.version_info >= (3, 9): + from importlib.metadata import metadata + from importlib.resources import files +else: + from importlib_metadata import metadata + from importlib_resources import files + from find_libpython import find_libpython def main(): """Sets the environment to run the real executable (returned)""" - package_name = "nmodl" - - if package_name not in working_set.by_key: - print ("INFO : Using nmodl-nightly Package (Developer Version)") - package_name = 'nmodl-nightly' - - assert ( - package_name in working_set.by_key - ), "NMODL package not found! Verify PYTHONPATH" + try: + metadata("nmodl-nightly") + print("INFO : Using nmodl-nightly Package (Developer Version)") + except Exception: + pass - NMODL_PREFIX = os.path.join(working_set.by_key[package_name].location, "nmodl") - NMODL_HOME = os.path.join(NMODL_PREFIX, ".data") - NMODL_BIN = os.path.join(NMODL_HOME, "bin") + NMODL_PREFIX = files("nmodl") + NMODL_HOME = NMODL_PREFIX / ".data" + NMODL_BIN = NMODL_HOME / "bin" # add libpython*.so path to environment os.environ["NMODL_PYLIB"] = find_libpython() # add nmodl home to environment (i.e. necessary for nrnunits.lib) - os.environ["NMODLHOME"] = NMODL_HOME + os.environ["NMODLHOME"] = str(NMODL_HOME) # set PYTHONPATH for embedded python to properly find the nmodl module - os.environ["PYTHONPATH"] = working_set.by_key[package_name].location + ':' + os.environ.get("PYTHONPATH", "") + os.environ["PYTHONPATH"] = ( + str(NMODL_PREFIX) + ":" + os.environ.get("PYTHONPATH", "") + ) - exe = os.path.join(NMODL_BIN, os.path.basename(sys.argv[0])) + exe = NMODL_BIN / os.path.basename(sys.argv[0]) st = os.stat(exe) os.chmod(exe, st.st_mode | stat.S_IEXEC) os.execv(exe, sys.argv) diff --git a/pyproject.toml b/pyproject.toml index a13024ea12..8e067eef5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,8 @@ dynamic = ["version"] dependencies = [ "find_libpython", "sympy>=1.3", - "setuptools", + "importlib-metadata;python_version<'3.9'", + "importlib-resources;python_version<'3.9'", ] scripts = {nmodl = "nmodl._binwrapper:main"} requires-python = ">=3.8" From 4128b68e8c41b822cde66869d4b0dffa3c5b0f76 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 11:53:02 +0100 Subject: [PATCH 098/209] Forgot one pip-compile --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4ad676b80c..f3b21f5632 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -148,7 +148,7 @@ stages: submodules: True - script: | brew install flex bison cmake python@3 - python3 -m pip install --upgrade pip setuptools + python3 -m pip install --upgrade pip setuptools 'pip-compile>=7.4.0' python3 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) displayName: 'Install Dependencies' - script: | From c3ceca395d10ac24e1d189f2672f19334bda8d32 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 12:00:08 +0100 Subject: [PATCH 099/209] Fix typo --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f3b21f5632..a34641e6c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -148,7 +148,7 @@ stages: submodules: True - script: | brew install flex bison cmake python@3 - python3 -m pip install --upgrade pip setuptools 'pip-compile>=7.4.0' + python3 -m pip install --upgrade pip setuptools 'pip-tools>=7.4.0' python3 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) displayName: 'Install Dependencies' - script: | From 36750f05a436278c25301382febc7700523d7d6a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 15:44:53 +0100 Subject: [PATCH 100/209] Replace `pkg_resources` with `importlib` --- nmodl/ast.py | 46 +++++++++++++++++++++++++++------------------- nmodl/dsl.py | 30 ++++++++++++++++++------------ 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/nmodl/ast.py b/nmodl/ast.py index f16bbf2d92..e21420e11b 100644 --- a/nmodl/ast.py +++ b/nmodl/ast.py @@ -1,5 +1,23 @@ +""" +Module for vizualization of NMODL abstract syntax trees (ASTs). +""" +import getpass +import json +import os +import sys +import tempfile +import webbrowser +from shutil import copytree + + +if sys.version_info >= (3, 9): + from importlib.resources import files +else: + from importlib_resources import files + +from ._nmodl import to_json from ._nmodl.ast import * # noqa -from pkg_resources import * + def view(nmodl_ast): """Visualize given NMODL AST in web browser @@ -14,31 +32,21 @@ def view(nmodl_ast): Returns: None """ - from ._nmodl import to_json - from distutils.dir_util import copy_tree - import getpass - import json - import os - import tempfile - import webbrowser - - resource = "ext/viz" - if resource_exists(__name__, resource) and resource_isdir(__name__, resource): - installed_viz_tool = resource_filename(__name__, resource) - else: + + path = files("nmodl") / "ext/viz" + if not path.is_dir(): raise FileNotFoundError("Could not find sample mod files") work_dir = os.path.join(tempfile.gettempdir(), getpass.getuser(), "nmodl") # first copy necessary files to temp work directory - copy_tree(installed_viz_tool, work_dir) + copytree(path, work_dir) # prepare json data - with open(os.path.join(work_dir, 'ast.js'), 'w') as outfile: - json_data = json.loads(to_json(nmodl_ast, True, True, True)) - outfile.write('var astRoot = %s;' % json.dumps(json_data)) + json_data = json.loads(to_json(nmodl_ast, True, True, True)) + with open(os.path.join(work_dir, "ast.js"), "w", encoding="utf-8") as outfile: + outfile.write(f"var astRoot = {json.dumps(json_data)};") # open browser with ast - url = 'file://' + os.path.join(work_dir, "index.html") + url = "file://" + os.path.join(work_dir, "index.html") webbrowser.open(url, new=1, autoraise=True) - diff --git a/nmodl/dsl.py b/nmodl/dsl.py index 746150fa85..f556c70cb9 100644 --- a/nmodl/dsl.py +++ b/nmodl/dsl.py @@ -1,10 +1,15 @@ -import os.path as osp -from pkg_resources import * +import sys + +if sys.version_info >= (3, 9): + from importlib.resources import files +else: + from importlib_resources import files from ._nmodl import * RESOURCE_DIR = "ext/example" + def list_examples(): """Returns a list of examples available @@ -13,10 +18,11 @@ def list_examples(): Returns: List of available examples """ - if resource_exists(__name__, RESOURCE_DIR) and resource_isdir(__name__, RESOURCE_DIR): - return resource_listdir(__name__, RESOURCE_DIR) - else: - raise FileNotFoundError("Could not find sample directory") + path = files("nmodl") / RESOURCE_DIR + if path.exists() and path.is_dir(): + return list(path.glob("*.mod")) + + raise FileNotFoundError("Could not find sample directory") def load_example(example): @@ -29,10 +35,10 @@ def load_example(example): Args: example: Filename of an example as provided by `list_examples()` Returns: - List of available examples + An path to the example as a string """ - resource = osp.join(RESOURCE_DIR, example) - if resource_exists(__name__, resource): - return resource_string(__name__, resource) - else: - raise FileNotFoundError("Could not find sample mod files") + path = files("nmodl") / RESOURCE_DIR / example + if path.exists(): + return str(path) + + raise FileNotFoundError(f"Could not find sample mod file {example}") From 652a98562a7ed738a26a7cc00b6e4146a75cc3bc Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 16:13:48 +0100 Subject: [PATCH 101/209] Fix CI --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2973ab173d..9b4c69d118 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -148,7 +148,7 @@ stages: submodules: True - script: | brew install flex bison cmake python@3 - python3 -m pip install --upgrade pip setuptools + python3 -m pip install --upgrade pip setuptools 'pip-tools>=7.4.0' python3 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) displayName: 'Install Dependencies' - script: | From 53a1456673de0920be84d664d08fe42585773f16 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 15:44:53 +0100 Subject: [PATCH 102/209] Replace `pkg_resources` with `importlib` --- nmodl/ast.py | 46 +++++++++++++++++++++++++++------------------- nmodl/dsl.py | 30 ++++++++++++++++++------------ 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/nmodl/ast.py b/nmodl/ast.py index f16bbf2d92..e21420e11b 100644 --- a/nmodl/ast.py +++ b/nmodl/ast.py @@ -1,5 +1,23 @@ +""" +Module for vizualization of NMODL abstract syntax trees (ASTs). +""" +import getpass +import json +import os +import sys +import tempfile +import webbrowser +from shutil import copytree + + +if sys.version_info >= (3, 9): + from importlib.resources import files +else: + from importlib_resources import files + +from ._nmodl import to_json from ._nmodl.ast import * # noqa -from pkg_resources import * + def view(nmodl_ast): """Visualize given NMODL AST in web browser @@ -14,31 +32,21 @@ def view(nmodl_ast): Returns: None """ - from ._nmodl import to_json - from distutils.dir_util import copy_tree - import getpass - import json - import os - import tempfile - import webbrowser - - resource = "ext/viz" - if resource_exists(__name__, resource) and resource_isdir(__name__, resource): - installed_viz_tool = resource_filename(__name__, resource) - else: + + path = files("nmodl") / "ext/viz" + if not path.is_dir(): raise FileNotFoundError("Could not find sample mod files") work_dir = os.path.join(tempfile.gettempdir(), getpass.getuser(), "nmodl") # first copy necessary files to temp work directory - copy_tree(installed_viz_tool, work_dir) + copytree(path, work_dir) # prepare json data - with open(os.path.join(work_dir, 'ast.js'), 'w') as outfile: - json_data = json.loads(to_json(nmodl_ast, True, True, True)) - outfile.write('var astRoot = %s;' % json.dumps(json_data)) + json_data = json.loads(to_json(nmodl_ast, True, True, True)) + with open(os.path.join(work_dir, "ast.js"), "w", encoding="utf-8") as outfile: + outfile.write(f"var astRoot = {json.dumps(json_data)};") # open browser with ast - url = 'file://' + os.path.join(work_dir, "index.html") + url = "file://" + os.path.join(work_dir, "index.html") webbrowser.open(url, new=1, autoraise=True) - diff --git a/nmodl/dsl.py b/nmodl/dsl.py index 746150fa85..f556c70cb9 100644 --- a/nmodl/dsl.py +++ b/nmodl/dsl.py @@ -1,10 +1,15 @@ -import os.path as osp -from pkg_resources import * +import sys + +if sys.version_info >= (3, 9): + from importlib.resources import files +else: + from importlib_resources import files from ._nmodl import * RESOURCE_DIR = "ext/example" + def list_examples(): """Returns a list of examples available @@ -13,10 +18,11 @@ def list_examples(): Returns: List of available examples """ - if resource_exists(__name__, RESOURCE_DIR) and resource_isdir(__name__, RESOURCE_DIR): - return resource_listdir(__name__, RESOURCE_DIR) - else: - raise FileNotFoundError("Could not find sample directory") + path = files("nmodl") / RESOURCE_DIR + if path.exists() and path.is_dir(): + return list(path.glob("*.mod")) + + raise FileNotFoundError("Could not find sample directory") def load_example(example): @@ -29,10 +35,10 @@ def load_example(example): Args: example: Filename of an example as provided by `list_examples()` Returns: - List of available examples + An path to the example as a string """ - resource = osp.join(RESOURCE_DIR, example) - if resource_exists(__name__, resource): - return resource_string(__name__, resource) - else: - raise FileNotFoundError("Could not find sample mod files") + path = files("nmodl") / RESOURCE_DIR / example + if path.exists(): + return str(path) + + raise FileNotFoundError(f"Could not find sample mod file {example}") From 43cccf24abba8869ef65f7552aa978b77141b5a7 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 17:09:36 +0100 Subject: [PATCH 103/209] Fix bug in setting path --- nmodl/_binwrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmodl/_binwrapper.py b/nmodl/_binwrapper.py index 6108b8d84c..35b502035b 100755 --- a/nmodl/_binwrapper.py +++ b/nmodl/_binwrapper.py @@ -38,7 +38,7 @@ def main(): # set PYTHONPATH for embedded python to properly find the nmodl module os.environ["PYTHONPATH"] = ( - str(NMODL_PREFIX) + ":" + os.environ.get("PYTHONPATH", "") + str(NMODL_PREFIX.parent) + ":" + os.environ.get("PYTHONPATH", "") ) exe = NMODL_BIN / os.path.basename(sys.argv[0]) From cf5b9ff302ca4a0ae97ae07afa06c16ba6b0d773 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 17:09:36 +0100 Subject: [PATCH 104/209] Fix bug in setting path --- nmodl/_binwrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nmodl/_binwrapper.py b/nmodl/_binwrapper.py index 6108b8d84c..35b502035b 100755 --- a/nmodl/_binwrapper.py +++ b/nmodl/_binwrapper.py @@ -38,7 +38,7 @@ def main(): # set PYTHONPATH for embedded python to properly find the nmodl module os.environ["PYTHONPATH"] = ( - str(NMODL_PREFIX) + ":" + os.environ.get("PYTHONPATH", "") + str(NMODL_PREFIX.parent) + ":" + os.environ.get("PYTHONPATH", "") ) exe = NMODL_BIN / os.path.basename(sys.argv[0]) From 8cc57239214e84d23853987a9f973836dba0b92b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 17:14:38 +0100 Subject: [PATCH 105/209] Be more specific about error raised --- nmodl/_binwrapper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nmodl/_binwrapper.py b/nmodl/_binwrapper.py index 35b502035b..036334ceb6 100755 --- a/nmodl/_binwrapper.py +++ b/nmodl/_binwrapper.py @@ -8,10 +8,10 @@ import sys if sys.version_info >= (3, 9): - from importlib.metadata import metadata + from importlib.metadata import metadata, PackageNotFoundError from importlib.resources import files else: - from importlib_metadata import metadata + from importlib_metadata import metadata, PackageNotFoundError from importlib_resources import files from find_libpython import find_libpython @@ -23,7 +23,7 @@ def main(): try: metadata("nmodl-nightly") print("INFO : Using nmodl-nightly Package (Developer Version)") - except Exception: + except PackageNotFoundError: pass NMODL_PREFIX = files("nmodl") From 9fabd812f3c9ca49e1f894b84d2efc0cf937809e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 17:14:38 +0100 Subject: [PATCH 106/209] Be more specific about error raised --- nmodl/_binwrapper.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nmodl/_binwrapper.py b/nmodl/_binwrapper.py index 35b502035b..036334ceb6 100755 --- a/nmodl/_binwrapper.py +++ b/nmodl/_binwrapper.py @@ -8,10 +8,10 @@ import sys if sys.version_info >= (3, 9): - from importlib.metadata import metadata + from importlib.metadata import metadata, PackageNotFoundError from importlib.resources import files else: - from importlib_metadata import metadata + from importlib_metadata import metadata, PackageNotFoundError from importlib_resources import files from find_libpython import find_libpython @@ -23,7 +23,7 @@ def main(): try: metadata("nmodl-nightly") print("INFO : Using nmodl-nightly Package (Developer Version)") - except Exception: + except PackageNotFoundError: pass NMODL_PREFIX = files("nmodl") From 932d06294363cd6a35a9a0caef64a28eb804228d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 19 Feb 2024 17:25:39 +0100 Subject: [PATCH 107/209] Add tests for CLI in CI For now they only run on MacOS because the manylinux images don't have libpython installed, and hence `find_libpython()` always returns None --- packaging/test_cli.bash | 8 ++++++++ pyproject.toml | 5 +++++ 2 files changed, 13 insertions(+) create mode 100644 packaging/test_cli.bash diff --git a/packaging/test_cli.bash b/packaging/test_cli.bash new file mode 100644 index 0000000000..0cbf02354f --- /dev/null +++ b/packaging/test_cli.bash @@ -0,0 +1,8 @@ +#!/bin/bash +# A simple set of tests checking if the NMODL CLI (called from Python) is +# working correctly +set -xe + +find "$1/test/" "$1/nmodl/ext/" \ + -name "*.mod" \ + -exec nmodl '{}' sympy --analytic \; diff --git a/pyproject.toml b/pyproject.toml index 98a28cf8d0..b38da75a4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,11 @@ test-extras = ["test"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } config-settings = {build-dir = "_build"} +test-command = [ + "pytest {package}/test/unit/pybind", + "bash {package}/packaging/test_wheel.bash {package}", + "bash {package}/packaging/test_cli.bash {package}", +] [tool.cibuildwheel.linux] before-all = "bash packaging/install_dependencies.sh" From 94a2a72e78c4f0f2d80eac281a203d5d4320fb92 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 20 Feb 2024 09:38:20 +0100 Subject: [PATCH 108/209] Fix for NMODL sympy visitor When using the sympy visitor, the NMODL Python module by default throws an error that `NMODL_PYLIB` is not set. The solution is to set all of these env variables as soon as we load the module. --- nmodl/__init__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/nmodl/__init__.py b/nmodl/__init__.py index 5d5498b60b..653d8b444c 100644 --- a/nmodl/__init__.py +++ b/nmodl/__init__.py @@ -1,9 +1,26 @@ +import os +import sys + +if sys.version_info >= (3, 9): + from importlib.resources import files + from importlib.metadata import version, PackageNotFoundError +else: + from importlib_resources import files + from importlib_metadata import version, PackageNotFoundError + +from find_libpython import find_libpython + +# add libpython*.so path to environment +os.environ["NMODL_PYLIB"] = find_libpython() + +# add nmodl home to environment (i.e. necessary for nrnunits.lib) +os.environ["NMODLHOME"] = str(files("nmodl") / ".data") + + try: # Try importing but catch exception in case bindings are not available from ._nmodl import NmodlDriver, to_json, to_nmodl # noqa - from importlib.metadata import version, PackageNotFoundError - try: __version__ = version("nmodl") except PackageNotFoundError: From 63382d6b3fd7bd62e3c5db06df7bf015e9cba5e5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 20 Feb 2024 10:36:35 +0100 Subject: [PATCH 109/209] Refactor generation of docs --- .github/workflows/nmodl-doc.yml | 26 +++++++------------------- docs/conf.py | 23 ++++------------------- docs/generate_docs.sh | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 38 deletions(-) create mode 100755 docs/generate_docs.sh diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index 5ec6d8b39c..8cb553f2d1 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -67,29 +67,17 @@ jobs: if: failure() && contains(github.event.head_commit.message, 'live-debug-docs') uses: mxschmitt/action-tmate@v3 - - name: Restore compiler cache - uses: actions/cache@v4 - with: - path: | - ${{runner.workspace}}/ccache - key: docs-${{github.ref}}-${{github.sha}} - restore-keys: | - docs-${{github.ref}}- - docs- - - name: Documentation id: documentation working-directory: ${{runner.workspace}}/nmodl run: | echo "------- Build Documentation -------"; - ccache -z - ccache -s - python3 setup.py build_ext --inplace docs -j 2 -G Ninja \ - -- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache; - ccache -s - cd _skbuild/linux-x86_64-3.8/setuptools/sphinx; - rm -rf doctest doctrees && touch .nojekyll; - echo "" > index.html; + # build wheel and install it + pip wheel . --wheel-dir wheelhouse/ -C build-dir=_build --no-deps + pip install wheelhouse/*.whl + bash docs/generate_docs.sh + touch public/.nojekyll + echo "" > public/index.html; echo "status=done" >> $GITHUB_OUTPUT env: CCACHE_DIR: ${{runner.workspace}}/ccache @@ -99,5 +87,5 @@ jobs: if: steps.documentation.outputs.status == 'done' && startsWith(github.ref, 'refs/heads/master') with: branch: gh-pages # The branch the action should deploy to. - folder: ${{runner.workspace}}/nmodl/_skbuild/linux-x86_64-3.8/setuptools/sphinx # The folder the action should deploy. + folder: ${{runner.workspace}}/nmodl/public clean: false # Automatically remove deleted files from the deploy branch diff --git a/docs/conf.py b/docs/conf.py index 8bfe193232..cee130352c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,29 +12,14 @@ # # All configuration values have a default; values that are commented out # serve to show the default. - # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # - - - -import os -import subprocess -import sys import textwrap -# The project needs to be built before documentation in the usual build folder -sys.path.insert(0, os.path.abspath('..')) - import nmodl # isort:skip -os.environ['PYTHONPATH'] = ':'.join(sys.path) - -# Run doxygen -subprocess.call('doxygen Doxyfile', shell=True) - # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. @@ -134,11 +119,11 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ["_static"] +# html_static_path = ["_static"] -# A list of paths that contain extra files not directly related to the -# documentation, such as robots.txt or .htaccess. Relative paths are taken -# as relative to the configuration directory. They are copied to the output +# A list of paths that contain extra files not directly related to the +# documentation, such as robots.txt or .htaccess. Relative paths are taken +# as relative to the configuration directory. They are copied to the output # directory. They will overwrite any existing file of the same name. html_extra_path = ["sphinx_doxygen"] diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh new file mode 100755 index 0000000000..41a4783e22 --- /dev/null +++ b/docs/generate_docs.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# script for generating documentation for NMODL +# note that the NMODL Python wheel must be installed +# for the script to work properly + +set -xe + +# the abs dir where this script is located (so we can call it from wherever) +script_dir="$(realpath "$(dirname "$0")")" + +cd "${script_dir}" +doxygen Doxyfile +sphinx-build . "${script_dir}/../public" +cd - From 02191dea4dba79a5d2ec8b4260cfddd4a37a4f19 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 20 Feb 2024 09:38:20 +0100 Subject: [PATCH 110/209] Fix for NMODL sympy visitor When using the sympy visitor, the NMODL Python module by default throws an error that `NMODL_PYLIB` is not set. The solution is to set all of these env variables as soon as we load the module. --- nmodl/__init__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/nmodl/__init__.py b/nmodl/__init__.py index 5d5498b60b..653d8b444c 100644 --- a/nmodl/__init__.py +++ b/nmodl/__init__.py @@ -1,9 +1,26 @@ +import os +import sys + +if sys.version_info >= (3, 9): + from importlib.resources import files + from importlib.metadata import version, PackageNotFoundError +else: + from importlib_resources import files + from importlib_metadata import version, PackageNotFoundError + +from find_libpython import find_libpython + +# add libpython*.so path to environment +os.environ["NMODL_PYLIB"] = find_libpython() + +# add nmodl home to environment (i.e. necessary for nrnunits.lib) +os.environ["NMODLHOME"] = str(files("nmodl") / ".data") + + try: # Try importing but catch exception in case bindings are not available from ._nmodl import NmodlDriver, to_json, to_nmodl # noqa - from importlib.metadata import version, PackageNotFoundError - try: __version__ = version("nmodl") except PackageNotFoundError: From 8073f4b4cbcbf52358724847c820ea980a2e358f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 21 Feb 2024 16:11:04 +0100 Subject: [PATCH 111/209] WIP on using azure for CI --- azure-pipelines.yml | 60 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a34641e6c7..91cd08dff0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -194,3 +194,63 @@ stages: env: SHELL: 'bash' displayName: 'Build Neuron and Run Integration Tests' + - job: 'manylinux_wheels' + timeoutInMinutes: 45 + pool: + vmImage: 'ubuntu-20.04' + steps: + - task: UsePythonVersion@0 + - checkout: self + submodules: True + condition: succeeded() + - script: | + if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then + export TAG="-nightly" + else + export TAG="" + fi + python3 -m pip install --upgrade pip + python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w + # change the name accordingly + python3 packaging/change_name.py pyproject.toml "nmodl$TAG" + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_NMODL="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse + condition: succeeded() + displayName: 'Building ManyLinux Wheels' + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' + condition: succeeded() + displayName: 'Publish wheel as build artifact' + - template: ci/upload-wheels.yml + - job: 'macos_wheels' + timeoutInMinutes: 45 + pool: + vmImage: 'macOS-11' + steps: + - checkout: self + submodules: True + condition: succeeded() + - script: | + brew install flex bison cmake ninja + condition: succeeded() + displayName: 'Install Dependencies' + - task: UsePythonVersion@0 + - script: | + if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then + export NMODL_NIGHTLY_TAG="-nightly" + else + export NMODL_NIGHTLY_TAG="" + fi + python3 -m pip install --upgrade pip + python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w + # change the name accordingly + python3 packaging/change_name.py pyproject.toml "nmodl$TAG" + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_NMODL="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse + condition: succeeded() + displayName: 'Build macos Wheel' + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' + condition: succeeded() + displayName: 'Publish wheel as build artifact' + - template: ci/upload-wheels.yml From 6a6433bda7299f72f1a811b0e90577f3fe53bcae Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 21 Feb 2024 17:28:37 +0100 Subject: [PATCH 112/209] Do not redefine `LINK_AGAINST_PYTHON` --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 55e3ce3e15..26ecac480f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,6 @@ set(NMODL_EXTRA_CXX_FLAGS "" CACHE STRING "Add extra compile flags for NMODL sources") separate_arguments(NMODL_EXTRA_CXX_FLAGS) -option(LINK_AGAINST_PYTHON "Should the Python library be linked or not" ON) option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) From b8f793bedbf2c89cc126ba63eae1ee30cf3e65ab Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 21 Feb 2024 18:09:25 +0100 Subject: [PATCH 113/209] Define skip builds in pyproject.toml This is so we can use this on multiple CIs without too much config --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b38da75a4d..4e35950392 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ NMODL_BUILD_WHEEL = "ON" [tool.setuptools_scm] [tool.cibuildwheel] +skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64"] test-command = [ "pytest {package}/test/unit/pybind", "bash {package}/packaging/test_wheel.bash {package}", From 6f83d3b56bdb7c28b99aaecef671f9af753d6da3 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 21 Feb 2024 18:10:40 +0100 Subject: [PATCH 114/209] Remove `env` from wheel CI jobs Everything is defined in pyproject.toml anyway --- .circleci/config.yml | 2 -- .github/workflows/nmodl-wheel.yml | 4 ---- 2 files changed, 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 390325239b..0d0736bba6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,8 +12,6 @@ jobs: - checkout - run: name: Build the Linux aarch64 wheels. - environment: - CIBW_SKIP: "pp* *-musllinux_aarch64" command: | python3 -m pip install --user cibuildwheel==2.16.5 python3 -m cibuildwheel --output-dir wheelhouse diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 4657fbfd75..0d3716ddee 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -35,10 +35,6 @@ jobs: - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse - # to supply options, put them in 'env', like: - env: - # we skip PyPy, 32-bit builds, and musllinux builds - CIBW_SKIP: "pp* *-win32 *-manylinux_i686 *-musllinux_i686 *-musllinux_x86_64" - uses: actions/upload-artifact@v4 with: From e54ee01d649d11cec838170ff71675b2b18f0491 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 10:55:54 +0100 Subject: [PATCH 115/209] WIP on azure pipeline Linux testing --- azure-pipelines.yml | 31 +++++++++++++++++++++++++++++++ packaging/test_cli.bash | 8 -------- packaging/test_wheel.bash | 2 ++ pyproject.toml | 11 ++++------- 4 files changed, 37 insertions(+), 15 deletions(-) delete mode 100644 packaging/test_cli.bash diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 91cd08dff0..a79f9d8eae 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -253,4 +253,35 @@ stages: pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' condition: succeeded() displayName: 'Publish wheel as build artifact' + - job: 'test Linux wheels' + strategy: + matrix: + ${{ if eq(variables.buildWheel, True) }}: + Python38: + python.version: '3.8' + Python39: + python.version: '3.9' + Python310: + python.version: '3.10' + Python311: + python.version: '3.11' + Python312: + python.version: '3.12' + ${{ if eq(variables.buildWheel, False) }}: + Python311: + python.version: '3.11' + - script: | + sudo apt-add-repository -y ppa:deadsnakes/ppa + sudo apt-get update + sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv + - script: | + package_dir="${PWD}" + tmp_dir="$(mktemp -d)" + cd "${tmp_dir}" + python$(python.version) -m venv "${tmp_dir}/env" + find ${package_dir} -name "*cp$(echo $(python.version) | tr -d '.')*.whl" -exec sh -c ". ${tmp_dir}/env/bin/activate && python -m pip install '{}[test]'" \; + . ${tmp_dir}/env/bin/activate && bash ${package_dir}/packaging/test_wheel.bash ${package_dir} + cd - + condition: succeeded() + displayName: 'Test Linux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml diff --git a/packaging/test_cli.bash b/packaging/test_cli.bash deleted file mode 100644 index 0cbf02354f..0000000000 --- a/packaging/test_cli.bash +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# A simple set of tests checking if the NMODL CLI (called from Python) is -# working correctly -set -xe - -find "$1/test/" "$1/nmodl/ext/" \ - -name "*.mod" \ - -exec nmodl '{}' sympy --analytic \; diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 111d73a34e..b97fc6a165 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -2,6 +2,8 @@ # A simple set of tests checking if a wheel is working correctly set -xe +python -m pytest "$1/test/unit/pybind" find "$1/test/" "$1/nmodl/ext/" \ -name "*.mod" \ + -exec nmodl '{}' sympy --analytic \; \ -exec python -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('{}')" \; diff --git a/pyproject.toml b/pyproject.toml index 4e35950392..2e76b1ec70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,24 +56,21 @@ NMODL_BUILD_WHEEL = "ON" [tool.cibuildwheel] skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64"] +test-extras = ["test"] test-command = [ - "pytest {package}/test/unit/pybind", "bash {package}/packaging/test_wheel.bash {package}", ] -test-extras = ["test"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } config-settings = {build-dir = "_build"} -test-command = [ - "pytest {package}/test/unit/pybind", - "bash {package}/packaging/test_wheel.bash {package}", - "bash {package}/packaging/test_cli.bash {package}", -] [tool.cibuildwheel.linux] before-all = "bash packaging/install_dependencies.sh" environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } +# the Linux wheel is not tested in cibuildwheel due to manylinux images not having +# libpython*.so, so this is tested manually in the CI +test-command = "true" [tool.pytest.ini_options] testpaths = "test/unit/pybind" From dc1d5cd0968c5b2fc5259357cc2472fb352c5b75 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:08:15 +0100 Subject: [PATCH 116/209] Maybe fix azure? --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a79f9d8eae..f21ea964fe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -274,7 +274,6 @@ stages: sudo apt-add-repository -y ppa:deadsnakes/ppa sudo apt-get update sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv - - script: | package_dir="${PWD}" tmp_dir="$(mktemp -d)" cd "${tmp_dir}" From 7787169e307dae984258d784798a32b355666ef6 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:12:29 +0100 Subject: [PATCH 117/209] Fix ordering in azure --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f21ea964fe..7672fdfc01 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -253,6 +253,7 @@ stages: pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' condition: succeeded() displayName: 'Publish wheel as build artifact' + - template: ci/upload-wheels.yml - job: 'test Linux wheels' strategy: matrix: @@ -283,4 +284,3 @@ stages: cd - condition: succeeded() displayName: 'Test Linux Wheel with Python $(python.version)' - - template: ci/upload-wheels.yml From bc7aa9ba7b837e2b24c1e5a1311cd0844032c491 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:17:22 +0100 Subject: [PATCH 118/209] Maybe now --- azure-pipelines.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7672fdfc01..50f9fdcdcd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -271,16 +271,17 @@ stages: ${{ if eq(variables.buildWheel, False) }}: Python311: python.version: '3.11' - - script: | - sudo apt-add-repository -y ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv - package_dir="${PWD}" - tmp_dir="$(mktemp -d)" - cd "${tmp_dir}" - python$(python.version) -m venv "${tmp_dir}/env" - find ${package_dir} -name "*cp$(echo $(python.version) | tr -d '.')*.whl" -exec sh -c ". ${tmp_dir}/env/bin/activate && python -m pip install '{}[test]'" \; - . ${tmp_dir}/env/bin/activate && bash ${package_dir}/packaging/test_wheel.bash ${package_dir} - cd - + steps: + - script: | + sudo apt-add-repository -y ppa:deadsnakes/ppa + sudo apt-get update + sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv + package_dir="${PWD}" + tmp_dir="$(mktemp -d)" + cd "${tmp_dir}" + python$(python.version) -m venv "${tmp_dir}/env" + find ${package_dir} -name "*cp$(echo $(python.version) | tr -d '.')*.whl" -exec sh -c ". ${tmp_dir}/env/bin/activate && python -m pip install '{}[test]'" \; + . ${tmp_dir}/env/bin/activate && bash ${package_dir}/packaging/test_wheel.bash ${package_dir} + cd - + displayName: 'Test Linux Wheel with Python $(python.version)' condition: succeeded() - displayName: 'Test Linux Wheel with Python $(python.version)' From dc60c567a51b15de8908f7aa203e05be7277ad7f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:18:27 +0100 Subject: [PATCH 119/209] Okay for real this time --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 50f9fdcdcd..9faf2f0fcd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -284,4 +284,4 @@ stages: . ${tmp_dir}/env/bin/activate && bash ${package_dir}/packaging/test_wheel.bash ${package_dir} cd - displayName: 'Test Linux Wheel with Python $(python.version)' - condition: succeeded() + condition: succeeded() From ab8f3649a2c875be8d339551c0222034f37a4380 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:20:14 +0100 Subject: [PATCH 120/209] Change name to please CI overlord --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9faf2f0fcd..981dfa4432 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -254,7 +254,7 @@ stages: condition: succeeded() displayName: 'Publish wheel as build artifact' - template: ci/upload-wheels.yml - - job: 'test Linux wheels' + - job: 'test_linux_wheels' strategy: matrix: ${{ if eq(variables.buildWheel, True) }}: From 37aff68ca1993af29ec76a5850d1d09b876f59c0 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:25:50 +0100 Subject: [PATCH 121/209] TEMP only build cp311 so we can debug azure faster --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 2e76b1ec70..3e4f647a7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ test-extras = ["test"] test-command = [ "bash {package}/packaging/test_wheel.bash {package}", ] +build = ["cp311*"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } From f3f49ea2c1b77f39f0d069df2743216ec62c867d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:29:05 +0100 Subject: [PATCH 122/209] Forgot the base image, thanks azure --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 981dfa4432..18ccb7a6e4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -255,6 +255,9 @@ stages: displayName: 'Publish wheel as build artifact' - template: ci/upload-wheels.yml - job: 'test_linux_wheels' + timeoutInMinutes: 45 + pool: + vmImage: 'ubuntu-20.04' strategy: matrix: ${{ if eq(variables.buildWheel, True) }}: From c759aa310485d4384fcbcf2d7d62223f350ff1d7 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:40:51 +0100 Subject: [PATCH 123/209] Depend on manylinux wheel first --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 18ccb7a6e4..c844e691e6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -255,6 +255,7 @@ stages: displayName: 'Publish wheel as build artifact' - template: ci/upload-wheels.yml - job: 'test_linux_wheels' + dependsOn: 'manylinux_wheels' timeoutInMinutes: 45 pool: vmImage: 'ubuntu-20.04' From b97887891981b8a067b1210890e405f09da2f108 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 11:42:18 +0100 Subject: [PATCH 124/209] SETUPTOOLS_PRETEND_VERSION is better --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c844e691e6..e5887d98b1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -213,7 +213,7 @@ stages: python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w # change the name accordingly python3 packaging/change_name.py pyproject.toml "nmodl$TAG" - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_NMODL="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse + SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() displayName: 'Building ManyLinux Wheels' - task: PublishBuildArtifacts@1 @@ -245,7 +245,7 @@ stages: python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w # change the name accordingly python3 packaging/change_name.py pyproject.toml "nmodl$TAG" - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_NMODL="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse + SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() displayName: 'Build macos Wheel' - task: PublishBuildArtifacts@1 From 81c27bb521f9f617a200a22edf7ecabc443fe7ca Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 13:02:09 +0100 Subject: [PATCH 125/209] DEBUG remove python --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e5887d98b1..9d8171b326 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -199,7 +199,6 @@ stages: pool: vmImage: 'ubuntu-20.04' steps: - - task: UsePythonVersion@0 - checkout: self submodules: True condition: succeeded() From 7eded52267c4ce8d5c414761e3be8a5225588878 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 13:17:06 +0100 Subject: [PATCH 126/209] Revert "DEBUG remove python" This reverts commit 81c27bb521f9f617a200a22edf7ecabc443fe7ca. --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9d8171b326..e5887d98b1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -199,6 +199,7 @@ stages: pool: vmImage: 'ubuntu-20.04' steps: + - task: UsePythonVersion@0 - checkout: self submodules: True condition: succeeded() From 8cae14bce944260da30451a7cec125908c04fe9b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 13:17:13 +0100 Subject: [PATCH 127/209] Do not use python.version for uploading wheels --- ci/upload-wheels.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/upload-wheels.yml b/ci/upload-wheels.yml index 29e52767e2..7091ec285f 100644 --- a/ci/upload-wheels.yml +++ b/ci/upload-wheels.yml @@ -1,7 +1,5 @@ steps: - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python.version)' - task: TwineAuthenticate@1 inputs: pythonUploadServiceConnection: AzureNMODLPypiNightly From 99b3eedda425febd81490c06419d9e04e9d60ad7 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 14:06:43 +0100 Subject: [PATCH 128/209] WIP on venv in azure --- azure-pipelines.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e5887d98b1..271f15d3d5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -194,6 +194,7 @@ stages: env: SHELL: 'bash' displayName: 'Build Neuron and Run Integration Tests' + - job: 'manylinux_wheels' timeoutInMinutes: 45 pool: @@ -222,6 +223,7 @@ stages: condition: succeeded() displayName: 'Publish wheel as build artifact' - template: ci/upload-wheels.yml + - job: 'macos_wheels' timeoutInMinutes: 45 pool: @@ -254,7 +256,8 @@ stages: condition: succeeded() displayName: 'Publish wheel as build artifact' - template: ci/upload-wheels.yml - - job: 'test_linux_wheels' + + - job: 'test_manylinux_wheels' dependsOn: 'manylinux_wheels' timeoutInMinutes: 45 pool: @@ -284,8 +287,12 @@ stages: tmp_dir="$(mktemp -d)" cd "${tmp_dir}" python$(python.version) -m venv "${tmp_dir}/env" - find ${package_dir} -name "*cp$(echo $(python.version) | tr -d '.')*.whl" -exec sh -c ". ${tmp_dir}/env/bin/activate && python -m pip install '{}[test]'" \; - . ${tmp_dir}/env/bin/activate && bash ${package_dir}/packaging/test_wheel.bash ${package_dir} + # activate the venv + . "${tmp_dir}/env/bin/activate" + # find manylinux wheels only + pattern="*cp$(echo $(python.version) | tr -d '.')-manylinux*.whl" + find ${package_dir} -name "${pattern}" -exec python -m pip install '{}[test]' \; + bash ${package_dir}/packaging/test_wheel.bash ${package_dir} cd - - displayName: 'Test Linux Wheel with Python $(python.version)' + displayName: 'Test manylinux wheel with Python $(python.version)' condition: succeeded() From dc2468482a9066f00d4a7138d03d3f7e0de62086 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 14:39:48 +0100 Subject: [PATCH 129/209] Debug azure --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 271f15d3d5..00e9d85a19 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -283,6 +283,7 @@ stages: sudo apt-add-repository -y ppa:deadsnakes/ppa sudo apt-get update sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv + set -xe package_dir="${PWD}" tmp_dir="$(mktemp -d)" cd "${tmp_dir}" From f0c0a3e277b92bad307f66168cef7e278bc61534 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 15:03:46 +0100 Subject: [PATCH 130/209] Download artifacts before testing wheels --- azure-pipelines.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 00e9d85a19..ae1b2af27a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -279,6 +279,8 @@ stages: Python311: python.version: '3.11' steps: + - download: current + patterns: '**/*.whl' - script: | sudo apt-add-repository -y ppa:deadsnakes/ppa sudo apt-get update From 339ee2315a6d94fefe99d0c3b46729112489fcf0 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 22 Feb 2024 16:07:23 +0100 Subject: [PATCH 131/209] Use pip for building wheels Also use SETUPTOOLS_SCM_PRETEND_VERSION in CI to set consistent version --- azure-pipelines.yml | 4 +++- packaging/build_wheels.bash | 16 ++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9b4c69d118..dfb80051c1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -226,10 +226,12 @@ stages: # dynamic `name` in `pyproject.toml` python3 -m pip install --user tomli tomli-w python3 packaging/change_name.py pyproject.toml "NMODL${TAG}" + export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" docker run --rm \ -w /root/nmodl \ -v $PWD:/root/nmodl \ -e NMODL_NIGHTLY_TAG=$TAG \ + -e SETUPTOOLS_SCM_PRETEND_VERSION=SETUPTOOLS_SCM_PRETEND_VERSION \ 'bluebrain/nmodl:wheel' \ packaging/build_wheels.bash linux $(python.version) condition: succeeded() @@ -303,7 +305,7 @@ stages: # dynamic `name` in `pyproject.toml` python3 -m pip install tomli tomli-w python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}" - packaging/build_wheels.bash osx $(python.version) + SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" packaging/build_wheels.bash osx $(python.version) condition: succeeded() displayName: 'Build macos Wheel' - task: PublishBuildArtifacts@1 diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 3dd900f81e..4a2dd1f143 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -12,8 +12,8 @@ set -xe # - python (>=3.8) # - C/C++ compiler -if ! [ -f setup.py ] && ! [ -f pyproject.toml ]; then - echo "Error: setup.py or pyproject.toml not found. Please launch $0 from the nmodl root dir" +if ! [ -f pyproject.toml ]; then + echo "Error: pyproject.toml not found. Please launch $0 from the nmodl root dir" exit 1 fi @@ -47,13 +47,13 @@ build_wheel_linux() { (( $skip )) && return 0 echo " - Installing build requirements" - pip install pip auditwheel setuptools build + pip install pip auditwheel echo " - Building..." - rm -rf dist _skbuild + rm -rf dist _build # Workaround for https://github.com/pypa/manylinux/issues/1309 git config --global --add safe.directory "*" - python -m build --wheel -o dist/ + python -m pip wheel . --wheel-dir dist/ --no-deps echo " - Repairing..." auditwheel repair dist/*.whl @@ -69,13 +69,13 @@ build_wheel_osx() { (( $skip )) && return 0 echo " - Installing build requirements" - pip install --upgrade delocate build + pip install --upgrade delocate echo " - Building..." - rm -rf dist _skbuild + rm -rf dist _build # the custom `build-dir` is a workaround for this issue: # https://gitlab.kitware.com/cmake/cmake/-/issues/20107 - python -m build --wheel -o dist/ -C build-dir=_build + python -m pip wheel . --wheel-dir dist/ -C build-dir=_build --no-deps echo " - Repairing..." delocate-wheel -w wheelhouse -v dist/*.whl # we started clean, there's a single wheel From c6657c88f673eb5320a6897f9b2784d016be9ba4 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Sat, 24 Feb 2024 23:33:43 +0100 Subject: [PATCH 132/209] Correctly pass env variables for manylinux wheel --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ae1b2af27a..4d98c736ec 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -214,7 +214,8 @@ stages: python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w # change the name accordingly python3 packaging/change_name.py pyproject.toml "nmodl$TAG" - SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse + export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" + CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() displayName: 'Building ManyLinux Wheels' - task: PublishBuildArtifacts@1 From b6e3165220dbb7a2d625794a2ac6cf59aaac4a62 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 07:14:37 +0100 Subject: [PATCH 133/209] Fix test script --- packaging/test_wheel.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 25f7eb8f78..8b74e9a1eb 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -2,7 +2,7 @@ # A simple set of tests checking if a wheel is working correctly set -xe -if ! [ -f setup.py ] && ! [ -f pyproject.toml ]; then +if ! [ -f pyproject.toml ]; then echo "Error: Please launch $0 from the root dir" exit 1 fi @@ -22,7 +22,7 @@ python_ver=$("$python_exe" -c "import sys; print('%d%d' % tuple(sys.version_info test_wheel () { # sample mod file for nrnivmodl check - local TEST_DIR="test_dir" + local TEST_DIR="test_dir" mkdir -p $TEST_DIR cp ../nmodl/ext/example/*.mod $TEST_DIR/ cp ../test/integration/mod/cabpump.mod ../test/integration/mod/var_init.inc $TEST_DIR/ From 1c87a1afd207e6f7900289115c58a347321393c2 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 07:22:40 +0100 Subject: [PATCH 134/209] Debug linux version --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4d98c736ec..d93766b8c2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -215,6 +215,7 @@ stages: # change the name accordingly python3 packaging/change_name.py pyproject.toml "nmodl$TAG" export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" + echo "$SETUPTOOLS_SCM_PRETEND_VERSION" CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() displayName: 'Building ManyLinux Wheels' From b3963854077557437e1a385ab68e46033fb83446 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 07:25:00 +0100 Subject: [PATCH 135/209] Fix version --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dfb80051c1..4d8e4128c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -231,7 +231,7 @@ stages: -w /root/nmodl \ -v $PWD:/root/nmodl \ -e NMODL_NIGHTLY_TAG=$TAG \ - -e SETUPTOOLS_SCM_PRETEND_VERSION=SETUPTOOLS_SCM_PRETEND_VERSION \ + -e SETUPTOOLS_SCM_PRETEND_VERSION=$SETUPTOOLS_SCM_PRETEND_VERSION \ 'bluebrain/nmodl:wheel' \ packaging/build_wheels.bash linux $(python.version) condition: succeeded() From 1248829904cd3805e70823cf57c2df2e3c938aba Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 07:48:35 +0100 Subject: [PATCH 136/209] Revert "Do not redefine `LINK_AGAINST_PYTHON`" This reverts commit 6a6433bda7299f72f1a811b0e90577f3fe53bcae. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 26ecac480f..55e3ce3e15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ set(NMODL_EXTRA_CXX_FLAGS "" CACHE STRING "Add extra compile flags for NMODL sources") separate_arguments(NMODL_EXTRA_CXX_FLAGS) +option(LINK_AGAINST_PYTHON "Should the Python library be linked or not" ON) option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) From 8051d5050d1a975d75cf850d72d20530b37a8a6b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 10:26:29 +0100 Subject: [PATCH 137/209] Revert "Do not redefine `LINK_AGAINST_PYTHON`" This reverts commit 6a6433bda7299f72f1a811b0e90577f3fe53bcae. --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 26ecac480f..55e3ce3e15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,6 +25,7 @@ set(NMODL_EXTRA_CXX_FLAGS "" CACHE STRING "Add extra compile flags for NMODL sources") separate_arguments(NMODL_EXTRA_CXX_FLAGS) +option(LINK_AGAINST_PYTHON "Should the Python library be linked or not" ON) option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) From 812a5fd7af9a98c1ffdb94f5a863065df0828ed7 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 10:37:04 +0100 Subject: [PATCH 138/209] Update test-command and test script --- packaging/test_wheel.bash | 64 ++++++++++++++++++++++++++++++++++----- pyproject.toml | 2 +- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index b97fc6a165..8e85b2aed0 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -1,9 +1,59 @@ -#!/bin/bash +#!/usr/bin/env bash # A simple set of tests checking if a wheel is working correctly -set -xe +set -eux -python -m pytest "$1/test/unit/pybind" -find "$1/test/" "$1/nmodl/ext/" \ - -name "*.mod" \ - -exec nmodl '{}' sympy --analytic \; \ - -exec python -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('{}')" \; +this_dir="$(dirname "$(realpath "$0")")" + +if [ "$#" -lt 2 ]; then + echo "Usage: $(basename "$0") python_exe python_wheel [use_virtual_env]" + exit 1 +fi + +# cli parameters +python_exe=$1 +python_wheel=$2 +use_venv=$3 #if $3 is not "false" then use virtual environment + +python_ver=$("$python_exe" -c "import sys; print('%d%d' % tuple(sys.version_info)[:2])") + + +test_wheel () { + # sample mod file for nrnivmodl check + TEST_DIR="$(mktemp -d)" + OUTPUT_DIR="$(mktemp -d)" + cp "${this_dir}/../nmodl/ext/example/"*.mod "$TEST_DIR/" + cp "${this_dir}/../test/integration/mod/cabpump.mod" "${this_dir}/../test/integration/mod/var_init.inc" "$TEST_DIR/" + for mod in "${TEST_DIR}/"*.mod + do + nmodl -o "${OUTPUT_DIR}" "${mod}" sympy --analytic + $python_exe -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('${mod}')" + done + $python_exe -m pytest -vvv "${this_dir}/../test/" +} + +echo "== Testing $python_wheel using $python_exe ($python_ver) ==" + +# creat python virtual environment and use `python` as binary name +# because it will be correct one from venv. +if [[ "$use_venv" != "false" ]]; then + echo " == Creating virtual environment == " + venv_name="$(mktemp -d)/nmodl_test_venv_${python_ver}" + $python_exe -m venv "$venv_name" + . "$venv_name/bin/activate" + python_exe="$(command -v python)" +else + echo " == Using global install == " +fi + +# install nmodl +$python_exe -m pip install -U pip +$python_exe -m pip install "${python_wheel}[test]" +$python_exe -m pip show nmodl || $python_exe -m pip show nmodl-nightly + +# run tests +test_wheel "$(command -v python)" + +# cleanup +if [[ "$use_venv" != "false" ]]; then + deactivate +fi diff --git a/pyproject.toml b/pyproject.toml index 3e4f647a7e..9f64cabe6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ NMODL_BUILD_WHEEL = "ON" skip = ["pp*", "*-win32", "*-manylinux_i686", "*-musllinux_i686", "*-musllinux_x86_64", "*-musllinux_aarch64"] test-extras = ["test"] test-command = [ - "bash {package}/packaging/test_wheel.bash {package}", + "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] build = ["cp311*"] From ab4d2e8cb946fca4102189c44886cb954607b2f2 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 11:46:36 +0100 Subject: [PATCH 139/209] Fix unbound variable --- packaging/test_wheel.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 8e85b2aed0..c060b14d1a 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -12,7 +12,7 @@ fi # cli parameters python_exe=$1 python_wheel=$2 -use_venv=$3 #if $3 is not "false" then use virtual environment +use_venv=${3:-} #if $3 is not "false" then use virtual environment python_ver=$("$python_exe" -c "import sys; print('%d%d' % tuple(sys.version_info)[:2])") From ade464f466f934b6381dbd5bb859057450f7e7f1 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 12:13:10 +0100 Subject: [PATCH 140/209] Fix getting dir on macOS Realpath is not defined until macOS 13 --- packaging/test_wheel.bash | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index c060b14d1a..6230c15bb6 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -2,7 +2,9 @@ # A simple set of tests checking if a wheel is working correctly set -eux -this_dir="$(dirname "$(realpath "$0")")" +# taken from: +# https://stackoverflow.com/q/3572030 +this_dir="$( cd "$(dirname "$0")" ; pwd -P )" if [ "$#" -lt 2 ]; then echo "Usage: $(basename "$0") python_exe python_wheel [use_virtual_env]" From 144b5795fe48b52fb34728143bf288d790ccc8e0 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 26 Feb 2024 18:35:41 +0100 Subject: [PATCH 141/209] Do not install headers when building wheel --- src/CMakeLists.txt | 4 +++- src/solver/CMakeLists.txt | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7a81895ff3..614ed9f673 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,4 +49,6 @@ endif() # Install executable # ============================================================================= install(TARGETS nmodl DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}bin) -install(FILES nmodl.hpp DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) +if(NOT NMODL_BUILD_WHEEL) + install(FILES nmodl.hpp DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) +endif() diff --git a/src/solver/CMakeLists.txt b/src/solver/CMakeLists.txt index 78c44c5d0d..1ffa266dee 100644 --- a/src/solver/CMakeLists.txt +++ b/src/solver/CMakeLists.txt @@ -9,10 +9,14 @@ cpp_cc_build_time_copy(INPUT "${CMAKE_CURRENT_SOURCE_DIR}/crout/crout.hpp" OUTPU "${CMAKE_BINARY_DIR}/include/crout/crout.hpp") # Eigen -file(COPY ${NMODL_PROJECT_SOURCE_DIR}/ext/eigen/Eigen DESTINATION ${CMAKE_BINARY_DIR}/include/) +if(NOT NMODL_BUILD_WHEEL) + file(COPY ${NMODL_PROJECT_SOURCE_DIR}/ext/eigen/Eigen DESTINATION ${CMAKE_BINARY_DIR}/include/) +endif() # ============================================================================= # Install solver headers and eigen from include # ============================================================================= -install(DIRECTORY ${CMAKE_BINARY_DIR}/include/ DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) +if(NOT NMODL_BUILD_WHEEL) + install(DIRECTORY ${CMAKE_BINARY_DIR}/include/ DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) +endif() From a47e449006002a42bf9f7f30b42aba6bb75b9077 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 27 Feb 2024 10:28:59 +0100 Subject: [PATCH 142/209] Be more general about the test requirements for wheel --- packaging/test_wheel.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 71abad95aa..6706037976 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -51,7 +51,7 @@ fi # install nmodl $python_exe -m pip install -U pip -$python_exe -m pip install "${python_wheel}" pytest +$python_exe -m pip install "${python_wheel}[test]" $python_exe -m pip show nmodl || $python_exe -m pip show nmodl-nightly # run tests From 1ead3a4c474cde193824cffa6f24bda0924f5aca Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 27 Feb 2024 13:22:39 +0100 Subject: [PATCH 143/209] Move nmodl Python bindings to `python/nmodl` Fixes #462 --- pyproject.toml | 1 - {nmodl => python/nmodl}/__init__.py | 0 {nmodl => python/nmodl}/_binwrapper.py | 0 {nmodl => python/nmodl}/ast.py | 0 {nmodl => python/nmodl}/dsl.py | 0 {nmodl => python/nmodl}/ext/example/exp2syn.mod | 0 {nmodl => python/nmodl}/ext/example/expsyn.mod | 0 {nmodl => python/nmodl}/ext/example/hh.mod | 0 {nmodl => python/nmodl}/ext/example/passive.mod | 0 {nmodl => python/nmodl}/ext/viz/css/tree.css | 0 {nmodl => python/nmodl}/ext/viz/index.html | 0 {nmodl => python/nmodl}/ext/viz/js/d3.min.js | 0 {nmodl => python/nmodl}/ext/viz/js/tree.js | 0 {nmodl => python/nmodl}/ode.py | 0 {nmodl => python/nmodl}/symtab.py | 0 {nmodl => python/nmodl}/visitor.py | 0 src/pybind/CMakeLists.txt | 2 +- 17 files changed, 1 insertion(+), 2 deletions(-) rename {nmodl => python/nmodl}/__init__.py (100%) rename {nmodl => python/nmodl}/_binwrapper.py (100%) rename {nmodl => python/nmodl}/ast.py (100%) rename {nmodl => python/nmodl}/dsl.py (100%) rename {nmodl => python/nmodl}/ext/example/exp2syn.mod (100%) rename {nmodl => python/nmodl}/ext/example/expsyn.mod (100%) rename {nmodl => python/nmodl}/ext/example/hh.mod (100%) rename {nmodl => python/nmodl}/ext/example/passive.mod (100%) rename {nmodl => python/nmodl}/ext/viz/css/tree.css (100%) rename {nmodl => python/nmodl}/ext/viz/index.html (100%) rename {nmodl => python/nmodl}/ext/viz/js/d3.min.js (100%) rename {nmodl => python/nmodl}/ext/viz/js/tree.js (100%) rename {nmodl => python/nmodl}/ode.py (100%) rename {nmodl => python/nmodl}/symtab.py (100%) rename {nmodl => python/nmodl}/visitor.py (100%) diff --git a/pyproject.toml b/pyproject.toml index 8e067eef5d..478809f776 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,6 @@ build-backend = "scikit_build_core.build" [tool.scikit-build] metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" -wheel.packages = ["nmodl"] logging.level = "DEBUG" [tool.scikit-build.cmake] diff --git a/nmodl/__init__.py b/python/nmodl/__init__.py similarity index 100% rename from nmodl/__init__.py rename to python/nmodl/__init__.py diff --git a/nmodl/_binwrapper.py b/python/nmodl/_binwrapper.py similarity index 100% rename from nmodl/_binwrapper.py rename to python/nmodl/_binwrapper.py diff --git a/nmodl/ast.py b/python/nmodl/ast.py similarity index 100% rename from nmodl/ast.py rename to python/nmodl/ast.py diff --git a/nmodl/dsl.py b/python/nmodl/dsl.py similarity index 100% rename from nmodl/dsl.py rename to python/nmodl/dsl.py diff --git a/nmodl/ext/example/exp2syn.mod b/python/nmodl/ext/example/exp2syn.mod similarity index 100% rename from nmodl/ext/example/exp2syn.mod rename to python/nmodl/ext/example/exp2syn.mod diff --git a/nmodl/ext/example/expsyn.mod b/python/nmodl/ext/example/expsyn.mod similarity index 100% rename from nmodl/ext/example/expsyn.mod rename to python/nmodl/ext/example/expsyn.mod diff --git a/nmodl/ext/example/hh.mod b/python/nmodl/ext/example/hh.mod similarity index 100% rename from nmodl/ext/example/hh.mod rename to python/nmodl/ext/example/hh.mod diff --git a/nmodl/ext/example/passive.mod b/python/nmodl/ext/example/passive.mod similarity index 100% rename from nmodl/ext/example/passive.mod rename to python/nmodl/ext/example/passive.mod diff --git a/nmodl/ext/viz/css/tree.css b/python/nmodl/ext/viz/css/tree.css similarity index 100% rename from nmodl/ext/viz/css/tree.css rename to python/nmodl/ext/viz/css/tree.css diff --git a/nmodl/ext/viz/index.html b/python/nmodl/ext/viz/index.html similarity index 100% rename from nmodl/ext/viz/index.html rename to python/nmodl/ext/viz/index.html diff --git a/nmodl/ext/viz/js/d3.min.js b/python/nmodl/ext/viz/js/d3.min.js similarity index 100% rename from nmodl/ext/viz/js/d3.min.js rename to python/nmodl/ext/viz/js/d3.min.js diff --git a/nmodl/ext/viz/js/tree.js b/python/nmodl/ext/viz/js/tree.js similarity index 100% rename from nmodl/ext/viz/js/tree.js rename to python/nmodl/ext/viz/js/tree.js diff --git a/nmodl/ode.py b/python/nmodl/ode.py similarity index 100% rename from nmodl/ode.py rename to python/nmodl/ode.py diff --git a/nmodl/symtab.py b/python/nmodl/symtab.py similarity index 100% rename from nmodl/symtab.py rename to python/nmodl/symtab.py diff --git a/nmodl/visitor.py b/python/nmodl/visitor.py similarity index 100% rename from nmodl/visitor.py rename to python/nmodl/visitor.py diff --git a/src/pybind/CMakeLists.txt b/src/pybind/CMakeLists.txt index 9600515e11..d1376ba2e1 100644 --- a/src/pybind/CMakeLists.txt +++ b/src/pybind/CMakeLists.txt @@ -77,7 +77,7 @@ foreach(file IN LISTS NMODL_PYTHON_FILES) cpp_cc_build_time_copy(INPUT ${NMODL_PROJECT_SOURCE_DIR}/nmodl/${file} OUTPUT ${CMAKE_BINARY_DIR}/lib/nmodl/${file}) endforeach() -file(COPY ${NMODL_PROJECT_SOURCE_DIR}/nmodl/ext DESTINATION ${CMAKE_BINARY_DIR}/lib/nmodl/) +file(COPY ${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/ext DESTINATION ${CMAKE_BINARY_DIR}/lib/nmodl/) # ============================================================================= # Install python binding components From 4dd0cde740744e6a4f10878dd59d1cfc45b0c902 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 27 Feb 2024 13:41:57 +0100 Subject: [PATCH 144/209] Minor fixes for testing --- packaging/test_wheel.bash | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packaging/test_wheel.bash b/packaging/test_wheel.bash index 6706037976..ac99d7bdf9 100755 --- a/packaging/test_wheel.bash +++ b/packaging/test_wheel.bash @@ -23,16 +23,14 @@ test_wheel () { # sample mod file for nrnivmodl check TEST_DIR="$(mktemp -d)" OUTPUT_DIR="$(mktemp -d)" - cp "${this_dir}/../nmodl/ext/example/"*.mod "$TEST_DIR/" + cp "${this_dir}/../python/nmodl/ext/example/"*.mod "$TEST_DIR/" cp "${this_dir}/../test/integration/mod/cabpump.mod" "${this_dir}/../test/integration/mod/var_init.inc" "$TEST_DIR/" - cd "${this_dir}" for mod in "${TEST_DIR}/"*.mod do nmodl -o "${OUTPUT_DIR}" "${mod}" sympy --analytic $python_exe -c "import nmodl; driver = nmodl.NmodlDriver(); driver.parse_file('${mod}')" done $python_exe -m pytest -vvv "${this_dir}/../test/" - cd - } echo "== Testing $python_wheel using $python_exe ($python_ver) ==" From a75d65653b83aad6a985a791c2dd815e34785fab Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 27 Feb 2024 15:00:46 +0100 Subject: [PATCH 145/209] Fix wheel.packages No idea why this doesn't work in the CI yet I can build the wheels just fine locally... --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 478809f776..0d057c2f46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ build-backend = "scikit_build_core.build" [tool.scikit-build] metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" +wheel.packages = ["python/nmodl"] logging.level = "DEBUG" [tool.scikit-build.cmake] From 3a8640e564c4fa6e8dcb5bc77738bd68e5f2255e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 27 Feb 2024 15:13:24 +0100 Subject: [PATCH 146/209] Copy also files from build dir --- src/pybind/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pybind/CMakeLists.txt b/src/pybind/CMakeLists.txt index d1376ba2e1..82304c1ae5 100644 --- a/src/pybind/CMakeLists.txt +++ b/src/pybind/CMakeLists.txt @@ -70,8 +70,8 @@ endif() # ============================================================================= file( GLOB NMODL_PYTHON_FILES - RELATIVE "${NMODL_PROJECT_SOURCE_DIR}/nmodl/" - CONFIGURE_DEPENDS "${NMODL_PROJECT_SOURCE_DIR}/nmodl/*.py") + RELATIVE "${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/" + CONFIGURE_DEPENDS "${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/*.py") foreach(file IN LISTS NMODL_PYTHON_FILES) cpp_cc_build_time_copy(INPUT ${NMODL_PROJECT_SOURCE_DIR}/nmodl/${file} OUTPUT From 6453a1ead465271ec2af5f1a9fb6a58cb7a1f50d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 27 Feb 2024 16:32:05 +0100 Subject: [PATCH 147/209] Revert "Copy also files from build dir" This reverts commit 3a8640e564c4fa6e8dcb5bc77738bd68e5f2255e. --- src/pybind/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pybind/CMakeLists.txt b/src/pybind/CMakeLists.txt index 82304c1ae5..d1376ba2e1 100644 --- a/src/pybind/CMakeLists.txt +++ b/src/pybind/CMakeLists.txt @@ -70,8 +70,8 @@ endif() # ============================================================================= file( GLOB NMODL_PYTHON_FILES - RELATIVE "${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/" - CONFIGURE_DEPENDS "${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/*.py") + RELATIVE "${NMODL_PROJECT_SOURCE_DIR}/nmodl/" + CONFIGURE_DEPENDS "${NMODL_PROJECT_SOURCE_DIR}/nmodl/*.py") foreach(file IN LISTS NMODL_PYTHON_FILES) cpp_cc_build_time_copy(INPUT ${NMODL_PROJECT_SOURCE_DIR}/nmodl/${file} OUTPUT From 8eb4c84985071dd88814e1747a17c5788b65169e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 27 Feb 2024 16:56:22 +0100 Subject: [PATCH 148/209] Fix for cmake build --- src/language/code_generator.cmake | 2 +- src/pybind/CMakeLists.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/language/code_generator.cmake b/src/language/code_generator.cmake index 0c6ccae181..a3dea0767f 100644 --- a/src/language/code_generator.cmake +++ b/src/language/code_generator.cmake @@ -34,9 +34,9 @@ set(CODE_GENERATOR_JINJA_FILES set(CODE_GENERATOR_PY_FILES ${PROJECT_SOURCE_DIR}/src/language/argument.py ${PROJECT_SOURCE_DIR}/src/language/code_generator.py + ${PROJECT_SOURCE_DIR}/src/language/language_parser.py ${PROJECT_SOURCE_DIR}/src/language/node_info.py ${PROJECT_SOURCE_DIR}/src/language/nodes.py - ${PROJECT_SOURCE_DIR}/src/language/language_parser.py ${PROJECT_SOURCE_DIR}/src/language/utils.py ) diff --git a/src/pybind/CMakeLists.txt b/src/pybind/CMakeLists.txt index d1376ba2e1..0d7516874b 100644 --- a/src/pybind/CMakeLists.txt +++ b/src/pybind/CMakeLists.txt @@ -70,11 +70,11 @@ endif() # ============================================================================= file( GLOB NMODL_PYTHON_FILES - RELATIVE "${NMODL_PROJECT_SOURCE_DIR}/nmodl/" - CONFIGURE_DEPENDS "${NMODL_PROJECT_SOURCE_DIR}/nmodl/*.py") + RELATIVE "${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/" + CONFIGURE_DEPENDS "${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/*.py") foreach(file IN LISTS NMODL_PYTHON_FILES) - cpp_cc_build_time_copy(INPUT ${NMODL_PROJECT_SOURCE_DIR}/nmodl/${file} OUTPUT + cpp_cc_build_time_copy(INPUT ${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/${file} OUTPUT ${CMAKE_BINARY_DIR}/lib/nmodl/${file}) endforeach() file(COPY ${NMODL_PROJECT_SOURCE_DIR}/python/nmodl/ext DESTINATION ${CMAKE_BINARY_DIR}/lib/nmodl/) From 9853a29e77e26645c6b82322ea3ec675ae599a45 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 09:18:30 +0100 Subject: [PATCH 149/209] Build wheels for all Python versions --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9f64cabe6c..8d8ae9329e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,6 @@ test-extras = ["test"] test-command = [ "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] -build = ["cp311*"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } From 15d94b560db658c4cc656bad3d0dd256a47d39f5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 09:23:40 +0100 Subject: [PATCH 150/209] Fix azure pipeline searching for wheels --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 55ef7fd210..aa0add847c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -283,7 +283,7 @@ stages: steps: - script: | dotless_version="$(echo "$(python.version)" | tr -d '.')" - find . -name "$cp$(python_dotless_version)-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) wheelhouse/{} \; + find $(Pipeline.Workspace) -name "$cp$(python_dotless_version)-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; condition: succeeded() displayName: 'Test manylinux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml From dfde76a8fddc12b06ca6c54dd42101aeb9052f24 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 11:44:11 +0100 Subject: [PATCH 151/209] Update docs --- CONTRIBUTING.rst | 5 ++--- INSTALL.rst | 45 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index c7ad6e594b..4ea7c5da9f 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -201,9 +201,8 @@ the Python API: 1. setup a sandbox environment with either *virtualenv*, *pyenv*, or *pipenv*. For instance with *virtualenv*: ``python -m venv .venv && source .venv/bin/activate`` -2. build the Python package with the command: ``python setup.py build`` -3. install *pytest* Python package: ``pip install pytest`` -4. execute the unit-tests: ``pytest`` +2. build the Python wheel with the command: ``python -m pip wheel . --no-deps`` +3. execute the unit-tests for the wheel: ``bash packaging/test_wheel.bash $(command -v python) WHEEL``, where ``WHEEL`` is the path to the wheel generated in the previous step. Memory Leaks and clang-tidy ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/INSTALL.rst b/INSTALL.rst index 72fff92bde..a37b0474de 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -45,12 +45,13 @@ of all dependencies we recommend using `homebrew `__: brew install flex bison cmake python3 -The necessary Python packages can then easily be added using the pip3 -command. +All of the Python dependencies (build, run, and development) can be installed +using: .. code:: sh - pip3 install --user -r requirements.txt + pip3 install --user pip-tools + pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) Make sure to have latest flex/bison in $PATH : @@ -74,11 +75,13 @@ installed along with the system toolchain: apt-get install flex bison gcc python3 python3-pip -The Python dependencies are installed using: +All of the Python dependencies (build, run, and development) can be installed +using: .. code:: sh - pip3 install --user -r requirements.txt + pip3 install --user pip-tools + pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) Build Project ------------- @@ -139,6 +142,19 @@ This should build the NMODL framework and install it into your pip user ``site-packages`` folder such that it becomes available as a Python module. +Building a wheel +~~~~~~~~~~~~~~~~ + +You can also build a wheel you can test and install in another environment using: + +.. code:: sh + + pip3 wheel . --no-deps [-C OPTION1=VALUE1 -C OPTION2=VALUE2...] [--wheel-dir DIRECTORY] + +where the various ``OPTION`` values describe the build options (for a list of +all available options, please consult the `reference `_). +Notably, due to a bug in CMake, on MacOS one should pass ``-C build-dir=DIRECTORY`` to the above. + When building without linking against libpython ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -210,6 +226,16 @@ example in your Python 3 interpeter as follows: SUFFIX hh } +You can also run all of the Python tests for a given wheel using: + +.. code:: sh + + bash packaging/test_wheel.bash PYTHON_EXECUTABLE WHEEL + +where ``PYTHON_EXECUTABLE`` should be replaced by the path to the Python +executable, and ``WHEEL`` should be replaced by the path to the wheel you wish +to test. + NMODL is now setup correctly! Generating Documentation @@ -219,9 +245,12 @@ In order to build the documentation you must have additionally ``pandoc`` installed. Use your system’s package manager to do this (e.g. ``sudo apt-get install pandoc``). -You can build the entire documentation simply by using sphinx from -``setup.py``: +You can build the entire documentation simply by using the ``generate_docs.sh`` +script: .. code:: sh - python3 setup.py build_ext --inplace docs + bash docs/generate_docs.sh + +The documentation will then be available in the ``public`` subdirectory. Note +that the Python package must be installed beforehand using ``pip install .``. From 7128f08513487f79c22acbe3364f9578f47c7c2c Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 12:27:50 +0100 Subject: [PATCH 152/209] Revert "Build wheels for all Python versions" This reverts commit 9853a29e77e26645c6b82322ea3ec675ae599a45. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8d8ae9329e..9f64cabe6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ test-extras = ["test"] test-command = [ "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] +build = ["cp311*"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } From 57b6001c7ceb85f14b817952d9a87eaf661e2441 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 12:28:27 +0100 Subject: [PATCH 153/209] Update azure testing again --- azure-pipelines.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index aa0add847c..e5048f4391 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -282,8 +282,9 @@ stages: python.version: '3.11' steps: - script: | + set -eux dotless_version="$(echo "$(python.version)" | tr -d '.')" - find $(Pipeline.Workspace) -name "$cp$(python_dotless_version)-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; + find $(Pipeline.Workspace) -name "$cp${python_dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; condition: succeeded() displayName: 'Test manylinux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml From e8abe2ad857aedf40423ad0671de59407b9bd889 Mon Sep 17 00:00:00 2001 From: JCGoran Date: Wed, 28 Feb 2024 13:07:49 +0100 Subject: [PATCH 154/209] Update docs/generate_docs.sh Co-authored-by: Luc Grosheintz --- docs/generate_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 41a4783e22..01f1e57f99 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -4,7 +4,7 @@ # note that the NMODL Python wheel must be installed # for the script to work properly -set -xe +set -xeu # the abs dir where this script is located (so we can call it from wherever) script_dir="$(realpath "$(dirname "$0")")" From 9f4e0a2b61e81aa46116377567d9773033023dd1 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 13:08:48 +0100 Subject: [PATCH 155/209] Yes fine it was an unbound variable --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e5048f4391..8eca9c8db3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -284,7 +284,7 @@ stages: - script: | set -eux dotless_version="$(echo "$(python.version)" | tr -d '.')" - find $(Pipeline.Workspace) -name "$cp${python_dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; + find $(Pipeline.Workspace) -name "$cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; condition: succeeded() displayName: 'Test manylinux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml From c142b9440ff120f2fed7c94d6d4c152244f45f1d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 13:34:13 +0100 Subject: [PATCH 156/209] I hate CI --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8eca9c8db3..523532633b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -284,7 +284,7 @@ stages: - script: | set -eux dotless_version="$(echo "$(python.version)" | tr -d '.')" - find $(Pipeline.Workspace) -name "$cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; + find $(Pipeline.Workspace) -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; condition: succeeded() displayName: 'Test manylinux Wheel with Python $(python.version)' - template: ci/upload-wheels.yml From b1afcd0923d6c10f86f86dbbb841a8df289fa0cb Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 28 Feb 2024 17:03:15 +0100 Subject: [PATCH 157/209] Fix doxygen issues when building docs --- docs/generate_docs.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 01f1e57f99..376e318428 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -6,8 +6,13 @@ set -xeu +# in order to create the docs, we first need to build NMODL +build_dir="$(mktemp -d)" +wheel_dir="$(mktemp -d)" +pip wheel . --no-deps --wheel-dir "$(cd "${wheel_dir}"; pwd -P)" -C build-dir="$(cd "${build_dir}"; pwd -P)" + # the abs dir where this script is located (so we can call it from wherever) -script_dir="$(realpath "$(dirname "$0")")" +script_dir="$(cd "$(dirname "$0")"; pwd -P)" cd "${script_dir}" doxygen Doxyfile From f44ffbbdbc21ca61555e1740ca46cb745ae0a080 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 11:29:47 +0100 Subject: [PATCH 158/209] Allow setting a custom build directory for docs --- docs/generate_docs.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 376e318428..51c2c93500 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -7,7 +7,11 @@ set -xeu # in order to create the docs, we first need to build NMODL -build_dir="$(mktemp -d)" +build_dir="${1:-"$(mktemp -d)"}" +if ! [ -d "${build_dir}" ] +then + mkdir -p "${build_dir}" +fi wheel_dir="$(mktemp -d)" pip wheel . --no-deps --wheel-dir "$(cd "${wheel_dir}"; pwd -P)" -C build-dir="$(cd "${build_dir}"; pwd -P)" From bc6d6924c32036f510ddbb6ee69aec3f6a0fd7ab Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 11:35:20 +0100 Subject: [PATCH 159/209] Minor updates to docs --- INSTALL.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index a37b0474de..7ebbc2dd00 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -127,8 +127,8 @@ to cmake as: -DBISON_EXECUTABLE=/usr/local/opt/bison/bin/bison \ -DCMAKE_INSTALL_PREFIX=$HOME/nmodl -Using Python setuptools -~~~~~~~~~~~~~~~~~~~~~~~ +Using the Python build system +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ If you are mainly interested in the NMODL Framework parsing and analysis tools and wish to use them from Python, we recommend building and @@ -253,4 +253,15 @@ script: bash docs/generate_docs.sh The documentation will then be available in the ``public`` subdirectory. Note -that the Python package must be installed beforehand using ``pip install .``. +that the above script actually builds NMODL using the Python build system in a +temporary directory; if you wish to make a persistent build directory (to +reduce the build time for instance), you can do so by specifying a custom build +directory: + +.. code:: sh + + bash docs/generate_docs.sh DIRECTORY + +where ``DIRECTORY`` is the directory where you wish to store the output of the +build (if the directory does not exist, the script will attempt to create one +for you). From 92578905b120267b0f1ac73a08bcf25c3282b8e1 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 11:49:03 +0100 Subject: [PATCH 160/209] Fix for azure testing manylinux --- azure-pipelines.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 523532633b..a4b38e1a28 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -281,10 +281,14 @@ stages: Python311: python.version: '3.11' steps: + - download: current + patterns: '*.whl' + displayName: "Make manylinux wheels available" - script: | set -eux dotless_version="$(echo "$(python.version)" | tr -d '.')" + find $(Pipeline.Workspace) -name "*cp${dotless_version}-manylinux*.whl" + find . -name "*cp${dotless_version}-manylinux*.whl" find $(Pipeline.Workspace) -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; condition: succeeded() displayName: 'Test manylinux Wheel with Python $(python.version)' - - template: ci/upload-wheels.yml From f8a43d98bbdedea5c6618df47a352547d3f5496d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 13:16:25 +0100 Subject: [PATCH 161/209] Maybe now azure? --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a4b38e1a28..2cb3039e3f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -282,7 +282,7 @@ stages: python.version: '3.11' steps: - download: current - patterns: '*.whl' + patterns: 'drop/*.whl' displayName: "Make manylinux wheels available" - script: | set -eux From a689939f545422327ce5b714608abd217534e189 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 13:54:48 +0100 Subject: [PATCH 162/209] Fix docs building again (and maybe for all) --- INSTALL.rst | 8 +++++++- docs/generate_docs.sh | 28 +++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index 7ebbc2dd00..75de7affbe 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -264,4 +264,10 @@ directory: where ``DIRECTORY`` is the directory where you wish to store the output of the build (if the directory does not exist, the script will attempt to create one -for you). +for you). You can also specify the path to the Python executable, as well as +whether or not to use a virtual env (anything other than the string ``false`` is +considered to be ``true``): + +.. code:: sh + + bash docs/generate_docs.sh DIRECTORY PYTHON_EXECUTABLE false diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 51c2c93500..9bf103ac38 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -1,19 +1,32 @@ #!/usr/bin/env bash # script for generating documentation for NMODL -# note that the NMODL Python wheel must be installed -# for the script to work properly set -xeu -# in order to create the docs, we first need to build NMODL +# in order to create the docs, we first need to build NMODL in some dir build_dir="${1:-"$(mktemp -d)"}" if ! [ -d "${build_dir}" ] then mkdir -p "${build_dir}" fi -wheel_dir="$(mktemp -d)" -pip wheel . --no-deps --wheel-dir "$(cd "${wheel_dir}"; pwd -P)" -C build-dir="$(cd "${build_dir}"; pwd -P)" + +# the Python executable to use (default: whatever `python3` is) +python_exe="${2:-"$(command -v python3)"}" + +# should we use a separate venv? +use_venv="${3:-}" + +if [ "${use_venv}" != "false" ] +then + venv_name="$(mktemp -d)" + $(command -v python3) -m venv "${venv_name}" + . "${venv_name}/bin/activate" + python_exe="$(command -v python)" +else + python_exe="$(command -v python3)" +fi +${python_exe} -m pip install ".[docs]" -C build-dir="$(cd "${build_dir}"; pwd -P)" # the abs dir where this script is located (so we can call it from wherever) script_dir="$(cd "$(dirname "$0")"; pwd -P)" @@ -22,3 +35,8 @@ cd "${script_dir}" doxygen Doxyfile sphinx-build . "${script_dir}/../public" cd - + +if [ "${use_venv}" != "false" ] +then + deactivate +fi From 44a0514ff9a022ed61a7f6fb31b0fd480daa4a96 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 13:58:09 +0100 Subject: [PATCH 163/209] Forgot to actually install Python --- azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2cb3039e3f..549add83a3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -286,9 +286,10 @@ stages: displayName: "Make manylinux wheels available" - script: | set -eux + sudo apt-add-repository -y ppa:deadsnakes/ppa + sudo apt-get update + sudo apt-get install -y python$(python.version) python$(python.version)-dev python$(python.version)-venv dotless_version="$(echo "$(python.version)" | tr -d '.')" - find $(Pipeline.Workspace) -name "*cp${dotless_version}-manylinux*.whl" - find . -name "*cp${dotless_version}-manylinux*.whl" find $(Pipeline.Workspace) -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python$(python.version) {} \; condition: succeeded() displayName: 'Test manylinux Wheel with Python $(python.version)' From 7c5be89ee9f140a27baf5ff3fbc61e42c377d240 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 14:31:23 +0100 Subject: [PATCH 164/209] Fix docs CI to match changes --- .github/workflows/nmodl-doc.yml | 17 +++++++---------- docs/generate_docs.sh | 1 + 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index 7e7d2f1bf1..33df5fe48b 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -61,27 +61,24 @@ jobs: pip3 install -U pip setuptools 'pip-tools>=7.4.0' pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) - # This step will set up an SSH connection on tmate.io for live debugging. - # To trigger it, simply add 'live-debug-docs' to your last pushed commit message. - - name: live debug session on failure - if: failure() && contains(github.event.head_commit.message, 'live-debug-docs') - uses: mxschmitt/action-tmate@v3 - - name: Documentation id: documentation working-directory: ${{runner.workspace}}/nmodl run: | echo "------- Build Documentation -------"; - # build wheel and install it - pip wheel . --wheel-dir wheelhouse/ -C build-dir=_build --no-deps - pip install wheelhouse/*.whl - bash docs/generate_docs.sh + bash docs/generate_docs.sh _build $(command -v python${PYTHON_VERSION}) touch public/.nojekyll echo "" > public/index.html; echo "status=done" >> $GITHUB_OUTPUT env: CCACHE_DIR: ${{runner.workspace}}/ccache + # This step will set up an SSH connection on tmate.io for live debugging. + # To trigger it, simply add 'live-debug-docs' to your last pushed commit message. + - name: live debug session on failure + if: failure() && contains(github.event.head_commit.message, 'live-debug-docs') + uses: mxschmitt/action-tmate@v3 + - name: Deploy 🚀 uses: JamesIves/github-pages-deploy-action@v4 if: steps.documentation.outputs.status == 'done' && startsWith(github.ref, 'refs/heads/master') diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 9bf103ac38..e904367e33 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -23,6 +23,7 @@ then $(command -v python3) -m venv "${venv_name}" . "${venv_name}/bin/activate" python_exe="$(command -v python)" + ${python_exe} -m pip install -U pip else python_exe="$(command -v python3)" fi From f2fbb2ec568ab588c7d99c7e343a5ec62ffddda8 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 15:03:08 +0100 Subject: [PATCH 165/209] Fix no effect statement in docs gen --- docs/generate_docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index e904367e33..283722c8b2 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -20,7 +20,7 @@ use_venv="${3:-}" if [ "${use_venv}" != "false" ] then venv_name="$(mktemp -d)" - $(command -v python3) -m venv "${venv_name}" + ${python_exe} -m venv "${venv_name}" . "${venv_name}/bin/activate" python_exe="$(command -v python)" ${python_exe} -m pip install -U pip From 9373afbd8a304cd0efb764e3542ca6083a20d6d5 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 15:43:31 +0100 Subject: [PATCH 166/209] Fix `change_name.py` script --- packaging/change_name.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packaging/change_name.py b/packaging/change_name.py index f3c453dc56..f54846c14a 100755 --- a/packaging/change_name.py +++ b/packaging/change_name.py @@ -2,10 +2,6 @@ """ Barebones utility for changing the name of the package in `pyproject.toml`. -Usage ------ -[SCRIPT] [PYPROJECT_FILE] [NAME] - Notes ----- Should only be used by the CI @@ -19,7 +15,9 @@ def main(): - parser = ArgumentParser() + parser = ArgumentParser( + description="Script for changing the name of a project in a pyproject.toml file", + ) parser.add_argument("pyproject_file", help="The path to the pyproject.toml file") parser.add_argument("name", help="The new name of the project") args = parser.parse_args() From 23bee8915ab62f137ebb7f09659216434550d16c Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 15:43:43 +0100 Subject: [PATCH 167/209] Leave in the `nmodl.hpp` header --- src/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 614ed9f673..7a81895ff3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,6 +49,4 @@ endif() # Install executable # ============================================================================= install(TARGETS nmodl DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}bin) -if(NOT NMODL_BUILD_WHEEL) - install(FILES nmodl.hpp DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) -endif() +install(FILES nmodl.hpp DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) From c15cbbcc2d83409ec609649c35303fc0ab7263fa Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Thu, 29 Feb 2024 16:19:25 +0100 Subject: [PATCH 168/209] Build everything --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9f64cabe6c..8d8ae9329e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,6 @@ test-extras = ["test"] test-command = [ "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] -build = ["cp311*"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } From 1ef357f16462d0867401c3903bbedd27287651fa Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 09:29:50 +0100 Subject: [PATCH 169/209] Refactor generation of docs --- docs/generate_docs.sh | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 283722c8b2..335ec96518 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -2,21 +2,28 @@ # script for generating documentation for NMODL -set -xeu +set -eu -# in order to create the docs, we first need to build NMODL in some dir -build_dir="${1:-"$(mktemp -d)"}" -if ! [ -d "${build_dir}" ] +if [ $# -lt 2 ] then - mkdir -p "${build_dir}" + echo "Usage: $(basename "$0") output_dir python_exe [use_virtual_env]" + exit 1 fi -# the Python executable to use (default: whatever `python3` is) -python_exe="${2:-"$(command -v python3)"}" - -# should we use a separate venv? +# the dir where we put the temporary build and the docs +output_dir="$1" +# path to the Python executable +python_exe="$2" use_venv="${3:-}" +if ! [ -d "${output_dir}" ] +then + mkdir -p "${output_dir}" +fi + +echo "== Building documentation files in: ${output_dir}/_docs ==" +echo "== Temporary project build directory is: ${output_dir}/_build ==" + if [ "${use_venv}" != "false" ] then venv_name="$(mktemp -d)" @@ -24,18 +31,16 @@ then . "${venv_name}/bin/activate" python_exe="$(command -v python)" ${python_exe} -m pip install -U pip -else - python_exe="$(command -v python3)" fi -${python_exe} -m pip install ".[docs]" -C build-dir="$(cd "${build_dir}"; pwd -P)" +${python_exe} -m pip install ".[docs]" -C build-dir="${output_dir}/_build" # the abs dir where this script is located (so we can call it from wherever) script_dir="$(cd "$(dirname "$0")"; pwd -P)" cd "${script_dir}" doxygen Doxyfile -sphinx-build . "${script_dir}/../public" cd - +sphinx-build docs/ "${output_dir}/_docs" if [ "${use_venv}" != "false" ] then From 23354035f6a05f61d2dbbc23ab63fcc90584e97d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 09:32:52 +0100 Subject: [PATCH 170/209] Update docs CI --- .github/workflows/nmodl-doc.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index 33df5fe48b..41c5bd13a9 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -66,9 +66,9 @@ jobs: working-directory: ${{runner.workspace}}/nmodl run: | echo "------- Build Documentation -------"; - bash docs/generate_docs.sh _build $(command -v python${PYTHON_VERSION}) - touch public/.nojekyll - echo "" > public/index.html; + bash docs/generate_docs.sh public $(command -v python${PYTHON_VERSION}) false + touch public/_docs/.nojekyll + echo "" > public/_docs/index.html echo "status=done" >> $GITHUB_OUTPUT env: CCACHE_DIR: ${{runner.workspace}}/ccache @@ -84,5 +84,5 @@ jobs: if: steps.documentation.outputs.status == 'done' && startsWith(github.ref, 'refs/heads/master') with: branch: gh-pages # The branch the action should deploy to. - folder: ${{runner.workspace}}/nmodl/public + folder: ${{runner.workspace}}/nmodl/public/_docs clean: false # Automatically remove deleted files from the deploy branch From 5931038d208c1fe097b2048bef40af9ad180fc0a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 09:35:53 +0100 Subject: [PATCH 171/209] Put back the version --- nmodl/__init__.py | 8 +------- src/pybind/pynmodl.cpp | 1 + 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/nmodl/__init__.py b/nmodl/__init__.py index 781d12faa4..c494b042cf 100644 --- a/nmodl/__init__.py +++ b/nmodl/__init__.py @@ -35,13 +35,7 @@ try: # Try importing but catch exception in case bindings are not available - from ._nmodl import NmodlDriver, to_json, to_nmodl # noqa - - try: - __version__ = version("nmodl") - except PackageNotFoundError: - # package is not installed - pass + from ._nmodl import NmodlDriver, to_json, to_nmodl, __version__ # noqa __all__ = ["NmodlDriver", "to_json", "to_nmodl"] except ImportError: diff --git a/src/pybind/pynmodl.cpp b/src/pybind/pynmodl.cpp index b4a8b80a0e..bb58f3fc77 100644 --- a/src/pybind/pynmodl.cpp +++ b/src/pybind/pynmodl.cpp @@ -138,6 +138,7 @@ void init_symtab_module(py::module& m); PYBIND11_MODULE(_nmodl, m_nmodl) { m_nmodl.doc() = "NMODL : Source-to-Source Code Generation Framework"; + m_nmodl.attr("__version__") = nmodl::Version::NMODL_VERSION; py::class_ _{m_nmodl, "nmodl::parser::NmodlDriver"}; py::class_ nmodl_driver( From 859ee448e654475aba7acc75cd2fb3c9d012ce39 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 09:41:29 +0100 Subject: [PATCH 172/209] Update INSTALL.rst --- INSTALL.rst | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index 75de7affbe..708491b2df 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -250,24 +250,11 @@ script: .. code:: sh - bash docs/generate_docs.sh - -The documentation will then be available in the ``public`` subdirectory. Note -that the above script actually builds NMODL using the Python build system in a -temporary directory; if you wish to make a persistent build directory (to -reduce the build time for instance), you can do so by specifying a custom build -directory: - -.. code:: sh - - bash docs/generate_docs.sh DIRECTORY - -where ``DIRECTORY`` is the directory where you wish to store the output of the -build (if the directory does not exist, the script will attempt to create one -for you). You can also specify the path to the Python executable, as well as -whether or not to use a virtual env (anything other than the string ``false`` is -considered to be ``true``): - -.. code:: sh - - bash docs/generate_docs.sh DIRECTORY PYTHON_EXECUTABLE false + bash docs/generate_docs.sh DIRECTORY PYTHON_EXECUTABLE [use_virtual_env] + +where ``DIRECTORY`` is where you want to put the output files. The HTML +documentation will then be available in ``DIRECTORY/_docs``, and the temporary +build will be stored in ``DIRECTORY/_build``. You can also specify that you do +not want to use a virtual environment by optionally passing ``false`` as the +third argument above (if not specified, the script first creates a temporary +env). From 8ad9f0fc2ffcd59a6523f3d6bfa1b4adb27fda58 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 09:51:57 +0100 Subject: [PATCH 173/209] Save valuable time and keystrokes by not using underscores --- .github/workflows/nmodl-doc.yml | 6 +++--- INSTALL.rst | 4 ++-- docs/generate_docs.sh | 11 +++++++---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index 41c5bd13a9..cbeefba372 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -67,8 +67,8 @@ jobs: run: | echo "------- Build Documentation -------"; bash docs/generate_docs.sh public $(command -v python${PYTHON_VERSION}) false - touch public/_docs/.nojekyll - echo "" > public/_docs/index.html + touch public/docs/.nojekyll + echo "" > public/docs/index.html echo "status=done" >> $GITHUB_OUTPUT env: CCACHE_DIR: ${{runner.workspace}}/ccache @@ -84,5 +84,5 @@ jobs: if: steps.documentation.outputs.status == 'done' && startsWith(github.ref, 'refs/heads/master') with: branch: gh-pages # The branch the action should deploy to. - folder: ${{runner.workspace}}/nmodl/public/_docs + folder: ${{runner.workspace}}/nmodl/public/docs clean: false # Automatically remove deleted files from the deploy branch diff --git a/INSTALL.rst b/INSTALL.rst index 708491b2df..c0a6f2c118 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -253,8 +253,8 @@ script: bash docs/generate_docs.sh DIRECTORY PYTHON_EXECUTABLE [use_virtual_env] where ``DIRECTORY`` is where you want to put the output files. The HTML -documentation will then be available in ``DIRECTORY/_docs``, and the temporary -build will be stored in ``DIRECTORY/_build``. You can also specify that you do +documentation will then be available in ``DIRECTORY/docs``, and the temporary +build will be stored in ``DIRECTORY/build``. You can also specify that you do not want to use a virtual environment by optionally passing ``false`` as the third argument above (if not specified, the script first creates a temporary env). diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 335ec96518..807f585f81 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -21,8 +21,11 @@ then mkdir -p "${output_dir}" fi -echo "== Building documentation files in: ${output_dir}/_docs ==" -echo "== Temporary project build directory is: ${output_dir}/_build ==" +build_dir="build" +docs_dir="docs" + +echo "== Building documentation files in: ${output_dir}/${docs_dir} ==" +echo "== Temporary project build directory is: ${output_dir}/${build_dir} ==" if [ "${use_venv}" != "false" ] then @@ -32,7 +35,7 @@ then python_exe="$(command -v python)" ${python_exe} -m pip install -U pip fi -${python_exe} -m pip install ".[docs]" -C build-dir="${output_dir}/_build" +${python_exe} -m pip install ".[docs]" -C build-dir="${output_dir}/${build_dir}" # the abs dir where this script is located (so we can call it from wherever) script_dir="$(cd "$(dirname "$0")"; pwd -P)" @@ -40,7 +43,7 @@ script_dir="$(cd "$(dirname "$0")"; pwd -P)" cd "${script_dir}" doxygen Doxyfile cd - -sphinx-build docs/ "${output_dir}/_docs" +sphinx-build docs/ "${output_dir}/${docs_dir}" if [ "${use_venv}" != "false" ] then From 2cc77338d972f94f9b6e0007b24770461e7b8dd9 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 10:35:20 +0100 Subject: [PATCH 174/209] Remove comment --- packaging/change_name.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packaging/change_name.py b/packaging/change_name.py index f54846c14a..478eb30c35 100755 --- a/packaging/change_name.py +++ b/packaging/change_name.py @@ -1,10 +1,6 @@ #!/usr/bin/env python3 """ Barebones utility for changing the name of the package in `pyproject.toml`. - -Notes ------ -Should only be used by the CI """ import re from argparse import ArgumentParser From 66b342a8a1ccacf62ecf9661368cb688f990f325 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 10:52:25 +0100 Subject: [PATCH 175/209] Use two azure runners to build wheels on MacOS One x86_64 and one arm64 --- azure-pipelines.yml | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 549add83a3..49f8488111 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -226,10 +226,43 @@ stages: displayName: 'Publish wheel as build artifact' - template: ci/upload-wheels.yml - - job: 'macos_wheels' + - job: 'macos_wheels_x86_64' timeoutInMinutes: 45 pool: - vmImage: 'macOS-11' + vmImage: 'macOS-12' + steps: + - checkout: self + submodules: True + condition: succeeded() + - script: | + brew install flex bison cmake ninja + condition: succeeded() + displayName: 'Install Dependencies' + - task: UsePythonVersion@0 + - script: | + if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then + export NMODL_NIGHTLY_TAG="-nightly" + else + export NMODL_NIGHTLY_TAG="" + fi + python3 -m pip install --upgrade pip + python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w + # change the name accordingly + python3 packaging/change_name.py pyproject.toml "nmodl$TAG" + SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse + condition: succeeded() + displayName: 'Build macos Wheel (x86_64)' + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' + condition: succeeded() + displayName: 'Publish wheel as build artifact' + - template: ci/upload-wheels.yml + + - job: 'macos_wheels_arm64' + timeoutInMinutes: 45 + pool: + vmImage: 'macOS-14' steps: - checkout: self submodules: True @@ -251,7 +284,7 @@ stages: python3 packaging/change_name.py pyproject.toml "nmodl$TAG" SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() - displayName: 'Build macos Wheel' + displayName: 'Build macos Wheel (arm64)' - task: PublishBuildArtifacts@1 inputs: pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' From e07a6d22af0740fd6492110896386ce221db3b8d Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 11:41:53 +0100 Subject: [PATCH 176/209] Update docgen --- docs/generate_docs.sh | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/generate_docs.sh b/docs/generate_docs.sh index 807f585f81..c0a756d326 100755 --- a/docs/generate_docs.sh +++ b/docs/generate_docs.sh @@ -4,17 +4,16 @@ set -eu -if [ $# -lt 2 ] +if [ $# -lt 1 ] then - echo "Usage: $(basename "$0") output_dir python_exe [use_virtual_env]" + echo "Usage: $(basename "$0") output_dir [python_exe]" exit 1 fi # the dir where we put the temporary build and the docs output_dir="$1" # path to the Python executable -python_exe="$2" -use_venv="${3:-}" +python_exe="${2:-"$(command -v python3)"}" if ! [ -d "${output_dir}" ] then @@ -27,14 +26,11 @@ docs_dir="docs" echo "== Building documentation files in: ${output_dir}/${docs_dir} ==" echo "== Temporary project build directory is: ${output_dir}/${build_dir} ==" -if [ "${use_venv}" != "false" ] -then - venv_name="$(mktemp -d)" - ${python_exe} -m venv "${venv_name}" - . "${venv_name}/bin/activate" - python_exe="$(command -v python)" - ${python_exe} -m pip install -U pip -fi +venv_name="${output_dir}/env" +${python_exe} -m venv "${venv_name}" +. "${venv_name}/bin/activate" +python_exe="$(command -v python)" +${python_exe} -m pip install -U pip ${python_exe} -m pip install ".[docs]" -C build-dir="${output_dir}/${build_dir}" # the abs dir where this script is located (so we can call it from wherever) @@ -45,7 +41,4 @@ doxygen Doxyfile cd - sphinx-build docs/ "${output_dir}/${docs_dir}" -if [ "${use_venv}" != "false" ] -then - deactivate -fi +deactivate From d93760e6ca73b923dd63bb1ab2bf67a09268c210 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 11:46:31 +0100 Subject: [PATCH 177/209] Update dosc --- INSTALL.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index c0a6f2c118..3d3ce529db 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -250,11 +250,9 @@ script: .. code:: sh - bash docs/generate_docs.sh DIRECTORY PYTHON_EXECUTABLE [use_virtual_env] + bash docs/generate_docs.sh DIRECTORY [PYTHON_EXECUTABLE] where ``DIRECTORY`` is where you want to put the output files. The HTML documentation will then be available in ``DIRECTORY/docs``, and the temporary -build will be stored in ``DIRECTORY/build``. You can also specify that you do -not want to use a virtual environment by optionally passing ``false`` as the -third argument above (if not specified, the script first creates a temporary -env). +build will be stored in ``DIRECTORY/build``. You can also specify the path to +the Python executable if it is not picked up automatically. From c21722caa6976ef2c0f5f873932635deac37b857 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 11:49:03 +0100 Subject: [PATCH 178/209] Update CI --- .github/workflows/nmodl-doc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index cbeefba372..8053257d36 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -66,7 +66,7 @@ jobs: working-directory: ${{runner.workspace}}/nmodl run: | echo "------- Build Documentation -------"; - bash docs/generate_docs.sh public $(command -v python${PYTHON_VERSION}) false + bash docs/generate_docs.sh public $(command -v python${PYTHON_VERSION}) touch public/docs/.nojekyll echo "" > public/docs/index.html echo "status=done" >> $GITHUB_OUTPUT From 713e32e3e2924119054957bad2da794275f9edf3 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 14:59:42 +0100 Subject: [PATCH 179/209] Remove useless flags --- CMakeLists.txt | 1 - src/solver/CMakeLists.txt | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f5ed91357..0e5988eb79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,6 @@ option(NMODL_BUILD_WHEEL "Flag to signal we are building a wheel" OFF) if(NMODL_BUILD_WHEEL) set(LINK_AGAINST_PYTHON OFF) set(NMODL_ENABLE_TESTS OFF) - set(FMT_INSTALL OFF) endif() # ============================================================================= diff --git a/src/solver/CMakeLists.txt b/src/solver/CMakeLists.txt index 1ffa266dee..78c44c5d0d 100644 --- a/src/solver/CMakeLists.txt +++ b/src/solver/CMakeLists.txt @@ -9,14 +9,10 @@ cpp_cc_build_time_copy(INPUT "${CMAKE_CURRENT_SOURCE_DIR}/crout/crout.hpp" OUTPU "${CMAKE_BINARY_DIR}/include/crout/crout.hpp") # Eigen -if(NOT NMODL_BUILD_WHEEL) - file(COPY ${NMODL_PROJECT_SOURCE_DIR}/ext/eigen/Eigen DESTINATION ${CMAKE_BINARY_DIR}/include/) -endif() +file(COPY ${NMODL_PROJECT_SOURCE_DIR}/ext/eigen/Eigen DESTINATION ${CMAKE_BINARY_DIR}/include/) # ============================================================================= # Install solver headers and eigen from include # ============================================================================= -if(NOT NMODL_BUILD_WHEEL) - install(DIRECTORY ${CMAKE_BINARY_DIR}/include/ DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) -endif() +install(DIRECTORY ${CMAKE_BINARY_DIR}/include/ DESTINATION ${NMODL_INSTALL_DIR_SUFFIX}include) From 5c67ac771c58d0b9cea59aba79160b330ac71c9a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 15:00:05 +0100 Subject: [PATCH 180/209] Remove useless CI job --- .github/workflows/nmodl-doc.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/nmodl-doc.yml b/.github/workflows/nmodl-doc.yml index 8053257d36..da1c062b35 100644 --- a/.github/workflows/nmodl-doc.yml +++ b/.github/workflows/nmodl-doc.yml @@ -55,12 +55,6 @@ jobs: with: fetch-depth: 0 - - name: Install Python3 dependencies - working-directory: ${{runner.workspace}}/nmodl - run: | - pip3 install -U pip setuptools 'pip-tools>=7.4.0' - pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) - - name: Documentation id: documentation working-directory: ${{runner.workspace}}/nmodl From ce0fabfd741cab3a1a26106b240954a53f13176e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 16:08:15 +0100 Subject: [PATCH 181/209] Remove MacOS 14 runner on azure as its unavailable --- azure-pipelines.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 49f8488111..b6fc53d39e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -259,39 +259,6 @@ stages: displayName: 'Publish wheel as build artifact' - template: ci/upload-wheels.yml - - job: 'macos_wheels_arm64' - timeoutInMinutes: 45 - pool: - vmImage: 'macOS-14' - steps: - - checkout: self - submodules: True - condition: succeeded() - - script: | - brew install flex bison cmake ninja - condition: succeeded() - displayName: 'Install Dependencies' - - task: UsePythonVersion@0 - - script: | - if [[ "$(RELEASEWHEELBUILD)" != "True" ]]; then - export NMODL_NIGHTLY_TAG="-nightly" - else - export NMODL_NIGHTLY_TAG="" - fi - python3 -m pip install --upgrade pip - python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w - # change the name accordingly - python3 packaging/change_name.py pyproject.toml "nmodl$TAG" - SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse - condition: succeeded() - displayName: 'Build macos Wheel (arm64)' - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: '$(Build.SourcesDirectory)/wheelhouse' - condition: succeeded() - displayName: 'Publish wheel as build artifact' - - template: ci/upload-wheels.yml - - job: 'test_manylinux_wheels' dependsOn: 'manylinux_wheels' timeoutInMinutes: 45 From c806063b7750c024c1e2a81d0e9abb88a773715a Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 16:08:35 +0100 Subject: [PATCH 182/209] Build and test all wheels on GHA --- .github/workflows/nmodl-wheel.yml | 75 +++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 0d3716ddee..8bc616f267 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -5,14 +5,23 @@ on: branches: - master pull_request: + branches: + - master + - releases/* + schedule: + - cron: "5 0 * * *" + branches: + - master + workflow_dispatch: jobs: - build_wheels: + build_wheels_nonlinux: name: Build wheels on ${{ matrix.os }} + timeout-minutes: 45 runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macOS-13, macOS-14] + os: [macOS-13, macOS-14] steps: - name: Setup Flex and Bison @@ -30,13 +39,71 @@ jobs: # Used to host cibuildwheel - uses: actions/setup-python@v3 + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.16.5 + + - name: Build all wheels + run: | + export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" + python -m cibuildwheel --output-dir wheelhouse + + - uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }} + path: ./wheelhouse/*.whl + + build_wheels_manylinux: + name: Build manylinux wheels + timeout-minutes: 45 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + # Used to host cibuildwheel + - uses: actions/setup-python@v3 + - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.16.5 - name: Build wheels - run: python -m cibuildwheel --output-dir wheelhouse + run: | + export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" + CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python -m cibuildwheel --output-dir wheelhouse - uses: actions/upload-artifact@v4 with: - name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + name: cibw-wheels-manylinux path: ./wheelhouse/*.whl + + # testing of wheels cannot be done in a manylinux image due to the lack of + # the Python library; as a workaround, we test each wheel in a separate + # version + test_wheels_manylinux: + name: Test manylinux wheel on Python ${{ matrix.python_version }} + runs-on: ubuntu-latest + needs: build_wheels_manylinux + timeout-minutes: 45 + strategy: + matrix: + python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + depends-on: manylinux_wheels + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Make manylinux wheels available + uses: actions/download-artifact@v2 + with: + name: cibw-wheels-manylinux + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python_version }} + + - name: Test manylinux Wheel with Python ${{ matrix.python_version }} + run: | + set -eux + dotless_version="$(echo "${matrix.python_version}" | tr -d '.')" + find ${{ github.workspace }} -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python${{ matrix.python_version }} {} \; From 1af5ec467fb8ffbae2924892998a729c84a9232f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 16:12:03 +0100 Subject: [PATCH 183/209] Enable non-master CI for now --- .github/workflows/nmodl-wheel.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 8bc616f267..9d9039ddee 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -5,9 +5,6 @@ on: branches: - master pull_request: - branches: - - master - - releases/* schedule: - cron: "5 0 * * *" branches: From b1ee0c750b1688d22492c4b16fb330cd8cbd4e5c Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 16:13:46 +0100 Subject: [PATCH 184/209] Maybe now? --- .github/workflows/nmodl-wheel.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 9d9039ddee..8699df1c56 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -1,9 +1,11 @@ name: Build wheels +concurrency: + group: ${{ github.workflow }}#${{ github.ref }} + cancel-in-progress: true + on: push: - branches: - - master pull_request: schedule: - cron: "5 0 * * *" From e43d04c644fde6856acf83f0dae68b4a41d07012 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Fri, 1 Mar 2024 17:17:02 +0100 Subject: [PATCH 185/209] Fix typo --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 8699df1c56..a38236916e 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -7,7 +7,7 @@ concurrency: on: push: pull_request: - schedule: + schedule: - cron: "5 0 * * *" branches: - master From 959b0d5ce3c6c3932e3fc210e10208696e038646 Mon Sep 17 00:00:00 2001 From: JCGoran Date: Fri, 1 Mar 2024 17:20:57 +0100 Subject: [PATCH 186/209] Update nmodl-wheel.yml --- .github/workflows/nmodl-wheel.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index a38236916e..2cff029ac1 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -6,11 +6,12 @@ concurrency: on: push: + branches: + - master pull_request: schedule: - cron: "5 0 * * *" - branches: - - master + workflow_dispatch: jobs: @@ -86,7 +87,6 @@ jobs: strategy: matrix: python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - depends-on: manylinux_wheels steps: - name: Checkout code uses: actions/checkout@v4 From fe0663947147120e159b47d5d88f9b4e39ece07f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 09:07:33 +0100 Subject: [PATCH 187/209] Download artifact v4 maybe? --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index a38236916e..b6cffca0a5 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -92,7 +92,7 @@ jobs: uses: actions/checkout@v4 - name: Make manylinux wheels available - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: name: cibw-wheels-manylinux From 1930073ead0b276edbc11acd9b8a16e2578ce2f1 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 09:09:23 +0100 Subject: [PATCH 188/209] Revert "Build everything" This reverts commit c15cbbcc2d83409ec609649c35303fc0ab7263fa. --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 780602cb6b..31ec49cc75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,7 @@ test-extras = ["test"] test-command = [ "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] +build = ["cp311*"] [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } From 85c76b945024a30aa16a3bb2242c54922844cbee Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 09:37:20 +0100 Subject: [PATCH 189/209] Cmon github expand the damn variable! --- .github/workflows/nmodl-wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 80b0276078..3d994f5f8f 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -104,5 +104,5 @@ jobs: - name: Test manylinux Wheel with Python ${{ matrix.python_version }} run: | set -eux - dotless_version="$(echo "${matrix.python_version}" | tr -d '.')" + dotless_version="$(echo "${{ matrix.python_version }}" | tr -d '.')" find ${{ github.workspace }} -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python${{ matrix.python_version }} {} \; From 73ab2502aed5d0c52001e751e98f879957323344 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 09:51:47 +0100 Subject: [PATCH 190/209] Final touches for CI --- .github/workflows/nmodl-wheel.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index 3d994f5f8f..af58cae0f6 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -34,10 +34,14 @@ jobs: run: | brew install flex bison - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch: 0 # Used to host cibuildwheel - - uses: actions/setup-python@v3 + - name: Set up Python + uses: actions/setup-python@v3 - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.16.5 @@ -58,10 +62,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch: 0 # Used to host cibuildwheel - - uses: actions/setup-python@v3 + - name: Set up Python + uses: actions/setup-python@v3 - name: Install cibuildwheel run: python -m pip install cibuildwheel==2.16.5 @@ -103,6 +111,5 @@ jobs: - name: Test manylinux Wheel with Python ${{ matrix.python_version }} run: | - set -eux dotless_version="$(echo "${{ matrix.python_version }}" | tr -d '.')" find ${{ github.workspace }} -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python${{ matrix.python_version }} {} \; From bd873262c41c15ec9e20c139f41237fc85429875 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 10:05:13 +0100 Subject: [PATCH 191/209] Last touches in CI --- .github/workflows/nmodl-wheel.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml index af58cae0f6..f46410b1ab 100644 --- a/.github/workflows/nmodl-wheel.yml +++ b/.github/workflows/nmodl-wheel.yml @@ -36,8 +36,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - with: - fetch: 0 + with: + fetch: 0 # Used to host cibuildwheel - name: Set up Python @@ -51,7 +51,8 @@ jobs: export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v4 + - name: Upload wheels + uses: actions/upload-artifact@v4 with: name: cibw-wheels-${{ matrix.os }} path: ./wheelhouse/*.whl @@ -79,7 +80,8 @@ jobs: export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python -m cibuildwheel --output-dir wheelhouse - - uses: actions/upload-artifact@v4 + - name: Upload wheels + uses: actions/upload-artifact@v4 with: name: cibw-wheels-manylinux path: ./wheelhouse/*.whl From 1a367f328d7e8990e16eaab72c37b97bac72fa94 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 10:43:58 +0100 Subject: [PATCH 192/209] WIP test circle CI wheels --- .circleci/config.yml | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d0736bba6..35de8f32d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2 jobs: - linux-aarch64-wheels: + build_wheels_manylinux_aarch64: working_directory: ~/linux-aarch64-wheels machine: image: default @@ -11,15 +11,50 @@ jobs: steps: - checkout - run: - name: Build the Linux aarch64 wheels. + name: Build the manylinux aarch64 wheels command: | python3 -m pip install --user cibuildwheel==2.16.5 python3 -m cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ + test_wheels_manylinux_aarch64: + parameters: + python_version: + type: string + default: "3.8" + docker: + - image: circleci/python:$python_version + steps: + - checkout + - run: + name: Test the manylinux aarch64 wheels + command: | + dotless_version="$(echo "${python_version}" | tr -d '.')" + find ${{ github.workspace }} -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python${{ matrix.python_version }} {} \; + workflows: version: 2 - all-tests: + build: jobs: - - linux-aarch64-wheels + - build_wheels_manylinux_aarch64 + + - test_wheels_manylinux_aarch64: + python_version: "3.8" + requires: build_wheels_manylinux_aarch64 + + - test_wheels_manylinux_aarch64: + python_version: "3.9" + requires: build_wheels_manylinux_aarch64 + + - test_wheels_manylinux_aarch64: + python_version: "3.10" + requires: build_wheels_manylinux_aarch64 + + - test_wheels_manylinux_aarch64: + python_version: "3.11" + requires: build_wheels_manylinux_aarch64 + + - test_wheels_manylinux_aarch64: + python_version: "3.12" + requires: build_wheels_manylinux_aarch64 From 93fcef304eb8dcc049d692fa01496c018e8fd85c Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 10:45:25 +0100 Subject: [PATCH 193/209] Fix useless imports in init --- python/nmodl/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/nmodl/__init__.py b/python/nmodl/__init__.py index c494b042cf..7b03e05f7f 100644 --- a/python/nmodl/__init__.py +++ b/python/nmodl/__init__.py @@ -3,10 +3,8 @@ if sys.version_info >= (3, 9): from importlib.resources import files - from importlib.metadata import version, PackageNotFoundError else: from importlib_resources import files - from importlib_metadata import version, PackageNotFoundError from find_libpython import find_libpython From 26e6193f33208420c37e46968a02959057c1dd7e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 10:48:18 +0100 Subject: [PATCH 194/209] Raise an error if NMODL is not available --- python/nmodl/__init__.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/python/nmodl/__init__.py b/python/nmodl/__init__.py index 7b03e05f7f..537a7ea042 100644 --- a/python/nmodl/__init__.py +++ b/python/nmodl/__init__.py @@ -31,10 +31,6 @@ ) -try: - # Try importing but catch exception in case bindings are not available - from ._nmodl import NmodlDriver, to_json, to_nmodl, __version__ # noqa +from ._nmodl import NmodlDriver, to_json, to_nmodl, __version__ # noqa - __all__ = ["NmodlDriver", "to_json", "to_nmodl"] -except ImportError: - print("[NMODL] [warning] :: Python bindings are not available") +__all__ = ["NmodlDriver", "to_json", "to_nmodl"] From 5e4f395b00c90badcf56f63df6a801796c050007 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 10:53:34 +0100 Subject: [PATCH 195/209] Fix CIRCLECI --- .circleci/config.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35de8f32d4..4cbf174355 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,23 +38,27 @@ workflows: build: jobs: - build_wheels_manylinux_aarch64 - - test_wheels_manylinux_aarch64: python_version: "3.8" - requires: build_wheels_manylinux_aarch64 + requires: + - build_wheels_manylinux_aarch64 - test_wheels_manylinux_aarch64: python_version: "3.9" - requires: build_wheels_manylinux_aarch64 + requires: + - build_wheels_manylinux_aarch64 - test_wheels_manylinux_aarch64: python_version: "3.10" - requires: build_wheels_manylinux_aarch64 + requires: + - build_wheels_manylinux_aarch64 - test_wheels_manylinux_aarch64: python_version: "3.11" - requires: build_wheels_manylinux_aarch64 + requires: + - build_wheels_manylinux_aarch64 - test_wheels_manylinux_aarch64: python_version: "3.12" - requires: build_wheels_manylinux_aarch64 + requires: + - build_wheels_manylinux_aarch64 From b18576b524e390e0d5bffb23910a04b96543e005 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 11:55:20 +0100 Subject: [PATCH 196/209] WIP on circleCI --- .circleci/config.yml | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4cbf174355..943c67edf8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,9 +22,8 @@ jobs: parameters: python_version: type: string - default: "3.8" docker: - - image: circleci/python:$python_version + - image: cimg/python:$python_version steps: - checkout - run: @@ -39,26 +38,8 @@ workflows: jobs: - build_wheels_manylinux_aarch64 - test_wheels_manylinux_aarch64: - python_version: "3.8" - requires: - - build_wheels_manylinux_aarch64 - - - test_wheels_manylinux_aarch64: - python_version: "3.9" - requires: - - build_wheels_manylinux_aarch64 - - - test_wheels_manylinux_aarch64: - python_version: "3.10" - requires: - - build_wheels_manylinux_aarch64 - - - test_wheels_manylinux_aarch64: - python_version: "3.11" - requires: - - build_wheels_manylinux_aarch64 - - - test_wheels_manylinux_aarch64: - python_version: "3.12" requires: - build_wheels_manylinux_aarch64 + matrix: + parameters: + python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] From 58582c7afb9b545965e9b061297d37d3c055da8f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 11:55:58 +0100 Subject: [PATCH 197/209] Remove all ARM stuff for now --- .circleci/config.yml | 45 ------------ .github/workflows/nmodl-wheel.yml | 117 ------------------------------ 2 files changed, 162 deletions(-) delete mode 100644 .circleci/config.yml delete mode 100644 .github/workflows/nmodl-wheel.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 943c67edf8..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: 2 - -jobs: - build_wheels_manylinux_aarch64: - working_directory: ~/linux-aarch64-wheels - machine: - image: default - # resource_class is what tells CircleCI to use an ARM worker for native arm builds - # https://circleci.com/product/features/resource-classes/ - resource_class: arm.medium - steps: - - checkout - - run: - name: Build the manylinux aarch64 wheels - command: | - python3 -m pip install --user cibuildwheel==2.16.5 - python3 -m cibuildwheel --output-dir wheelhouse - - store_artifacts: - path: wheelhouse/ - - test_wheels_manylinux_aarch64: - parameters: - python_version: - type: string - docker: - - image: cimg/python:$python_version - steps: - - checkout - - run: - name: Test the manylinux aarch64 wheels - command: | - dotless_version="$(echo "${python_version}" | tr -d '.')" - find ${{ github.workspace }} -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python${{ matrix.python_version }} {} \; - -workflows: - version: 2 - build: - jobs: - - build_wheels_manylinux_aarch64 - - test_wheels_manylinux_aarch64: - requires: - - build_wheels_manylinux_aarch64 - matrix: - parameters: - python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] diff --git a/.github/workflows/nmodl-wheel.yml b/.github/workflows/nmodl-wheel.yml deleted file mode 100644 index f46410b1ab..0000000000 --- a/.github/workflows/nmodl-wheel.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: Build wheels - -concurrency: - group: ${{ github.workflow }}#${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: - - master - pull_request: - schedule: - - cron: "5 0 * * *" - - workflow_dispatch: - -jobs: - build_wheels_nonlinux: - name: Build wheels on ${{ matrix.os }} - timeout-minutes: 45 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [macOS-13, macOS-14] - - steps: - - name: Setup Flex and Bison - if: runner.os == 'Windows' - run: | - choco install winflexbison3 -y - - - name: Setup Flex and Bison - if: runner.os == 'macOS' - run: | - brew install flex bison - - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch: 0 - - # Used to host cibuildwheel - - name: Set up Python - uses: actions/setup-python@v3 - - - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.16.5 - - - name: Build all wheels - run: | - export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" - python -m cibuildwheel --output-dir wheelhouse - - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: cibw-wheels-${{ matrix.os }} - path: ./wheelhouse/*.whl - - build_wheels_manylinux: - name: Build manylinux wheels - timeout-minutes: 45 - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch: 0 - - # Used to host cibuildwheel - - name: Set up Python - uses: actions/setup-python@v3 - - - name: Install cibuildwheel - run: python -m pip install cibuildwheel==2.16.5 - - - name: Build wheels - run: | - export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" - CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python -m cibuildwheel --output-dir wheelhouse - - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: cibw-wheels-manylinux - path: ./wheelhouse/*.whl - - # testing of wheels cannot be done in a manylinux image due to the lack of - # the Python library; as a workaround, we test each wheel in a separate - # version - test_wheels_manylinux: - name: Test manylinux wheel on Python ${{ matrix.python_version }} - runs-on: ubuntu-latest - needs: build_wheels_manylinux - timeout-minutes: 45 - strategy: - matrix: - python_version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Make manylinux wheels available - uses: actions/download-artifact@v4 - with: - name: cibw-wheels-manylinux - - - name: Set up Python ${{ matrix.python_version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python_version }} - - - name: Test manylinux Wheel with Python ${{ matrix.python_version }} - run: | - dotless_version="$(echo "${{ matrix.python_version }}" | tr -d '.')" - find ${{ github.workspace }} -name "*cp${dotless_version}-manylinux*.whl" -exec bash packaging/test_wheel.bash python${{ matrix.python_version }} {} \; From 71b58fafb39d0c47aaec005657811204eb51d7ab Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 11:59:50 +0100 Subject: [PATCH 198/209] Put back empty circleCI config --- .circleci/config.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..32f6af387a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,19 @@ +# NOTE this file serves one purpose: to silence Circle CI. +# Delete at will. +# +# Copied&modified from: https://circleci.com/docs/sample-config/ +version: 2.1 + +# Define the jobs we want to run for this project +jobs: + build: + docker: + - image: cimg/base:2023.03 + steps: + - run: echo "this is the build job" + +# Orchestrate our job run sequence +workflows: + build_and_test: + jobs: + - build From a74c76cc6db621453b30b70e64962432bf51881b Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 12:05:04 +0100 Subject: [PATCH 199/209] Add requirements.txt --- requirements.txt | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..665e4f40c0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,26 @@ +# build time dependencies +scikit-build-core +setuptools-scm>=8.0 +Jinja2>=2.9.3 +PyYAML>=3.13 +# runtime dependencies +find_libpython +sympy>=1.3 +importlib-metadata;python_version<'3.9' +importlib-resources;python_version<'3.9' +# dependencies for test +pytest>=3.3.0 +pytest-cov +numpy +# dependencies for docs +jupyter-client +jupyter +myst_parser<2.0.0 +mistune<3 +nbconvert +nbsphinx>=0.3.2 +sphinxcontrib-applehelp<1.0.3 +sphinxcontrib-htmlhelp<=2.0.0 +sphinx<6 +sphinx-rtd-theme +docutils<0.20 From 747053b5b480cc84fbcb159e6e23186cb7539cbc Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 12:08:37 +0100 Subject: [PATCH 200/209] Put back requirements in CI --- .github/workflows/coverage.yml | 4 ++-- .github/workflows/nmodl-ci.yml | 4 ++-- .github/workflows/sonarsource.yml | 4 ++-- INSTALL.rst | 6 ++---- azure-pipelines.yml | 8 ++++---- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 24da18c1bb..91532521fc 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -45,8 +45,8 @@ jobs: - name: Install Python3 dependencies working-directory: ${{runner.workspace}}/nmodl run: | - pip3 install -U pip setuptools 'pip-tools>=7.4.0' - pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) + pip3 install -U pip setuptools + pip3 install --user -r requirements.txt - name: Restore compiler cache uses: actions/cache@v4 with: diff --git a/.github/workflows/nmodl-ci.yml b/.github/workflows/nmodl-ci.yml index 784bd936c5..903107f640 100644 --- a/.github/workflows/nmodl-ci.yml +++ b/.github/workflows/nmodl-ci.yml @@ -82,8 +82,8 @@ jobs: - name: Install Python3 dependencies working-directory: ${{runner.workspace}}/nmodl run: | - python3 -m pip install -U pip setuptools 'pip-tools>=7.4.0' - python3 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) + python3 -m pip install -U pip setuptools + python3 -m pip install --user -r requirements.txt - name: Install neuron-nightly if: ${{matrix.config.enable_usecases == 'On'}} diff --git a/.github/workflows/sonarsource.yml b/.github/workflows/sonarsource.yml index 0b13f21b00..dcc03d5ee6 100644 --- a/.github/workflows/sonarsource.yml +++ b/.github/workflows/sonarsource.yml @@ -31,8 +31,8 @@ jobs: - name: Install Python3 dependencies working-directory: ${{runner.workspace}}/nmodl run: | - pip3 install -U pip setuptools 'pip-tools>=7.4.0' - pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) + pip3 install -U pip setuptools + pip3 install --user -r requirements.txt - name: Install sonar-scanner and build-wrapper uses: SonarSource/sonarcloud-github-c-cpp@v2 - name: Configure project diff --git a/INSTALL.rst b/INSTALL.rst index 3d3ce529db..9c4fc6af46 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -50,8 +50,7 @@ using: .. code:: sh - pip3 install --user pip-tools - pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) + pip3 install --user -r requirements.txt Make sure to have latest flex/bison in $PATH : @@ -80,8 +79,7 @@ using: .. code:: sh - pip3 install --user pip-tools - pip3 install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) + pip3 install --user -r requirements.txt Build Project ------------- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ed45d8bd04..663a792dcd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,8 +87,8 @@ stages: sudo apt-get install -y g++-9 flex bison libfl-dev cython libx11-dev libxcomposite-dev libncurses-dev mpich sudo apt-get install -y python3.8 python3.8-dev python3.8-venv ninja-build sudo apt-get remove -y python3-importlib-metadata - python3.8 -m pip install --upgrade pip setuptools 'pip-tools>=7.4.0' - python3.8 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) + python3.8 -m pip install --upgrade pip setuptools + python3.8 -m pip install --user -r requirements.txt # we manually get version 3.15.0 to make sure that changes in the cmake # files do not require unsupported versions of cmake in our package. wget --quiet --output-document=- "https://github.com/Kitware/CMake/releases/download/$CMAKE_VER/$CMAKE_PKG.tar.gz" | tar xzpf - @@ -148,8 +148,8 @@ stages: submodules: True - script: | brew install flex bison cmake python@3 - python3 -m pip install --upgrade pip setuptools 'pip-tools>=7.4.0' - python3 -m pip install --user -r <(pip-compile --all-build-deps --all-extras --no-strip-extras 2>&1) + python3 -m pip install --upgrade pip setuptools + python3 -m pip install --user -r requirements.txt displayName: 'Install Dependencies' - script: | export PATH=/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH; From 62612ccc845bbe6450d6d3f73fb200161dbaac9f Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 12:32:10 +0100 Subject: [PATCH 201/209] Remove checking python dependencies in cmake --- CMakeLists.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e5988eb79..4754ca8222 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,15 +172,6 @@ endif() message(STATUS "CHECKING FOR PYTHON") find_package(PythonInterp 3.8 REQUIRED) cpp_cc_strip_python_shims(EXECUTABLE "${PYTHON_EXECUTABLE}" OUTPUT PYTHON_EXECUTABLE) -include(cmake/hpc-coding-conventions/cpp/cmake/bbp-find-python-module.cmake) -cpp_cc_find_python_module(jinja2 2.9.3 REQUIRED) -if(NMODL_ENABLE_TESTS) - cpp_cc_find_python_module(pytest 3.3.0 REQUIRED) -endif() -if(NOT NMODL_BUILD_WHEEL) - cpp_cc_find_python_module(sympy 1.3 REQUIRED) -endif() -cpp_cc_find_python_module(yaml 3.12 REQUIRED) # ============================================================================= # Compiler specific flags for external submodules From 031dfdbff3d9beac8b8e262a3188a7550bd05c06 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 12:59:04 +0100 Subject: [PATCH 202/209] Put back the init try so CI passes --- python/nmodl/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/nmodl/__init__.py b/python/nmodl/__init__.py index 537a7ea042..dbc53cbe90 100644 --- a/python/nmodl/__init__.py +++ b/python/nmodl/__init__.py @@ -31,6 +31,11 @@ ) -from ._nmodl import NmodlDriver, to_json, to_nmodl, __version__ # noqa +try: + # Try importing but catch exception in case bindings are not available + from ._nmodl import NmodlDriver, to_json, to_nmodl # noqa + from ._nmodl import __version__ -__all__ = ["NmodlDriver", "to_json", "to_nmodl"] + __all__ = ["NmodlDriver", "to_json", "to_nmodl"] +except ImportError: + print("[NMODL] [warning] :: Python bindings are not available") From 6cbcdec6489d742410a34f051419ace8ac6f59d9 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Mon, 4 Mar 2024 16:49:32 +0100 Subject: [PATCH 203/209] Change back name of wheel --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 53d06757f6..8a2dfc75fa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -213,7 +213,7 @@ stages: python3 -m pip install --upgrade pip python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w # change the name accordingly - python3 packaging/change_name.py pyproject.toml "nmodl$TAG" + python3 packaging/change_name.py pyproject.toml "NMODL${TAG}" export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" echo "$SETUPTOOLS_SCM_PRETEND_VERSION" CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python3 -m cibuildwheel --output-dir wheelhouse @@ -248,7 +248,7 @@ stages: python3 -m pip install --upgrade pip python3 -m pip install cibuildwheel==2.16.5 tomli tomli-w # change the name accordingly - python3 packaging/change_name.py pyproject.toml "nmodl$TAG" + python3 packaging/change_name.py pyproject.toml "NMODL${NMODL_NIGHTLY_TAG}" SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() displayName: 'Build macos Wheel (x86_64)' From 2450a2b6202aa643794a689b88a18d6afa097e66 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 5 Mar 2024 11:31:40 +0100 Subject: [PATCH 204/209] Add minor fixes to script for building wheels --- packaging/build_wheels.bash | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/packaging/build_wheels.bash b/packaging/build_wheels.bash index 4a2dd1f143..5dfb0fc493 100755 --- a/packaging/build_wheels.bash +++ b/packaging/build_wheels.bash @@ -83,34 +83,31 @@ build_wheel_osx() { deactivate } -# platform for which wheel to be build -platform=$1 +# platform for which wheel we are building +platform="$1" -# python version for which wheel to be built; 3* (default) means all python 3 versions -python_wheel_version=3* -if [ "$2" ]; then - python_wheel_version=$2 -fi +# python version for which wheel we are building; 3* (default) means all python 3 versions +python_wheel_version="${2:-3*}" # MAIN -case "$1" in +case "${platform}" in linux) - python_wheel_version=${python_wheel_version//[-._]/} - for py_bin in /opt/python/cp${python_wheel_version}*/bin/python; do + python_wheel_version="${python_wheel_version//[-._]/}" + for py_bin in /opt/python/cp${python_wheel_version}-cp${python_wheel_version}/bin/python; do build_wheel_linux "$py_bin" done ;; osx) - for py_bin in /Library/Frameworks/Python.framework/Versions/${python_wheel_version}*/bin/python3; do + for py_bin in /Library/Frameworks/Python.framework/Versions/${python_wheel_version}/bin/python3; do build_wheel_osx "$py_bin" done ;; *) - echo "Usage: $(basename $0) [version]" + echo "Usage: $(basename "$0") [version]" exit 1 ;; From dd2d11f8730df82bfc6b8214dd71274aa8ccc566 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 5 Mar 2024 14:11:48 +0100 Subject: [PATCH 205/209] WIP try to use newer docker NMODL image to build wheels --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 31ec49cc75..dab22c84a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,13 +61,13 @@ test-command = [ "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] build = ["cp311*"] +manylinux-x86_64-image = "docker.io/bluebrain/nmodl:wheel-py12" [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } config-settings = {build-dir = "_build"} [tool.cibuildwheel.linux] -before-all = "bash packaging/install_dependencies.sh" environment = { PATH = "/nmodlwheel/flex/bin:/nmodlwheel/bison/bin:$PATH" } # the Linux wheel is not tested in cibuildwheel due to manylinux images not having # libpython*.so, so this is tested manually in the CI From c96495e78dd180d9dff865bbd93b8c165ad10779 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 5 Mar 2024 14:37:43 +0100 Subject: [PATCH 206/209] Build all wheels again --- packaging/install_dependencies.sh | 37 ------------------------------- pyproject.toml | 1 - 2 files changed, 38 deletions(-) delete mode 100644 packaging/install_dependencies.sh diff --git a/packaging/install_dependencies.sh b/packaging/install_dependencies.sh deleted file mode 100644 index 3c98d18b4c..0000000000 --- a/packaging/install_dependencies.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash - -# TODO put all of this inside of a Docker image to save time - -set -xe - -yum -y install \ - git \ - wget \ - make \ - vim \ - curl \ - unzip \ - autoconf \ - automake \ - make \ - openssh-server \ - libtool \ - && yum -y clean all \ - && rm -rf /var/cache - -curl -L -o flex-2.6.4.tar.gz https://github.com/westes/flex/files/981163/flex-2.6.4.tar.gz \ - && tar -xvzf flex-2.6.4.tar.gz \ - && cd flex-2.6.4 \ - && ./configure --prefix=/nmodlwheel/flex \ - && make -j 3 install \ - && cd .. \ - && rm -rf flex-2.6.4.tar.gz flex-2.6.4 -curl -L -o bison-3.7.3.tar.gz https://ftp.gnu.org/gnu/bison/bison-3.7.3.tar.gz \ - && tar -xvzf bison-3.7.3.tar.gz \ - && cd bison-3.7.3 \ - && ./configure --prefix=/nmodlwheel/bison \ - && make -j 3 install \ - && cd .. \ - && rm -rf bison-3.7.3.tar.gz bison-3.7.3 - -set +xe diff --git a/pyproject.toml b/pyproject.toml index dab22c84a1..fd8454c6ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,6 @@ test-extras = ["test"] test-command = [ "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] -build = ["cp311*"] manylinux-x86_64-image = "docker.io/bluebrain/nmodl:wheel-py12" [tool.cibuildwheel.macos] From 800714aa2b6889eda85e868e94e54e4e2314089e Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Tue, 5 Mar 2024 15:37:52 +0100 Subject: [PATCH 207/209] Use primary Docker image when building Linux wheels --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fd8454c6ee..2ad156088d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,7 @@ test-extras = ["test"] test-command = [ "bash {package}/packaging/test_wheel.bash python3 {wheel} false", ] -manylinux-x86_64-image = "docker.io/bluebrain/nmodl:wheel-py12" +manylinux-x86_64-image = "docker.io/bluebrain/nmodl:wheel" [tool.cibuildwheel.macos] environment = { PATH = "/opt/homebrew/opt/flex/bin:/opt/homebrew/opt/bison/bin:/usr/local/opt/flex/bin:/usr/local/opt/bison/bin:$PATH", MACOSX_DEPLOYMENT_TARGET = "10.15" } From d4c8d35fca37c77ed1e6071e6b53be9d80972f75 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 6 Mar 2024 09:22:07 +0100 Subject: [PATCH 208/209] Remove useless echo in CI --- azure-pipelines.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8a2dfc75fa..aba410351d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -215,7 +215,6 @@ stages: # change the name accordingly python3 packaging/change_name.py pyproject.toml "NMODL${TAG}" export SETUPTOOLS_SCM_PRETEND_VERSION="$(git describe --tags | cut -d '-' -f 1,2 | tr - .)" - echo "$SETUPTOOLS_SCM_PRETEND_VERSION" CIBW_ENVIRONMENT_PASS_LINUX='SETUPTOOLS_SCM_PRETEND_VERSION' python3 -m cibuildwheel --output-dir wheelhouse condition: succeeded() displayName: 'Building ManyLinux Wheels' From 08d67fa36a32f80ce19feb157fac23e978f6f716 Mon Sep 17 00:00:00 2001 From: Goran Jelic-Cizmek Date: Wed, 6 Mar 2024 09:38:37 +0100 Subject: [PATCH 209/209] Update docs --- INSTALL.rst | 2 ++ packaging/README.rst | 72 +++++++++++++++++++++++--------------------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index 9c4fc6af46..65bf1e2ca8 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -180,6 +180,8 @@ libpywrapper two environment variables need to be present: building without linking against libpython we must set ``NMODL_PYLIB`` before running cmake! +.. _testing-installed-module: + Testing the Installed Module ---------------------------- diff --git a/packaging/README.rst b/packaging/README.rst index eb2d8ddf7e..f4cc83cff0 100644 --- a/packaging/README.rst +++ b/packaging/README.rst @@ -4,8 +4,8 @@ Building Python Wheels Note: This is only slightly adapted from NEURONs `scripts `__. -NMODL wheels are built in a manylinux2010 image. Since the generic -docker image is very basic (CentOS 6) a new image, which brings updated +NMODL wheels are built in a manylinux2014 image. Since the generic +docker image is very basic (CentOS 7), a new image, which brings updated cmake3 (3.12), flex and bison was prepared and made available at https://hub.docker.com/r/bluebrain/nmodl (tag: wheel). @@ -18,7 +18,7 @@ Linux `here `__ and on OS X `here `__. On Ubuntu system we typically do: -:: +.. code::sh sudo apt install docker.io sudo groupadd docker @@ -26,56 +26,58 @@ Ubuntu system we typically do: Logout and log back in to have docker service properly configured. -Pull and start the docker image -------------------------------- +Launch the wheel building +------------------------- -We mount local neuron repository inside docker as a volume to preserve -any code changed. We can use -v option to mount the local folder as: +For building the wheel we use the ``cibuilwwheel`` utility, which can be installed using: -:: +.. code::sh - docker run -v /home/user/nmodl:/root/nmodl -it bluebrain/nmodl:wheel bash + pip install cibuildwheel -where ``/home/user/nmodl`` is the NMODL repository on the host machine. -We mount this directory inside docker at location ``/root/nmodl`` inside -the container. +Then to build a wheel for the current platform, run: -Note that for OS X there is no docker image but on a system where all -dependencies exist, you have to perform next building step. +.. code::sh -Launch the wheel building -------------------------- + cibuildwheel -Once we are inside docker container, we can start building wheels. There -is a build script that loops over the pythons ``>=3.5`` in -``/opt/python``, build and audit the generated wheels. Results are -placed in this wheelhouse directory. +If you have Docker installed, you can also build the Linux wheels using: -:: +.. code::sh - cd /root/nmodl - bash packaging/python/build_wheels.bash linux + cibuildwheel --platform linux -For OSX on a system with the all dependencies you have to clone the -NMODL repository and do: +Note that, if you happen to have Podman installed instead of Docker, you can +set ``CIBW_CONTAINER_ENGINE=podman`` to use Podman instead of Docker for this +task. -:: +Furthermore, in order to build wheels on MacOS, you need to install an official +CPython installer from `python.org `__. - cd nrn - bash packaging/python/build_wheels.bash osx +For a complete list of all available customization options for +``cibuildwheel``, please consult the +`documentation `__. -Updating neuron_wheel docker image ----------------------------------- +Testing the wheel +----------------- + +On MacOS, the testing of the wheel is launched automatically when running +``cibuildwheel``. On Linux, you will need to test the wheel manually, please +see :ref:`testing-installed-module` for the instructions. + + +Updating the NMODL Docker image +------------------------------- -If you have changed Dockerfile, you can build the new image as: +If you have changed the Dockerfile, you can build the new image as: -:: +.. code::sh docker build -t bluebrain/nmodl:tag . -and then push image to hub.docker.com as: +and then push the image to hub.docker.com as: -:: +.. code::sh docker login --username= - docker push bluebrain/nmodl:tag + docker push bluebrain/nmodl:tag