Merge branch 'main' into initial-volume-enum

.github/workflows/python.yml

@@ -1,61 +1,247 @@
-name: Python (Linux)
+# Python workflow for building and testing wheels.
+#
+# TODO: Ideally the runtime dependencies would be installed by pip from pyproject.toml
+# However, this is not currently possible to do with pip: https://github.com/pypa/pip/issues/11440
+
+name: Python
 
 on:
   push:
-    branches: [ main ]
   pull_request:
 
+jobs:
 
-env:
-  CARGO_TERM_COLOR: always
+# Mac OS X does not work with the HDF5 crate for some reason.
+#  macos:
+#    runs-on: macos-latest
+#    strategy:
+#      matrix:
+#        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
+#    steps:
+#      - uses: actions/checkout@v4
+#        with:
+#          submodules: true
+#      - uses: actions/setup-python@v5
+#        with:
+#          python-version: ${{ matrix.python-version }}
+#          architecture: x64
+#      - uses: dtolnay/rust-toolchain@stable
+#      - name: Build wheels - x86_64
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: x86_64
+#          args: --release --out dist --sdist -m pywr-python/Cargo.toml
+#      - name: Install built wheel - x86_64
+#        run: |
+#          pip install pytest h5py pandas polars pyarrow click
+#          pip install pywr --no-index --find-links dist --force-reinstall --no-deps
+#          python -c "import pywr"
+#          python -m pytest
+#      - name: Build wheels - universal2
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: universal2-apple-darwin
+#          args: -i ${{ matrix.python-version }} --release --out dist -m pywr-python/Cargo.toml
+#      - name: Install built wheel - universal2
+#        run: |
+#          pip install pytest h5py pandas polars pyarrow click
+#          pip install pywr --no-index --find-links dist --force-reinstall --no-deps
+#          python -c "import pywr"
+#          python -m pytest
+#      - name: Upload wheels
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: wheels
+#          path: dist
 
-jobs:
-  lint:
-    runs-on: ubuntu-latest
+  windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: true
-      - name: Install HDF5
-        run: |
-          sudo apt-get update
-          sudo apt-get install libhdf5-dev ocl-icd-opencl-dev zlib1g-dev
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - uses: dtolnay/rust-toolchain@stable
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
         with:
-          python-version: "3.x"
-      - name: Install Dependencies
+          target: ${{ matrix.target }}
+          args: -i ${{ matrix.python-version }} --release --out dist -m pywr-python/Cargo.toml
+      - name: Install built wheel
         run: |
-          pip install pre-commit
-          pre-commit install-hooks
-      - name: Lint with pre-commit
-        run: pre-commit run --all-files
-
-  test:
+          pip install pytest h5py pandas polars pyarrow click
+          pip install pywr --no-index --find-links dist --force-reinstall --no-deps
+          python -c "import pywr"
+          python -m pytest
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
 
+  linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
-
+        python-version: [ '3.9', '3.10', '3.11', '3.12' ]
     steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-    - name: Install HDF5
-      run: |
-        sudo apt-get update
-        sudo apt-get install libhdf5-dev ocl-icd-opencl-dev liblzma-dev zlib1g-dev
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Build and test
-      run: |
-        python -m venv venv
-        source venv/bin/activate
-        python -m pip install maturin pytest pytest-benchmark pandas polars
-        maturin build --interpreter python${{ matrix.python-version }} --manylinux off
-        ls -la target/wheels
-        python -m pip install target/wheels/pywr*.whl
-        pytest tests
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: auto
+          args: -i ${{ matrix.python-version }} --release --out dist -m pywr-python/Cargo.toml
+      - name: Install built wheel
+        run: |
+          pip install pytest h5py pandas polars pyarrow click
+          pip install pywr --no-index --find-links dist --force-reinstall --no-deps
+          python -c "import pywr"
+          python -m pytest
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+#  linux-cross:
+#    runs-on: ubuntu-latest
+#    strategy:
+#      matrix:
+#        target: [aarch64, armv7, s390x, ppc64le, ppc64]
+#    steps:
+#      - uses: actions/checkout@v4
+#      - uses: actions/setup-python@v4
+#        with:
+#          python-version: 3.9
+#      - name: Build wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.target }}
+#          manylinux: auto
+#          args: --release --out dist -m python/Cargo.toml
+#      - uses: uraimo/[email protected]
+#        if: matrix.target != 'ppc64'
+#        name: Install built wheel
+#        with:
+#          arch: ${{ matrix.target }}
+#          distro: ubuntu20.04
+#          githubToken: ${{ github.token }}
+#          install: |
+#            apt-get update
+#            apt-get install -y --no-install-recommends python3 python3-pip
+#            pip3 install -U pip
+#          run: |
+#            pip3 install crfs --no-index --find-links dist/ --force-reinstall
+#            python3 -c "import crfs"
+#      - name: Upload wheels
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: wheels
+#          path: dist
+
+#  musllinux:
+#    runs-on: ubuntu-latest
+#    strategy:
+#      matrix:
+#        target:
+#          - x86_64-unknown-linux-musl
+#          - i686-unknown-linux-musl
+#    steps:
+#      - uses: actions/checkout@v4
+#      - uses: actions/setup-python@v4
+#        with:
+#          python-version: 3.9
+#          architecture: x64
+#      - name: Build wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.target }}
+#          manylinux: musllinux_1_2
+#          args: --release --out dist -m python/Cargo.toml
+#      - name: Install built wheel
+#        if: matrix.target == 'x86_64-unknown-linux-musl'
+#        uses: addnab/docker-run-action@v3
+#        with:
+#          image: alpine:latest
+#          options: -v ${{ github.workspace }}:/io -w /io
+#          run: |
+#            apk add py3-pip
+#            pip3 install -U pip
+#            pip3 install crfs --no-index --find-links /io/dist/ --force-reinstall
+#            python3 -c "import crfs"
+#      - name: Upload wheels
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: wheels
+#          path: dist
+#
+#  musllinux-cross:
+#    runs-on: ubuntu-latest
+#    strategy:
+#      matrix:
+#        platform:
+#          - target: aarch64-unknown-linux-musl
+#            arch: aarch64
+#          - target: armv7-unknown-linux-musleabihf
+#            arch: armv7
+#    steps:
+#      - uses: actions/checkout@v4
+#      - uses: actions/setup-python@v4
+#        with:
+#          python-version: 3.9
+#      - name: Build wheels
+#        uses: PyO3/maturin-action@v1
+#        with:
+#          target: ${{ matrix.platform.target }}
+#          manylinux: musllinux_1_2
+#          args: --release --out dist -m python/Cargo.toml
+#      - uses: uraimo/[email protected]
+#        name: Install built wheel
+#        with:
+#          arch: ${{ matrix.platform.arch }}
+#          distro: alpine_latest
+#          githubToken: ${{ github.token }}
+#          install: |
+#            apk add py3-pip
+#            pip3 install -U pip
+#          run: |
+#            pip3 install crfs --no-index --find-links dist/ --force-reinstall
+#            python3 -c "import crfs"
+#      - name: Upload wheels
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: wheels
+#          path: dist
+#
+#  release:
+#    name: Release
+#    runs-on: ubuntu-latest
+#    if: "startsWith(github.ref, 'refs/tags/')"
+#    needs: [ macos, windows, linux, linux-cross, musllinux, musllinux-cross ]
+#    steps:
+#      - uses: actions/download-artifact@v3
+#        with:
+#          name: wheels
+#      - uses: actions/setup-python@v4
+#        with:
+#          python-version: 3.9
+#      - name: Publish to PyPI
+#        env:
+#          TWINE_USERNAME: __token__
+#          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+#        run: |
+#          pip install --upgrade twine
+#          twine upload --skip-existing *

Cargo.toml

@@ -38,9 +38,9 @@ thiserror = "1.0.25"
 time = { version = "0.3", features = ["serde", "serde-well-known", "serde-human-readable", "macros"] }
 num = "0.4.0"
 ndarray = "0.15.3"
-polars = { version = "0.36.2", features = ["lazy", "rows", "ndarray"] }
-pyo3-polars = "0.10.0"
-pyo3 = { version = "0.20.0" }
+polars = { version = "0.37.0", features = ["lazy", "rows", "ndarray"] }
+pyo3-polars = "0.11.1"
+pyo3 = { version = "0.20.2" }
 tracing = "0.1"
 csv = "1.1"
 hdf5 = { version="0.8.1" }

README.md

@@ -117,21 +117,18 @@ git submodule init
 git submodule update
 ```
 
-Rust is required for installation of the Python extension. To create a Python development installation 
-requires first compiling the Rust library and then installing the Python package in editable model.
+Rust is required for installation of the Python extension. To create a Python development installation
+requires first compiling the Rust library and then the Python extension. The following example uses
+a virtual environment to install the Python dependencies, compile the Pywr extension and run the Pywr Python CLI.
 
-```bash
-cd pywr-python
-maturin develop
-pip install -e .
-```
-
-Alternatively use the `develop.sh` script to run the above two commands.
-
-Once this is complete the following will run a simple command line application from Python.
 
 ```bash
-python -m pywr
+python -m venv .venv # create a new virtual environment
+source .venv/bin/activate # activate the virtual environment (linux)
+# .venv\Scripts\activate # activate the virtual environment (windows)
+pip install maturin  # install maturin for building the Python extension
+maturin develop # compile the Pywr Python extension
+python -m pywr  # run the Pywr Python CLI
 ```
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
@@ -145,7 +142,7 @@ python -m pywr
 ### Rust CLI
 
 A basic command line interface is included such that you can use this version of Pywr without Python.
-This CLI is in the `pywr-cli` crate. 
+This CLI is in the `pywr-cli` crate.
 
 To see the CLI commands available run the following:
 

pyproject.toml

@@ -0,0 +1,26 @@
+[project]
+name = "pywr"
+version = "2.0.0beta"
+description = ""
+authors = [
+    {name="James Tomlinson", email="[email protected]>"}
+]
+readme = "README.md"
+requires-python = ">=3.9"
+license = "MIT OR Apache-2.0"
+dependencies = [
+  "pandas",
+  "polars",
+  "pyarrow",
+  "click"
+]
+
+[build-system]
+requires = ["maturin>=1.0,<2.0"]
+build-backend = "maturin"
+
+[tool.maturin]
+manifest-path = "pywr-python/Cargo.toml"
+python-source = "pywr-python"
+# "extension-module" tells pyo3 we want to build an extension module (skips linking against libpython.so)
+features = ["pyo3/extension-module"]

pywr-python/Cargo.toml

@@ -18,6 +18,8 @@ pyo3 = {  workspace = true }
 pyo3-polars = {  workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+time = { workspace = true, features = ["serde", "serde-well-known", "serde-human-readable", "macros"] }
+
 pywr-core = { path="../pywr-core" }
 pywr-schema = { path="../pywr-schema" }