build: automated versioning with setuptools_scm and improved workflows (

#182) * ci: upgrade deploy and test ci * build: use `setuptools_scm` for dynamic versioning * build: update package versioning * build: use post-release versioning * docs: update email * fix: update java, use `uv` and point to tests dir * build: allow future pandas `stack` * docs: show docs also for older versions * chore: remove `requirements.yaml` * fix: clean up * test: skip url_retrieve test in gh runner * chore: set gh env
PyPSA · Jul 31, 2024 · d78c885 · d78c885
1 parent b947fde
commit d78c885
Show file tree

Hide file tree

Showing 13 changed files with 223 additions and 173 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
diff --git a/.github/workflows/deploy.yml → .github/workflows/release.yml b/.github/workflows/deploy.yml → .github/workflows/release.yml
@@ -1,36 +1,46 @@
-name: Deploy release
+name: Release
 
 on:
   push:
     tags:
-    - v*.*.*
+      - v*.*.*
 
 jobs:
-  build-n-publish:
-    name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI
+  build:
+    # Build the Python SDist and wheel, performs metadata and readme linting
+    name: Build and verify package
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hynek/build-and-inspect-python-package@v2
 
+  release:
+    # Publish a GitHub release from the given git tag
+    name: Create GitHub Release
+    needs: [build]
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v5
-      with:
-        python-version: 3.9
+      - uses: actions/checkout@v4
+      - uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true
 
-    - name: Install pypa/build
-      run: >-
-        python -m
-        pip install
-        build
-        --user
-    - name: Build a binary wheel and a source tarball
-      run: >-
-        python -m
-        build
-        --sdist
-        --wheel
-        --outdir dist/
-        .
+  publish:
+    # Publish the built SDist and wheel from "dist" job to PyPI
+    name: Publish to PyPI
+    needs: [build]
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/powerplantmatching/
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: Packages
+          path: dist
+      - uses: pypa/gh-action-pypi-publish@release/v1
 
   update-dataset:
     name: Update powerplants.csv in repository
@@ -70,4 +80,4 @@ jobs:
         git commit -m '[github-actions.ci] auto update `powerplants.csv`' || exit 0 
         git push origin master
       env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,77 @@
+name: Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ '*' ]
+  schedule:
+  - cron: "0 5 * * TUE"
+
+# Cancel any in-progress runs when a new run is triggered
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    # Build the Python SDist and wheel, performs metadata and readme linting
+    name: Build and verify package
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Needed for setuptools_scm
+      - uses: hynek/build-and-inspect-python-package@v2
+        id: baipp
+
+    outputs:
+      python-versions: ${{ steps.baipp.outputs.supported_python_classifiers_json_array }}
+
+  test:
+    # Test package build in matrix of OS and Python versions
+    name: Test package
+    needs: [build]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ${{ fromJSON(needs.build.outputs.python-versions) }}
+        os:
+        - ubuntu-latest
+        - macos-latest
+        - windows-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0 # Needed for setuptools_scm
+
+    - name: Set up Python ${{ matrix.python-version }} on ${{ matrix.os }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Set up Java
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: 22
+
+    - name: Download package
+      uses: actions/download-artifact@v4
+      with:
+        name: Packages
+        path: dist
+
+    - name: Install package and dependencies
+      run: |
+        python -m pip install uv
+        uv pip install --system "$(ls dist/*.whl)[dev]"
+
+    - name: Test with pytest
+      run: |
+        pytest
+      env:
+        GITHUB_ACTIONS: true
+
diff --git a/doc/conf.py b/doc/conf.py
@@ -4,6 +4,8 @@
 # list see the documentation:
 # https://www.sphinx-doc.org/en/master/usage/configuration.html
 
+from importlib.metadata import version as get_version
+
 # -- Path setup --------------------------------------------------------------
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -14,13 +16,17 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
-
 # -- Project information -----------------------------------------------------
 
 project = "powerplantmatching"
 copyright = "2021-2024, Fabian Hofmann, Fabian Gotzens, Jonas Hörsch, Martha Frysztacki"
 author = "Fabian Hofmann, Fabian Gotzens, Jonas Hörsch, Martha Frysztacki"
 
+# -- Version information -------------------------------------------------
+
+# The short X.Y version.
+release: str = get_version("powerplantmatching")
+version: str = ".".join(release.split(".")[:2])
 
 # -- General configuration ---------------------------------------------------
 

diff --git a/powerplantmatching/__init__.py b/powerplantmatching/__init__.py
@@ -21,17 +21,26 @@
 power plant databases.
 """
 
-__version__ = "0.5.15"
-__author__ = "Fabian Hofmann"
-__copyright__ = "Copyright 2017-2024 Technical University of Berlin"
-# The rough hierarchy of this package is
-# core, utils, heuristics, cleaning, matching, collection, data
+from importlib.metadata import version
 
 from . import core, data, heuristics, plot, utils
 from .accessor import PowerPlantAccessor
 from .collection import powerplants
 from .core import get_config, package_config
 
+__author__ = "Fabian Hofmann"
+__copyright__ = "Copyright 2017-2024 Technical University of Berlin"
+# The rough hierarchy of this package is
+# core, utils, heuristics, cleaning, matching, collection, data
+
+# e.g. "0.5.15" or "0.5.15.post27+g761e814.d20240722" (if installed from master branch)
+__version__ = version("powerplantmatching")
+# e.g. "0.5.15", without the post part (if it exists, otherwise the same as __version__)
+latest_release = __version__.split(".post")[0]
+
+assert latest_release != "0.1", "setuptools_scm could not find the version number"
+
+
 __all__ = [
     "powerplants",
     "get_config",

diff --git a/powerplantmatching/collection.py b/powerplantmatching/collection.py
@@ -169,7 +169,7 @@ def powerplants(
             Arguments passed to powerplantmatching.collection.Collection.
 
     """
-    from . import __version__
+    from . import latest_release
 
     if config is None:
         if config_update is None:
@@ -200,7 +200,7 @@ def powerplants(
 
     if from_url:
         fn = _data_out("matched_data_red.csv", config)
-        url = config["matched_data_url"].format(tag="v" + __version__)
+        url = config["matched_data_url"].format(tag="v" + latest_release)
         logger.info(f"Retrieving data from {url}")
         df = (
             pd.read_csv(url, index_col=0)
@@ -247,7 +247,9 @@ def powerplants(
             matched = matched[matched.lat.notnull()]
 
     if isinstance(matched.columns, pd.MultiIndex):
-        matched.stack().drop_duplicates(["Name", "Fueltype", "Country"]).unstack(-1)
+        matched.stack(future_stack=True).drop_duplicates(
+            ["Name", "Fueltype", "Country"]
+        ).unstack(-1)
     else:
         matched.drop_duplicates(["Name", "Fueltype", "Country"])
 

diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py
@@ -307,7 +307,7 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None):
     sdf = (
         df.assign(Set=lambda df: df.Set.where(df.Set != "PP"))
         .assign(Fueltype=lambda df: df.Fueltype.where(df.Set != "Other"))
-        .stack(1)
+        .stack(1, future_stack=True)
         .reindex(rel_scores.index, level=1)
         .groupby(level=0)
         .agg(props_for_groups)

diff --git a/powerplantmatching/utils.py b/powerplantmatching/utils.py
@@ -641,7 +641,7 @@ def fill_geoposition(
         if isinstance(df.columns, pd.MultiIndex):
             new_data = (
                 df.drop(columns=["lat", "lon"])
-                .stack()
+                .stack(future_stack=True)
                 .join(locs, on=["Name", "Country"])
                 .unstack(-1)
                 .reindex(columns=df.columns)

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,7 +1,76 @@
-# Formater and linter settings
+[build-system]
+requires = ["setuptools>=64", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name="powerplantmatching"
+dynamic = ["version"]
+description="Toolset for generating and managing Power Plant Data"
+readme="README.md"
+authors = [{ name = "Fabian Hofmann (FIAS)", email = "[email protected]" },
+{ name = "Jonas Hoersch (KIT)" },
+{ name = "Fabian Gotzens (FZ Jülich)" }]
+license = { file = "LICENSE" }
+classifiers=[
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Environment :: Console",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+    "Natural Language :: English",
+    "Operating System :: OS Independent",
+]
+
+requires-python = ">=3.9"
+
+dependencies = [
+    "numpy",
+    "scipy",
+    "pandas>=0.24.0",
+    "networkx>=1.10",
+    "pycountry",
+    "country_converter",
+    "xlrd",
+    "seaborn",
+    "pyyaml >=5.1.0",
+    "requests",
+    "matplotlib",
+    "geopy",
+    "xlrd",
+    "unidecode",
+    "entsoe-py >=0.3.1",
+    "deprecation",
+    "tqdm",
+    "openpyxl",
+]
+
+[project.urls]
+Homepage = "https://github.com/PyPSA/powerplantmatching"
+Source = "https://github.com/PyPSA/powerplantmatching"
+
+[project.optional-dependencies]
+docs= [
+    "numpydoc",
+    "sphinx",
+    "sphinx-book-theme",
+    "nbsphinx",
+    "sphinx-automodapi",
+]
+dev= ["pre-commit", "pytest", "pytest-cov"]
+
+ # setuptools_scm settings
+
+[tool.setuptools_scm]
+version_scheme = "post-release"
+
+[tool.setuptools.packages.find]
+include = ["powerplantmatching"]
+
+# Formatter and linter settings
 
 [tool.ruff]
-line-length = 88
 extend-include = ['*.ipynb']
 
 [tool.ruff.lint]
@@ -21,4 +90,4 @@ select = [
 [tool.pytest.ini_options]
 filterwarnings = [
     "error::FutureWarning",      # Raise all FutureWarnings as errors
-]
+]