Merge branch 'develop'

.github/workflows/python-publish.yml

@@ -49,38 +49,6 @@ jobs:
     - name: Tests
       run: pytest
 
-  # notify-failure:
-  #   if: failure()
-  #   runs-on: ubuntu-latest
-  #   needs: [quality, checks]
-  #   name: Notify failure
-  #   steps:
-  #     - uses: jdcargile/[email protected]
-  #       with:
-  #         github-token: ${{ github.token }}
-  #         ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI_F }}
-  #         # notification-summary: ${{ steps.qa.outputs.status }}
-  #         notification-summary: ❌ Build failed on anemoi.datasets!
-  #         notification-color: dc3545
-  #         timezone: Europe/Paris
-  #         verbose-logging: true
-
-  # notify-success:
-  #   if: success()
-  #   runs-on: ubuntu-latest
-  #   needs: [quality, checks]
-  #   name: Notify success
-  #   steps:
-  #     - uses: jdcargile/[email protected]
-  #       with:
-  #         github-token: ${{ github.token }}
-  #         ms-teams-webhook-uri: ${{ secrets.MS_TEAMS_WEBHOOK_URI_F }}
-  #         # notification-summary: ${{ steps.qa.outputs.status }}
-  #         notification-summary: ✅ New commit on anemoi.datasets
-  #         notification-color: 17a2b8
-  #         timezone: Europe/Paris
-  #         verbose-logging: true
-
   deploy:
 
     if: ${{ github.event_name == 'release' }}
@@ -93,24 +61,16 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.10'
-
-    - name: Check that tag version matches code version
-      run: |
-        tag=${GITHUB_REF#refs/tags/}
-        version=$(python setup.py --version)
-        echo 'tag='$tag
-        echo "version file="$version
-        test "$tag" == "$version"
+        python-version: 3.x
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+        pip install build wheel twine
     - name: Build and publish
       env:
         TWINE_USERNAME: __token__
         TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
       run: |
-        python setup.py sdist
+        python -m build
         twine upload dist/*

.gitignore

@@ -159,6 +159,8 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
+_version.py
+
 *.grib
 *.onnx
 *.ckpt
@@ -184,3 +186,4 @@ _build/
 ?.*
 ~*
 *.sync
+*.dot

.pre-commit-config.yaml

@@ -51,23 +51,6 @@ repos:
     - --exit-non-zero-on-fix
     - --preview
 
-# - repo: https://github.com/thclark/pre-commit-sphinx
-#   rev: 0.0.1
-#   hooks:
-#     - id: build-docs
-#       additional_dependencies:
-#         - sphinx
-#         - sphinx_rtd_theme
-#         - nbsphinx
-#         - pandoc
-#       args:
-#         - --cache-dir
-#         - docs/_build/doctrees
-#         - --html-dir
-#         - docs/_build/html
-#         - --source-dir
-#         - docs
-#       language_version: python3
 
 - repo: https://github.com/sphinx-contrib/sphinx-lint
   rev: v0.9.1
@@ -80,8 +63,8 @@ repos:
   hooks:
     - id: rstfmt
 
-# - repo: https://github.com/b8raoult/pre-commit-docconvert
-#   rev: "0.1.0"
-#   hooks:
-#   - id: docconvert
-#     args: ["-o", "numpy"]
+- repo: https://github.com/b8raoult/pre-commit-docconvert
+  rev: "0.1.4"
+  hooks:
+  - id: docconvert
+    args: ["numpy"]

.readthedocs.yaml

@@ -3,11 +3,15 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.10"
+    python: "3.11"
 
 sphinx:
   configuration: docs/conf.py
 
 python:
    install:
-     - requirements: docs/requirements.txt
+    - requirements: docs/requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

docs/building/handling-missing-dates.rst

@@ -2,6 +2,8 @@
  Handling missing dates
 ########################
 
+By default, the package will raise an error if there are missing dates.
+
 Missing dates can be handled by specifying a list of dates in the
 configuration file. The dates should be in the same format as the dates
 in the time series. The missing dates will be filled ``np.nan`` values.

docs/conf.py

@@ -12,7 +12,7 @@
 #
 # import os
 # import sys
-# sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.join(os.path.abspath('.'), 'src'))
 
 import datetime
 import os

pyproject.toml

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
+
+[build-system]
+requires = ["setuptools>=60", "setuptools-scm>=8.0"]
+
+[project]
+description = "A package to hold various functions to support training of ML models on ECMWF data."
+name = "anemoi-dataset"
+
+dynamic = ["version"]
+license = { file = "LICENSE" }
+requires-python = ">=3.9"
+
+authors = [
+    { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "[email protected]" },
+]
+
+keywords = ["tools", "datasets", "ai"]
+
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+    "anemoi-utils[provenance]>=0.1.7",
+    "zarr",
+    "pyyaml",
+    "numpy",
+    "tqdm",
+    "semantic-version",
+]
+
+[project.optional-dependencies]
+remote = ["boto3", "requests", "s3fs"]
+
+create = [
+    "climetlab>=0.22.1",        # "earthkit-data"
+    "earthkit-meteo",
+    "pyproj",
+    "ecmwflibs>=0.6.3",
+]
+
+docs = ["sphinx", "sphinx_rtd_theme", "nbsphinx", "pandoc"]
+
+all = [
+    "boto3",
+    "requests",
+    "s3fs",
+    "climetlab>=0.22.1",        # "earthkit-data"
+    "earthkit-meteo",
+    "pyproj",
+    "ecmwflibs>=0.6.3",
+]
+
+dev = [
+    "boto3",
+    "requests",
+    "s3fs",
+    "climetlab>=0.22.1",        # "earthkit-data"
+    "earthkit-meteo",
+    "pyproj",
+    "ecmwflibs>=0.6.3",
+    "sphinx",
+    "sphinx_rtd_theme",
+    "nbsphinx",
+    "pandoc",
+]
+
+[project.urls]
+Homepage = "https://github.com/ecmwf/anemoi-datasets/"
+Documentation = "https://anemoi-datasets.readthedocs.io/"
+Repository = "https://github.com/ecmwf/anemoi-datasets/"
+Issues = "https://github.com/ecmwf/anemoi-datasets/issues"
+# Changelog = "https://github.com/ecmwf/anemoi-datasets/CHANGELOG.md"
+
+[project.scripts]
+anemoi-datasets = "anemoi.datasets.__main__:main"
+
+[tool.setuptools_scm]
+version_file = "src/anemoi/datasets/_version.py"
+
+[tool.setuptools.package-data]
+"anemoi.datasets.data" = ["*.css"]