diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45b5cef..784c6ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,9 +8,15 @@ on: - 'develop' tags-ignore: - '**' + paths-ignore: + - "docs/**" + - "README.md" # Trigger the workflow on pull request pull_request: + paths-ignore: + - "docs/**" + - "README.md" # Trigger the workflow manually workflow_dispatch: @@ -18,6 +24,9 @@ on: # Trigger after public PR approved for CI pull_request_target: types: [labeled] + paths-ignore: + - "docs/**" + - "README.md" jobs: # Run CI including downstream packages on self-hosted runners @@ -28,7 +37,7 @@ jobs: with: earthkit: ecmwf/earthkit@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true - python_qa: true + python_qa: false secrets: inherit diff --git a/.github/workflows/legacy-ci.yml b/.github/workflows/legacy-ci.yml index bcefba1..68d70a8 100644 --- a/.github/workflows/legacy-ci.yml +++ b/.github/workflows/legacy-ci.yml @@ -24,43 +24,6 @@ defaults: shell: bash -l {0} jobs: - pre-commit: - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - uses: actions/setup-python@v4 - with: - python-version: 3.x - - uses: pre-commit/action@v3.0.0 - - unit-tests: - name: unit-tests (3.10) - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - name: Install Conda environment with Micromamba - uses: mamba-org/provision-with-micromamba@v14 - with: - environment-file: tests/environment-unit-tests.yml - environment-name: DEVELOP - channels: conda-forge - cache-env: true - extra-specs: | - python=3.10 - - name: Install package - run: | - python -m pip install . --no-deps - - name: Run tests - run: | - make unit-tests - documentation: if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} runs-on: ubuntu-latest @@ -73,73 +36,18 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.ref }} - name: Install Conda environment with Micromamba - uses: mamba-org/provision-with-micromamba@v12 + uses: mamba-org/setup-micromamba@v1 with: environment-file: tests/environment-docs-tests.yml environment-name: DEVELOP channels: conda-forge - cache-env: true - cache-env-key: ubuntu-latest-3.10 - extra-specs: | + cache-environment: true + cache-env-key: ubuntu-latest-3.10-no-eccodes + create-args: >- python=3.10 - name: Install package run: | - python -m pip install --no-deps . + python -m pip install --no-deps -e . - name: Build documentation run: | make docs-build - - # integration-tests: - # needs: [unit-tests] - # if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - # runs-on: ubuntu-latest - # defaults: - # run: - # shell: bash -l {0} - - # strategy: - # matrix: - # include: - # - python-version: "3.10" - # # extra: -minver # This will need to be uncommented and environment-minver.yml updated if we want to publish on conda - - # steps: - # - uses: actions/checkout@v3 - # with: - # ref: ${{ github.event.pull_request.head.sha || github.ref }} - # - name: Install Conda environment with Micromamba - # uses: mamba-org/provision-with-micromamba@v12 - # with: - # environment-file: tests/environment-unit-tests${{ matrix.extra }}.yml - # environment-name: DEVELOP${{ matrix.extra }} - # channels: conda-forge - # cache-env: true - # cache-env-key: ubuntu-latest-${{ matrix.python-version }}${{ matrix.extra }}. - # extra-specs: | - # python=${{matrix.python-version }} - # - name: Install package - # run: | - # python -m pip install --no-deps . - # - name: Run tests - # run: | - # make unit-tests - - # distribution: - # needs: [integration-tests, documentation] - # if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - # runs-on: ubuntu-latest - - # steps: - # - uses: actions/checkout@v3 - # with: - # ref: ${{ github.event.pull_request.head.sha || github.ref }} - # - name: Build distributions - # run: | - # $CONDA/bin/python -m pip install build - # $CONDA/bin/python -m build - # - name: Publish a Python distribution to PyPI - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') - # uses: pypa/gh-action-pypi-publish@release/v1 - # with: - # user: __token__ - # password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/python-pull-request.yml b/.github/workflows/python-pull-request.yml new file mode 100644 index 0000000..2c38063 --- /dev/null +++ b/.github/workflows/python-pull-request.yml @@ -0,0 +1,12 @@ +name: Code Quality checks for PRs + +on: + push: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + quality: + uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2 + with: + skip-hooks: "no-commit-to-branch" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ffa512..d74808d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,32 +1,48 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer + - id: trailing-whitespace # Trailing whitespace checker + - id: end-of-file-fixer # Ensure files end in a newline - id: check-json - - id: check-yaml + - id: check-yaml # Check YAML files for syntax errors only + args: [--unsafe, --allow-multiple-documents] - id: check-toml # - id: check-added-large-files - - id: debug-statements + - id: debug-statements # Check for debugger imports and py37+ breakpoint() - id: mixed-line-ending + - id: no-commit-to-branch # Prevent committing to main / master + - id: check-merge-conflict # Check for files that contain merge conflict + exclude: /README\.rst$|^docs/.*\.rst$ - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort + args: + - -l 120 + - --force-single-line-imports + - --profile black - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 24.8.0 hooks: - id: black + args: [--line-length=120] - repo: https://github.com/keewis/blackdoc rev: v0.3.8 hooks: - id: blackdoc additional_dependencies: [black==23.3.0] -- repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 + exclude: xr_engine_profile_rst\.py +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.9 + hooks: + - id: ruff + exclude: '(dev/.*|.*_)\.py$' + args: + - --line-length=120 + - --fix + - --exit-non-zero-on-fix + - --preview - repo: https://github.com/executablebooks/mdformat rev: 0.7.14 hooks: @@ -37,11 +53,11 @@ repos: hooks: - id: pretty-format-yaml args: [--autofix, --preserve-quotes] - - id: pretty-format-toml - args: [--autofix] -- repo: https://github.com/PyCQA/pydocstyle.git - rev: 6.1.1 - hooks: - - id: pydocstyle - additional_dependencies: [toml] - exclude: tests|docs +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.0 + hooks: + - id: sphinx-lint +- repo: https://github.com/tox-dev/pyproject-fmt + rev: "2.2.4" + hooks: + - id: pyproject-fmt diff --git a/Makefile b/Makefile index df7b78e..52d27b3 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ COV_REPORT := html setup: pre-commit install -default: qa unit-tests type-check +default: qa unit-tests qa: pre-commit run --all-files @@ -14,6 +14,9 @@ qa: unit-tests: cd .. && python -m pytest earthkit -vv --cov=. --cov-report=$(COV_REPORT) && cd - +type-check: + python -m mypy . + conda-env-update: $(CONDA) env update $(CONDAFLAGS) -f environment.yml diff --git a/docs/examples/01_grib_file.ipynb b/docs/examples/01_grib_file.ipynb index ad4015c..e01c0df 100644 --- a/docs/examples/01_grib_file.ipynb +++ b/docs/examples/01_grib_file.ipynb @@ -44,7 +44,6 @@ "source": [ "import io\n", "import matplotlib.pyplot as plt\n", - "import numpy as np\n", "from earthkit import data, regrid\n", "\n", "fs = data.from_source(\"file\", \"tz_1000.grib\")" diff --git a/docs/examples/02_grib_fdb.ipynb b/docs/examples/02_grib_fdb.ipynb index f1ae078..3767577 100644 --- a/docs/examples/02_grib_fdb.ipynb +++ b/docs/examples/02_grib_fdb.ipynb @@ -35,7 +35,6 @@ "source": [ "import io\n", "import matplotlib.pyplot as plt\n", - "import numpy as np\n", "import pyfdb\n", "from earthkit import data, regrid" ] diff --git a/docs/examples/03_polytope_timeseries.ipynb b/docs/examples/03_polytope_timeseries.ipynb new file mode 100644 index 0000000..c8c1eed --- /dev/null +++ b/docs/examples/03_polytope_timeseries.ipynb @@ -0,0 +1,7377 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-09-19 15:19:23 - INFO - Key read from /home/maaw/.polytopeapirc\n", + "2024-09-19 15:19:23 - INFO - Sending request...\n", + "{'request': 'class: od\\n'\n", + " \"date: '20240917'\\n\"\n", + " 'domain: g\\n'\n", + " 'expver: 0079\\n'\n", + " 'feature:\\n'\n", + " ' axis: step\\n'\n", + " ' points:\\n'\n", + " ' - - -9.109280931080349\\n'\n", + " ' - 38.78655345978706\\n'\n", + " ' type: timeseries\\n'\n", + " 'levtype: sfc\\n'\n", + " 'number: 1/to/50\\n'\n", + " 'param: 164/167/169\\n'\n", + " 'step: 0/to/360\\n'\n", + " 'stream: enfo\\n'\n", + " \"time: '0000'\\n\"\n", + " 'type: pf\\n',\n", + " 'verb': 'retrieve'}\n", + "2024-09-19 15:19:23 - INFO - Polytope user key found in session cache for user maaw\n", + "2024-09-19 15:19:24 - INFO - Request accepted. Please poll ../requests/93457f02-9fd0-46f7-8a42-0f80bfd80af9 for status\n", + "2024-09-19 15:19:24 - INFO - Polytope user key found in session cache for user maaw\n", + "2024-09-19 15:19:24 - INFO - Checking request status (../requests/93457f02-9fd0-46f7-8a42-0f80bfd80af9)...\n", + "2024-09-19 15:19:24 - INFO - The current status of the request is 'queued'\n", + "2024-09-19 15:19:25 - INFO - The current status of the request is 'processing'\n", + "2024-09-19 15:20:16 - INFO - The current status of the request is 'processed'\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "4016a9b2bc9643a0983e31eeeda091a7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "93457f02-9fd0-46f7-8a42-0f80bfd80af9.grib: 0%| | 0.00/547k [00:00, ?B/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import earthkit.data\n", + "\n", + "request = {\n", + " \"class\": \"od\",\n", + " \"stream\" : \"enfo\",\n", + " \"type\" : \"pf\",\n", + " \"date\" : \"20240917\",\n", + " \"time\" : \"0000\",\n", + " \"levtype\" : \"sfc\",\n", + " \"expver\" : \"0079\", \n", + " \"domain\" : \"g\",\n", + " \"param\" : \"164/167/169\",\n", + " \"number\" : \"1/to/50\",\n", + " \"step\": \"0/to/360\",\n", + " \"feature\" : {\n", + " \"type\" : \"timeseries\",\n", + " \"points\": [[-9.109280931080349, 38.78655345978706]],\n", + " \"axis\": \"step\",\n", + " },\n", + "}\n", + "\n", + "ds = earthkit.data.from_source(\"polytope\", \"ecmwf-mars\", request, stream=False, address='polytope.ecmwf.int')" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
<xarray.Dataset>\n", + "Dimensions: (x: 1, y: 1, z: 1, number: 50, datetime: 1, t: 145)\n", + "Coordinates:\n", + " * x (x) float64 -9.104\n", + " * y (y) float64 38.81\n", + " * z (z) int64 0\n", + " * number (number) int64 1 2 3 4 5 6 7 8 9 10 ... 42 43 44 45 46 47 48 49 50\n", + " * datetime (datetime) <U20 '2024-09-17T00:00:00Z'\n", + " * t (t) datetime64[ns] 2024-09-17 2024-09-17T01:00:00 ... 2024-10-02\n", + "Data variables:\n", + " tcc (x, y, z, number, datetime, t) float64 0.1876 0.2019 ... 0.9239\n", + " 2t (x, y, z, number, datetime, t) float64 293.0 292.4 ... 296.1 294.5\n", + " ssrd (x, y, z, number, datetime, t) float64 0.0 0.0 ... 3.189e+08\n", + "Attributes:\n", + " class: od\n", + " Forecast date: 2024-09-17T00:00:00Z\n", + " domain: g\n", + " expver: 0079\n", + " levtype: sfc\n", + " number: 1\n", + " stream: enfo\n", + " type: pf
<xarray.Dataset>\n", + "Dimensions: (x: 1, y: 1, t: 1, number: 1, z: 22)\n", + "Coordinates:\n", + " * x (x) float64 38.91\n", + " * y (y) float64 350.9\n", + " * t (t) <U20 '2024-09-18T00:00:00Z'\n", + " * number (number) int64 1\n", + " * z (z) int64 1 2 3 5 7 10 20 30 ... 300 400 500 600 700 850 925 1000\n", + "Data variables:\n", + " q (x, y, t, number, z) float64 4.112e-06 4.091e-06 ... 0.006511\n", + " o3 (x, y, t, number, z) float64 4.657e-06 8.39e-06 ... 7.385e-08\n", + "Attributes:\n", + " class: od\n", + " domain: g\n", + " expver: 0079\n", + " levtype: pl\n", + " number: 1\n", + " stream: enfo\n", + " type: pf
<xarray.Dataset>\n", + "Dimensions: (datetimes: 1, number: 1, steps: 1, points: 1093)\n", + "Coordinates:\n", + " * datetimes (datetimes) <U20 '2024-09-17T12:00:00Z'\n", + " * number (number) int64 1\n", + " * steps (steps) int64 0\n", + " * points (points) int64 0 1 2 3 4 5 6 ... 1087 1088 1089 1090 1091 1092\n", + " x (points) float64 37.08 37.08 37.08 37.15 ... 42.0 42.0 42.0 42.07\n", + " y (points) float64 351.1 351.2 351.3 351.1 ... 351.6 351.7 351.7\n", + " z (points) float64 0.0 0.0 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 0.0\n", + "Data variables:\n", + " 2t (datetimes, number, steps, points) float64 298.7 298.0 ... 297.4\n", + " ssrd (datetimes, number, steps, points) float64 0.0 0.0 ... 0.0 0.0\n", + "Attributes:\n", + " class: od\n", + " Forecast date: 2024-09-17T12:00:00Z\n", + " domain: g\n", + " expver: 0079\n", + " levtype: sfc\n", + " number: 1\n", + " step: 0\n", + " stream: enfo\n", + " type: pf\n", + " date: 2024-09-17T12:00:00Z" + ], + "text/plain": [ + "