diff --git a/.github/workflows/durations.yml b/.github/workflows/durations.yml new file mode 100644 index 0000000000..d296d4c3e7 --- /dev/null +++ b/.github/workflows/durations.yml @@ -0,0 +1,63 @@ +name: Update Durations + +on: + # every Sunday at 00:00 UTC + # https://crontab.guru/#0_0_*_*_0 + schedule: + - cron: '0 0 * * 0' + + # https://docs.github.com/en/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_dispatch + workflow_dispatch: + +jobs: + update-durations: + runs-on: ubuntu-latest + permissions: + # necessary to open PR + # https://github.com/peter-evans/create-pull-request#action-inputs + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v3 + + - name: download recent artifacts + run: | + gh run list \ + --branch main \ + --workflow tests \ + --limit 10 \ + --json databaseId \ + --jq '.[].databaseId' \ + | xargs \ + -n 1 \ + gh run download \ + --dir ${{ runner.temp }}/artifacts/ \ + --pattern '*-all' \ + || true + env: + GITHUB_TOKEN: ${{ github.token }} + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: combine recent durations from artifacts + run: python ./tools/durations/combine.py ${{ runner.temp }}/artifacts/ + + - name: create updated durations PR + uses: peter-evans/create-pull-request@v5 + with: + push-to-fork: conda-bot/conda + token: ${{ secrets.DURATIONS_TOKEN }} + branch: update-durations + delete-branch: true + commit-message: Update test durations + author: Conda Bot <18747875+conda-bot@users.noreply.github.com> + committer: Conda Bot <18747875+conda-bot@users.noreply.github.com> + title: 🤖 Update test durations + body: | + Aggregate recent test durations for each test and update the durations file. + + [durations.yml]: ${{ github.server_url }}/${{ github.repository }}/blob/main/.github/workflows/durations.yml + + This PR was created automatically by the [`durations.yml`][durations.yml] workflow. diff --git a/tools/durations/Linux.json b/tools/durations/Linux.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tools/durations/Linux.json @@ -0,0 +1 @@ +{} diff --git a/tools/durations/Windows.json b/tools/durations/Windows.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tools/durations/Windows.json @@ -0,0 +1 @@ +{} diff --git a/tools/durations/combine.py b/tools/durations/combine.py new file mode 100644 index 0000000000..b42de633fe --- /dev/null +++ b/tools/durations/combine.py @@ -0,0 +1,56 @@ +# Copyright (C) 2014 Anaconda, Inc +# SPDX-License-Identifier: BSD-3-Clause +""" +Script to combine test durations from all runs. + +If the tests splits are looking uneven or the test suite has +siginificantly changed, update ./tools/durations/${OS}.json in the root of the +repository and pytest-split may work better. + +``` +$ gh run list --branch +$ gh run download --dir ./artifacts/ +$ python ./tools/durations/combine.py ./artifacts/ +$ git add ./tools/durations/ +$ git commit -m "Update test durations" +$ git push +``` +""" +import json +from pathlib import Path +from statistics import fmean +from sys import argv + +combined: dict[str, dict[str, list[float]]] = {} + +durations_dir = Path(__file__).parent +artifacts_dir = Path(argv[-1]).expanduser().resolve() + +# aggregate all new durations +for path in artifacts_dir.glob("**/*.json"): + os = path.stem + combined_os = combined.setdefault(os, {}) + + data = json.loads(path.read_text()) + for key, value in data.items(): + combined_os.setdefault(key, []).append(value) + +# aggregate new and old durations while discarding durations that no longer exist +for path in durations_dir.glob("**/*.json"): + os = path.stem + combined_os = combined.setdefault(os, {}) + + data = json.loads(path.read_text()) + for key in set(combined_os).intersection(durations_dir.glob("**/*.json")): + combined_os.setdefault(key, []).append(data[key]) + +# write out averaged durations +for os, combined_os in combined.items(): + (durations_dir / f"{os}.json").write_text( + json.dumps( + {key: fmean(values) for key, values in combined_os.items()}, + indent=4, + sort_keys=True, + ) + + "\n" # include trailing newline + ) diff --git a/tools/durations/macOS.json b/tools/durations/macOS.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/tools/durations/macOS.json @@ -0,0 +1 @@ +{}