CI: use audbcards cache and pre-fill (#14)

* Use audbcards cache and pre-fill

* Use extra file to confgiure repository

* Fix linter

.github/workflows/doc.yml

@@ -21,8 +21,8 @@ jobs:
     - name: Cache
       uses: actions/cache@v3
       with:
-        path: docs/cache
-        key: datasets
+        path: .cache/audbcards
+        key: audbcards
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
@@ -39,7 +39,10 @@ jobs:
       run: pip install -r docs/requirements.txt
 
     - name: Test building documentation
-      run: python -m sphinx docs/ docs/build/ -b html -W
+      run: |
+        export AUDBCARDS_CACHE_ROOT=".cache/audbcards"
+        python pre-fill-cache.py
+        python -m sphinx docs/ docs/build/ -b html -W
 
     #- name: Check links in documentation
     #  run: python -m sphinx docs/ docs/build/ -b linkcheck -W

.github/workflows/pages.yml

@@ -10,6 +10,12 @@ jobs:
       with:
         fetch-depth: 2
 
+    - name: Cache
+      uses: actions/cache@v3
+      with:
+        path: .cache/audbcards
+        key: audbcards
+
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
@@ -29,6 +35,8 @@ jobs:
 
     - name: Build documentation
       run: |
+        export AUDBCARDS_CACHE_ROOT=".cache/audbcards"
+        python pre-fill-cache.py
         python -m sphinx docs/ docs/build/ -b html
 
     - name: Deploy documentation to Github pages

.github/workflows/publish.yml

@@ -13,6 +13,12 @@ jobs:
       with:
         fetch-depth: 2
 
+    - name: Cache
+      uses: actions/cache@v3
+      with:
+        path: .cache/audbcards
+        key: audbcards
+
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
@@ -32,6 +38,8 @@ jobs:
 
     - name: Build documentation
       run: |
+        export AUDBCARDS_CACHE_ROOT=".cache/audbcards"
+        python pre-fill-cache.py
         python -m sphinx docs/ build/html -b html
 
     - name: Deploy documentation to Github pages

docs/conf.py

@@ -1,6 +1,7 @@
-import audb
 import audeer
 
+from repository import repository
+
 
 # Project -----------------------------------------------------------------
 project = "datasets"
@@ -42,11 +43,7 @@
     (
         "datasets",  # folder name
         "Datasets",  # datasets overview page header
-        audb.Repository(
-            "data-public",
-            "https://audeering.jfrog.io/artifactory",
-            "artifactory",
-        ),
+        repository,
         True,  # don't show audio examples
     ),
 ]

pre-fill-cache.py

@@ -0,0 +1,56 @@
+# As some datasets contain large archives,
+# holding several files
+# the audb cache can get very large
+# when downloading all media examples.
+# To avoid this,
+# we pre-download all the media files here,
+# but clear the audb cache after each dataset.
+import os
+import shutil
+import tempfile
+
+import audb
+import audbcards
+import audeer
+
+from repository import repository
+
+
+def cache_media_path(dataset: audbcards.Dataset) -> str:
+    r"""Return path to example media in audbcards cache.
+
+    Args:
+        dataset: dataset object
+
+    Returns:
+        path to example media file in ``audbcards`` cache
+
+    """
+    default_audbcards_cache_root = (
+        os.environ.get("AUDBCARDS_CACHE_ROOT") or audbcards.config.CACHE_ROOT
+    )
+    return audeer.path(
+        default_audbcards_cache_root,
+        dataset.name,
+        dataset.version,
+        f"{dataset.name}-{dataset.version}-player-media",
+        dataset.example_media,
+    )
+
+
+_, _, free = shutil.disk_usage("/")
+print(f"Free disk space: {free // (2**30):d} GiB")
+
+audb.config.REPOSITORIES = [repository]
+df = audb.available(only_latest=True)
+datasets = list(df.index)
+print(f"Number of datasets: {len(datasets)}")
+for name in datasets:
+    version = df.loc[name, "version"]
+    with tempfile.TemporaryDirectory() as audb_cache_root:
+        audb.config.CACHE_ROOT = audb_cache_root
+        ds = audbcards.Dataset(name, version)
+        if ds.example_media is not None and not os.path.exists(cache_media_path(ds)):
+            print(f"{name}, v{version}: {ds.example_media}", flush=True)
+            dc = audbcards.Datacard(ds, sphinx_src_dir="docs")
+            dc.player()

pyproject.toml

@@ -0,0 +1,129 @@
+# ----- codespell ---------------------------------------------------------
+[tool.codespell]
+builtin = 'clear,rare,informal,usage,names'
+skip = './build'
+
+
+# ----- ruff --------------------------------------------------------------
+#
+[tool.ruff]
+cache-dir = '~/.cache/ruff'
+
+[tool.ruff.format]
+docstring-code-format = true
+
+[tool.ruff.lint]
+select = [
+    'D',     # pydocstyle
+    'E',     # pycodestyle errors
+    'F',     # Pyflakes
+    'I',     # isort
+    'N',     # pep8-naming
+    'W',     # pycodestyle warnings
+]
+
+extend-ignore = [
+    'D100',  # Missing docstring in public module
+    'D101',  # Missing docstring in public class
+    'D102',  # Missing docstring in public method
+    'D103',  # Missing docstring in public function
+    'D104',  # Missing docstring in public package
+    'D107',  # Missing docstring in `__init__`
+]
+
+[tool.ruff.lint.per-file-ignores]
+'__init__.py' = [
+    'F401',  # * imported but unused
+]
+'common.py' = [
+    'D105',  # Missing docstring in magic method
+]
+
+
+# ----- I: isort -----
+#
+# Check correct order/syntax of import statements
+#
+[tool.ruff.lint.isort]
+
+# All from imports have their own line, e.g.
+#
+#   from .utils import util_a
+#   from .utils import util_b
+#
+force-single-line = true
+
+# Sort by module names
+# and not import before from, e.g.
+#
+#     from datetime import date
+#     import os
+#
+force-sort-within-sections = true
+
+# Ensure we have two empty lines
+# after last import
+lines-after-imports = 2
+
+# Group all audEERING packages into a separate section, e.g.
+#
+#     import os
+#
+#     import numpy as np
+#
+#     import audmath
+#
+section-order = [
+    'future',
+    'standard-library',
+    'third-party',
+    'audeering',
+    'first-party',
+    'local-folder',
+]
+[tool.ruff.lint.isort.sections]
+'audeering' = [
+    'audb',
+    'audbcards',
+    'audbackend',
+    'audbenchmark',
+    'audbgui',
+    'audeer',
+    'audformat',
+    'audinterface',
+    'audiofile',
+    'audmath',
+    'audmetric',
+    'audmodel',
+    'audobject',
+    'audonnx',
+    'audpann',
+    'audplot',
+    'audresample',
+    'audtorch',
+    'auglib',
+    'auglibgui',
+    'auvad',
+    'opensmile',
+    'productionmodels',
+    'sphinx-audeering-theme',
+]
+
+
+# ----- N: pep8-naming -----
+#
+# Check variable/class names follow PEP8 naming convention
+#
+[tool.ruff.lint.pep8-naming]
+ignore-names = [
+    'config',  # allow lowercase class name
+    'test_*',  # allow uppercase name when testing a class
+]
+
+
+# ----- W: pycodestyle -----
+#
+# Check docstrings follow selected convention
+#
+[tool.ruff.lint.pydocstyle]
+convention = 'google'

repository.py

@@ -0,0 +1,8 @@
+import audb
+
+
+repository = audb.Repository(
+    "data-public",
+    "https://audeering.jfrog.io/artifactory",
+    "artifactory",
+)