ADD: Add in bare bones of package

.github/workflows/ci.yaml

@@ -0,0 +1,62 @@
+name: CI
+env:
+  micromamba_version: 1.3
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch: # allows you to trigger manually
+
+jobs:
+  pre-job:
+    runs-on: ubuntu-latest
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          concurrent_skipping: 'same_content'
+          skip_after_successful_duplicate: 'false'
+          do_not_skip: '["pull_request", "workflow_dispatch", "schedule"]'
+          paths_ignore: '["**/docs/**"]'
+
+  build:
+    needs: pre-job
+    if: ${{ needs.pre-job.outputs.should_skip != 'true' }}
+    name: python-${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9', '3.10', '3.11']
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - uses: actions/checkout@v3
+      - uses: mamba-org/provision-with-micromamba@main
+        with:
+          environment-name: act-test-data-dev
+          environment-file: ci/environment.yml
+          extra-specs: |
+            python=${{ matrix.python-version }}
+      - name: Set up conda environment
+        run: |
+          python -m pip install -e .
+          conda list
+      - name: Run Tests
+        run: |
+          python -m pytest --cov=./ --cov-report=xml --verbose
+      - name: Upload code coverage to Codecov
+        uses: codecov/[email protected]
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          env_vars: OS,PYTHON
+          name: codecov-umbrella
+          fail_ci_if_error: false

.github/workflows/pypi-release.yml

@@ -0,0 +1,74 @@
+name: Build and Upload pythia-datasets to PyPI
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  build-artifacts:
+    runs-on: ubuntu-latest
+    if: github.repository == 'ARM-DOE/act-test-data'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install setuptools setuptools-scm wheel twine check-manifest
+      - name: Build tarball and wheels
+        run: |
+          git clean -xdf
+          git restore -SW .
+          python -m build --sdist --wheel .
+      - name: Check built artifacts
+        run: |
+          python -m twine check dist/*
+          pwd
+          if [ -f dist/act-test-data-0.0.0.tar.gz ]; then
+            echo "❌ INVALID VERSION NUMBER"
+            exit 1
+          else
+            echo "✅ Looks good"
+          fi
+      - uses: actions/upload-artifact@v3
+        with:
+          name: releases
+          path: dist
+
+  test-built-dist:
+    needs: build-artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/setup-python@v4
+        name: Install Python
+        with:
+          python-version: 3.11
+      - uses: actions/download-artifact@v3
+        with:
+          name: releases
+          path: dist
+      - name: List contents of built dist
+        run: |
+          ls -ltrh
+          ls -ltrh dist
+  upload-to-pypi:
+    needs: test-built-dist
+    if: github.event_name == 'release'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: releases
+          path: dist
+      - name: Publish package to PyPI
+        uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_TOKEN }}
+          verbose: true

MANIFEST.in

@@ -0,0 +1,14 @@
+include LICENSE
+include README.md
+
+recursive-include act_test_data *.py
+recursive-exclude * __pycache__
+recursive-exclude * data
+recursive-exclude * *.py[co]
+
+include *.md
+include *.toml
+include *.txt
+prune tests*
+prune ci*
+prune data*

README.md

@@ -1,2 +1,75 @@
 # act-test-data
-Test data for the atmospheric data community toolkit (ACT)
+[![CI](https://github.com/ARM-DOE/act-test-data/actions/workflows/ci.yaml/badge.svg)](https://github.com/ARM-DOE/act-test-data/actions/workflows/ci.yaml)
+[![PyPI Version](https://img.shields.io/pypi/v/act-test-data.svg)](https://pypi.python.org/pypi/act-test-data)
+[![Conda Version](https://img.shields.io/conda/vn/conda-forge/act-test-data.svg)](https://anaconda.org/conda-forge/act-test-data)
+
+A place to share radar data with the community, shared between the open radar packages
+
+## Sample data sets
+
+These files are used as sample data in openradar examples/notebooks and are downloaded by `open-radar-data` package:
+
+- `sample_file`
+
+## Adding new datasets
+
+To add a new dataset file, please follow these steps:
+
+1. Add the dataset file to the `data/` directory
+2. From the command line, run `python make_registry.py` script to update the registry file residing in `open_radar_data/registry.txt`
+3. Commit and push your changes to GitHub
+
+## Using datasets in notebooks and/or scripts
+
+- Ensure the `act-test-data` package is installed in your environment
+
+  ```bash
+  python -m pip install act-test-data
+
+  # or
+
+  python -m pip install git+https://github.com/ARM-DOE/act-test-data
+
+  # or
+
+  conda install -c conda-forge act-test-data
+  ```
+
+- Import `DATASETS` and inspect the registry to find out which datasets are available
+
+  ```python
+  In [1]: from act_test_data import DATASETS
+
+  In [2]: DATASETS.registry_files
+  Out[2]: ['sample_file.nc`]
+  ```
+
+- To fetch a data file of interest, use the `.fetch` method and provide the filename of the data file. This will
+
+  - download and cache the file if it doesn't exist already.
+  - retrieve and return the local path
+
+  ```python
+  In [4]: filepath = DATASETS.fetch('sample_data.nc')
+
+  In [5]: filepath
+  Out[5]: '/Users/mgrover/Library/Caches/act-test-data/sample_sgp_data.nc'
+  ```
+
+- Once you have access to the local filepath, you can then use it to load your dataset into pandas or xarray or your package of choice:
+
+  ```python
+  In [6]: radar = pyart.io.read(filepath)
+  ```
+
+## Changing the default data cache location
+
+The default cache location (where the data are saved on your local system) is dependent on the operating system. You can use the `locate()` method to identify it:
+
+```python
+from act_test_data import locate
+locate()
+```
+
+The location can be overwritten by the `ACT_TEST_DATA_DIR` environment
+variable to the desired destination.

act_test_data/__init__.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# flake8: noqa
+"""Top-level module for pythia-datasets ."""
+from pkg_resources import DistributionNotFound, get_distribution
+
+from .dataset import DATASETS, locate
+
+try:
+    __version__ = get_distribution(__name__).version
+except DistributionNotFound:  # pragma: no cover
+    # package is not installed
+    __version__ = 'unknown'  # pragma: no cover

act_test_data/dataset.py

@@ -0,0 +1,27 @@
+import pkg_resources
+import pooch
+
+DATASETS = pooch.create(
+    path=pooch.os_cache('open-radar-data'),
+    base_url='https://github.com/openradar/open-radar-data/raw/main/data/',
+    env='ACT_TEST_DATA_DIR',
+)
+
+with pkg_resources.resource_stream('open_radar_data', 'registry.txt') as registry_file:
+    DATASETS.load_registry(registry_file)
+
+
+def locate():
+    """The absolute path to the sample data storage location on disk.
+    This is where the data are saved on your computer. The location is
+    dependent on the operating system. The folder locations are defined by the
+    ``appdirs``  package (see the `appdirs documentation
+    <https://github.com/ActiveState/appdirs>`__).
+    The location can be overwritten by the ``PYTHIA_DATASETS_DIR`` environment
+    variable to the desired destination.
+    Returns
+    -------
+    path : str
+        The local data storage location.
+    """
+    return str(DATASETS.abspath)

ci/environment.yml

@@ -0,0 +1,9 @@
+name: act-test-data-dev
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - pip
+  - pooch
+  - pre-commit
+  - pytest-cov

make_registry.py

@@ -0,0 +1,10 @@
+import os
+import pathlib
+
+import pooch
+
+if __name__ == '__main__':
+    here = pathlib.Path(os.path.dirname(__file__))
+    data_dir = here / 'data'
+
+    pooch.make_registry(data_dir, here / 'act_test_data/registry.txt')

pyproject.toml

@@ -0,0 +1,89 @@
+[project]
+name = "open-radar-data"
+description = """Provides utility functions for accessing data repository for openradar examples/notebooks"""
+requires-python = ">=3.9"
+license = {file = "LICENSE"}
+keywords = ["act-test-data", "atmosphere", "meteorology"]
+authors = [
+    {name = "Atmospheric Data Community Toolkit Dev Team"},
+]
+classifiers=[
+    'Development Status :: 2 - Pre-Alpha',
+    'License :: OSI Approved :: MIT License',
+    'Natural Language :: English',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
+    'Topic :: Scientific/Engineering',
+    'Operating System :: OS Independent',
+    'Intended Audience :: Science/Research',
+]
+dynamic = ["version", "dependencies", "readme"]
+
+[build-system]
+requires = [
+    "setuptools>=45",
+    "wheel",
+    "setuptools_scm[toml]>=7.0",
+]
+build-backend = "setuptools.build_meta"
+
+[project.urls]
+homepage = "https://github.com/ARM-DOE/act-test-data"
+documentation = "https://github.com/ARM-DOE/act-test-data"
+repository = "https://github.com/ARM-DOE/act-test-data"
+tracker = "https://github.com/ARM-DOE/act-test-data/issues"
+
+[tool.setuptools]
+packages = ["act_test_data"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+readme = {file = ["README.md"], content-type = "text/markdown"}
+
+[tool.setuptools_scm]
+write_to = "arm_test_data/version.py"
+version_scheme = "post-release"
+local_scheme= "dirty-tag"
+fallback_version = "999"
+
+
+[tool.black]
+line-length = 100
+target-version = ['py39']
+skip-string-normalization = true
+
+[tool.ruff]
+target-version = "py39"
+builtins = ["ellipsis"]
+exclude = [
+    ".eggs",
+    "doc",
+]
+
+# E402: module level import not at top of file
+# E501: line too long - let black worry about that
+# E731: do not assign a lambda expression, use a def
+ignore = [
+    "E402",
+    "E501",
+    "E731",
+]
+select = [
+    # Pyflakes
+    "F",
+    # Pycodestyle
+    "E",
+    "W",
+    # isort
+    "I",
+    # Pyupgrade
+    "UP",
+]
+
+[tool.ruff.isort]
+known-first-party = ["act-test-data"]
+
+[tool.check-manifest]
+ignore = ["docs/*", "tests/*", "ci/*"]

requirements.txt

@@ -0,0 +1 @@
+pooch

tests/test_dataset.py

@@ -0,0 +1,14 @@
+import pathlib
+
+from open_radar_data import DATASETS, locate
+
+
+def test_registry():
+    files = DATASETS.registry_files
+    assert len(files) > 0
+
+
+def test_locate():
+    p = locate()
+    assert 'open-radar-data' in p
+    assert pathlib.Path(p)