Add ci

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,7 +2,7 @@
   <summary><b>Before submitting</b></summary>
 
 - [ ] Was this discussed/agreed via a Github issue? (no need for typos and docs improvements)
-- [ ] Did you read the [contributor guideline](https://github.com/Lightning-AI/pytorch-lightning/blob/main/.github/CONTRIBUTING.md), Pull Request section?
+- [ ] Did you read the [contributor guideline](https://github.com/Lightning-AI/lit-data/blob/main/.github/CONTRIBUTING.md), Pull Request section?
 - [ ] Did you make sure to update the docs?
 - [ ] Did you write any new necessary tests?
 

.github/workflows/check-typing.yml

@@ -0,0 +1,82 @@
+name: Check formatting flow
+
+on:
+  workflow_call:
+    inputs:
+      actions-ref:
+        description: "Version of actions, normally the same as workflow"
+        required: true
+        type: string
+      python-version:
+        description: "Python version to use"
+        default: "3.9"
+        required: false
+        type: string
+      extra-typing:
+        description: "Package extra to be installed for type checks + include mypy"
+        default: "test"
+        required: false
+        type: string
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  mypy:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up Python 🐍 ${{ inputs.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Install dependencies
+        timeout-minutes: 20
+        run: |
+          # don't use --upgrade to respect the version installed via setup.py
+          pip install -e '.[${{ inputs.extra-typing }}]' mypy \
+            --extra-index-url https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip list
+
+      - name: Pull reusable 🤖 actions️
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.actions-ref }}
+          path: .cicd
+          repository: Lightning-AI/utilities
+      - name: Print 🖨️ dependencies
+        uses: ./.cicd/.github/actions/pip-list
+        with:
+          unfold: true
+
+      # see: https://github.com/python/mypy/issues/10600#issuecomment-857351152
+      - run: yes | mypy src --install-types || true
+
+      - name: Check typing
+        # mypy uses the config file found in the following order:
+        # 1. mypy.ini
+        # 2. pyproject.toml
+        # 3. setup.cfg
+        # 4. $XDG_CONFIG_HOME/mypy/config
+        # 5. ~/.config/mypy/config
+        # 6. ~/.mypy.ini
+        # https://mypy.readthedocs.io/en/stable/config_file.html
+        run: mypy
+
+      - name: suggest ignores
+        if: failure()
+        run: |
+          mypy --no-error-summary 2>&1 \
+            | tr ':' ' ' \
+            | awk '{print $1}' \
+            | sort \
+            | uniq \
+            | sed 's/\.py//g;  s|src/||g;  s|\/__init__||g;  s|\/|\.|g' \
+            | xargs -I {} echo '"{}",' \
+          || true

.github/workflows/ci-checks.yml

@@ -25,7 +25,7 @@ jobs:
       azure-dir: ""
 
   check-package:
-    uses: Lightning-AI/utilities/.github/workflows/check-package.yml@main
+    uses: ./check-typing.yml
     with:
       actions-ref: main
       import-name: "lit_data"
@@ -39,4 +39,4 @@ jobs:
   check-docs:
     uses: Lightning-AI/utilities/.github/workflows/check-docs.yml@main
     with:
-      requirements-file: "_requirements/docs.txt"
+      requirements-file: "requirements/docs.txt"

.github/workflows/ci-testing.yml

@@ -41,11 +41,16 @@ jobs:
       - name: Set min. dependencies
         if: matrix.requires == 'oldest'
         run: |
-          for fpath in ('requirements.txt', '_requirements/test.txt'):
+          for fpath in ('requirements.txt', 'requirements/test.txt'):
               req = open(fpath).read().replace('>=', '==')
               open(fpath, 'w').write(req)
         shell: python
 
+      - name: Display dependencies
+        run: |
+          cat requirements.txt
+          cat requirements/test.txt
+
       - name: Get pip cache dir
         id: pip-cache
         run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
@@ -65,7 +70,7 @@ jobs:
 
       - name: Tests
         run: |
-          coverage run --source lit_data -m pytest src tests -v
+          coverage run --source lit_data -m pytest tests -v
 
       - name: Statistics
         if: success()

.github/workflows/docs-build.yml

@@ -10,7 +10,7 @@ jobs:
   build-docs:
     uses: Lightning-AI/utilities/.github/workflows/check-docs.yml@main
     with:
-      requirements-file: "_requirements/docs.txt"
+      requirements-file: "requirements/docs.txt"
 
   # https://github.com/marketplace/actions/deploy-to-github-pages
   docs-deploy:

.github/workflows/release-pypi.yml

@@ -21,8 +21,10 @@ jobs:
 
       - name: Install dependencies
         run: pip install -U build twine
+
       - name: Build package
         run: python -m build
+
       - name: Check package
         run: twine check dist/*
 

.pre-commit-config.yaml

@@ -1,89 +1,113 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 default_language_version:
   python: python3
 
 ci:
   autofix_prs: true
   autoupdate_commit_msg: "[pre-commit.ci] pre-commit suggestions"
-  autoupdate_schedule: "monthly"
+  autoupdate_schedule: quarterly
   # submodules: true
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
-      - id: check-case-conflict
+        # keep formatting in README flexible
+        exclude: README.md
+      - id: check-json
       - id: check-yaml
       - id: check-toml
-      - id: check-json
-      - id: check-added-large-files
       - id: check-docstring-first
+      - id: check-executables-have-shebangs
+      - id: check-case-conflict
+      - id: check-added-large-files
+        args: ["--maxkb=350", "--enforce-all"]
+        exclude: |
+          (?x)^(
+              docs/source-pytorch/_static/images/general/fast_2.gif|
+              docs/source-pytorch/_static/images/mnist_imgs/pt_to_pl.jpg|
+              docs/source-pytorch/_static/images/lightning_module/pt_to_pl.png|
+              docs/source-pytorch/_static/images/general/pl_quick_start_full_compressed.gif|
+              docs/source-pytorch/_static/images/general/pl_overview_flat.jpg|
+              docs/source-pytorch/_static/images/general/pl_overview.gif|
+              src/lightning/fabric/CHANGELOG.md|
+              src/lightning/pytorch/CHANGELOG.md
+          )$
       - id: detect-private-key
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.9.0
+    rev: v3.15.0
     hooks:
       - id: pyupgrade
         args: ["--py38-plus"]
         name: Upgrade code
 
-  - repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.10.0
-    hooks:
-      - id: python-use-type-annotations
-
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.5
+    rev: v2.2.6
     hooks:
       - id: codespell
         additional_dependencies: [tomli]
-        #args: ["--write-changes"]
+        #args: ["--write-changes"] # uncomment if you want to get automatic fixing
 
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.3
-    hooks:
-      - id: prettier
-        # https://prettier.io/docs/en/options.html#print-width
-        args: ["--print-width=120"]
-
-  - repo: https://github.com/myint/docformatter
+  - repo: https://github.com/PyCQA/docformatter
     rev: v1.7.5
     hooks:
       - id: docformatter
         additional_dependencies: [tomli]
         args: ["--in-place"]
 
-  - repo: https://github.com/psf/black
-    rev: 23.7.0
+  - repo: https://github.com/asottile/yesqa
+    rev: v1.5.0
     hooks:
-      - id: black
-        name: Black code
+      - id: yesqa
+        name: Unused noqa
+        additional_dependencies:
+          #- pep8-naming
+          - flake8-pytest-style
+          - flake8-bandit
+          - flake8-simplify
+          - flake8-return
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.2.0"
+    hooks:
+      - id: ruff
+        args: ["--fix", "--preview"]
+      - id: ruff-format
+        args: ["--preview"]
 
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.16
+    rev: 0.7.17
     hooks:
       - id: mdformat
         additional_dependencies:
           - mdformat-gfm
-          - mdformat-black
+          #- mdformat-black
           - mdformat_frontmatter
-        exclude: CHANGELOG.md
-
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-        additional_dependencies:
-          - pep8-naming
-          - pydocstyle
-          #- flake8-comprehensions
-          #- flake8-pytest-style
-          #- flake8-return
-          #- flake8-simplify
+        exclude: |
+          (?x)^(
+              src/data/CHANGELOG.md|
+              README.md
+          )$
 
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.277
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.1.0
     hooks:
-      - id: ruff
-        args: ["--fix"]
+      - id: prettier
+        # https://prettier.io/docs/en/options.html#print-width
+        args: ["--print-width=120"]

.readthedocs.yml

@@ -17,4 +17,4 @@ formats: all
 python:
   version: 3.7
   install:
-    - requirements: _requirements/docs.txt
+    - requirements: requirements/docs.txt

CHANGELOG.md

@@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased] - YYYY-MM-DD
+## \[Unreleased\] - YYYY-MM-DD
 
 ### Added
 

MANIFEST.in

@@ -5,7 +5,7 @@ recursive-exclude __pycache__  *.py[cod] *.orig
 
 # Include the README and CHANGELOG
 include *.md
-recursive-include src *.md
+recursive-include lit_data *.md
 
 # Include the license file
 include LICENSE
@@ -28,7 +28,7 @@ exclude docs
 
 # Include the Requirements
 include requirements.txt
-recursive-include _requirements *.tx;t
+recursive-include requirements *.tx;t
 
 # Exclude Makefile
 exclude Makefile

Makefile

@@ -7,14 +7,14 @@ export SPHINX_MOCK_REQUIREMENTS=0
 
 test: clean
 	pip install -q -r requirements.txt
-	pip install -q -r _requirements/test.txt
+	pip install -q -r requirements/test.txt
 
 	# use this to run tests
-	python -m coverage run --source lit_data -m pytest src tests -v --flake8
+	python -m coverage run --source lit_data -m pytest src -v --flake8
 	python -m coverage report
 
 docs: clean
-	pip install . --quiet -r _requirements/docs.txt
+	pip install . --quiet -r requirements/docs.txt
 	python -m sphinx -b html -W --keep-going docs/source docs/build
 
 clean:

README.md

@@ -32,7 +32,7 @@ Lightning Data can be installed with `pip`:
 <!--pytest.mark.skip-->
 
 ```bash
-pip install --no-cache-dir git+https://github.com/Lightning-AI/pytorch-lightning.git@master
+pip install --no-cache-dir git+https://github.com/Lightning-AI/lit-data.git@master
 ```
 
 ## 🏁 Quick Start