From 50529ddb4f390a92cd5e1b0298d10c0da8ce9b27 Mon Sep 17 00:00:00 2001 From: tchaton Date: Fri, 16 Feb 2024 08:19:24 +0000 Subject: [PATCH] update --- .github/workflows/ci-testing.yml | 2 +- .pre-commit-config.yaml | 112 ++++++++------ Makefile | 2 +- lit_data/CHANGELOG.md | 47 ++++++ pyproject.toml | 254 +++++++++++++++++++------------ requirements/requirements.txt | 5 +- requirements/test.txt | 10 +- 7 files changed, 286 insertions(+), 146 deletions(-) create mode 100644 lit_data/CHANGELOG.md diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 20905c2a..3d3db3d8 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -65,7 +65,7 @@ jobs: - name: Tests run: | - coverage run --source lit_data -m pytest src tests -v + coverage run --source lit_data -m pytest tests -v - name: Statistics if: success() diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea7b710c..295c2f1f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,89 +1,113 @@ +# Copyright The Lightning AI team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + default_language_version: python: python3 ci: autofix_prs: true autoupdate_commit_msg: "[pre-commit.ci] pre-commit suggestions" - autoupdate_schedule: "monthly" + autoupdate_schedule: quarterly # submodules: true repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - - id: check-case-conflict + # keep formatting in README flexible + exclude: README.md + - id: check-json - id: check-yaml - id: check-toml - - id: check-json - - id: check-added-large-files - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-case-conflict + - id: check-added-large-files + args: ["--maxkb=350", "--enforce-all"] + exclude: | + (?x)^( + docs/source-pytorch/_static/images/general/fast_2.gif| + docs/source-pytorch/_static/images/mnist_imgs/pt_to_pl.jpg| + docs/source-pytorch/_static/images/lightning_module/pt_to_pl.png| + docs/source-pytorch/_static/images/general/pl_quick_start_full_compressed.gif| + docs/source-pytorch/_static/images/general/pl_overview_flat.jpg| + docs/source-pytorch/_static/images/general/pl_overview.gif| + src/lightning/fabric/CHANGELOG.md| + src/lightning/pytorch/CHANGELOG.md + )$ - id: detect-private-key - repo: https://github.com/asottile/pyupgrade - rev: v3.9.0 + rev: v3.15.0 hooks: - id: pyupgrade args: ["--py38-plus"] name: Upgrade code - - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.10.0 - hooks: - - id: python-use-type-annotations - - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell additional_dependencies: [tomli] - #args: ["--write-changes"] + #args: ["--write-changes"] # uncomment if you want to get automatic fixing - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 - hooks: - - id: prettier - # https://prettier.io/docs/en/options.html#print-width - args: ["--print-width=120"] - - - repo: https://github.com/myint/docformatter + - repo: https://github.com/PyCQA/docformatter rev: v1.7.5 hooks: - id: docformatter additional_dependencies: [tomli] args: ["--in-place"] - - repo: https://github.com/psf/black - rev: 23.7.0 + - repo: https://github.com/asottile/yesqa + rev: v1.5.0 hooks: - - id: black - name: Black code + - id: yesqa + name: Unused noqa + additional_dependencies: + #- pep8-naming + - flake8-pytest-style + - flake8-bandit + - flake8-simplify + - flake8-return + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.2.0" + hooks: + - id: ruff + args: ["--fix", "--preview"] + - id: ruff-format + args: ["--preview"] - repo: https://github.com/executablebooks/mdformat - rev: 0.7.16 + rev: 0.7.17 hooks: - id: mdformat additional_dependencies: - mdformat-gfm - - mdformat-black + #- mdformat-black - mdformat_frontmatter - exclude: CHANGELOG.md - - - repo: https://github.com/asottile/yesqa - rev: v1.5.0 - hooks: - - id: yesqa - additional_dependencies: - - pep8-naming - - pydocstyle - #- flake8-comprehensions - #- flake8-pytest-style - #- flake8-return - #- flake8-simplify + exclude: | + (?x)^( + src/data/CHANGELOG.md| + README.md + )$ - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.277 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.1.0 hooks: - - id: ruff - args: ["--fix"] + - id: prettier + # https://prettier.io/docs/en/options.html#print-width + args: ["--print-width=120"] diff --git a/Makefile b/Makefile index e1d83cd4..56f3a147 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ test: clean pip install -q -r requirements/test.txt # use this to run tests - python -m coverage run --source lit_data -m pytest src tests -v --flake8 + python -m coverage run --source lit_data -m pytest src -v --flake8 python -m coverage report docs: clean diff --git a/lit_data/CHANGELOG.md b/lit_data/CHANGELOG.md new file mode 100644 index 00000000..255c28a4 --- /dev/null +++ b/lit_data/CHANGELOG.md @@ -0,0 +1,47 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). + +## [unReleased] - 2024-MM-DD + +### Added + +- + +- + +- + +### Changed + +- + +- + +- + +### Deprecated + +- + +- + +- + +### Removed + +- + +- + +- + +### Fixed + +- + +- + +- diff --git a/pyproject.toml b/pyproject.toml index d3a55d64..ee320fb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,21 @@ +# Copyright The Lightning AI team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [metadata] -license_file = "LICENSE" -description-file = "README.md" +name = "lit_data" +author = "Lightning-AI et al." +url = "https://github.com/Lightning-AI/lit-data" [build-system] requires = [ @@ -9,125 +24,174 @@ requires = [ ] -[tool.check-manifest] -ignore = [ - "*.yml", - ".github", - ".github/*" -] - - -[tool.pytest.ini_options] -norecursedirs = [ - ".git", - ".github", - "dist", - "build", - "docs", -] -addopts = [ - "--strict-markers", - "--doctest-modules", - "--color=yes", - "--disable-pytest-warnings", -] -filterwarnings = [ - "error::FutureWarning", -] -xfail_strict = true -junit_duration_report = "call" - -[tool.coverage.report] -exclude_lines = [ - "pragma: no cover", - "pass", -] - -[tool.codespell] -#skip = '*.py' -quiet-level = 3 -# comma separated list of words; waiting for: -# https://github.com/codespell-project/codespell/issues/2839#issuecomment-1731601603 -# also adding links until they ignored by its: nature -# https://github.com/codespell-project/codespell/issues/2243#issuecomment-1732019960 -#ignore-words-list = "" - - [tool.black] -# https://github.com/psf/black line-length = 120 -exclude = "(.eggs|.git|.hg|.mypy_cache|.venv|_build|buck-out|build|dist)" +exclude = '(_notebooks/.*)' [tool.docformatter] recursive = true -wrap-summaries = 120 +# this need to be shorter as some docstings are r"""... +wrap-summaries = 119 wrap-descriptions = 120 blank = true - -[tool.mypy] -files = [ - "src", -] -install_types = true -non_interactive = true -disallow_untyped_defs = true -ignore_missing_imports = true -show_error_codes = true -warn_redundant_casts = true -warn_unused_configs = true -warn_unused_ignores = true -allow_redefinition = true -# disable this rule as the Trainer attributes are defined in the connectors, not in its __init__ -disable_error_code = "attr-defined" -# style choices -warn_no_return = false +[tool.codespell] +# Todo: enable also python files in a next step +skip = '*.py' +quiet-level = 3 +# comma separated list of words; waiting for: +# https://github.com/codespell-project/codespell/issues/2839#issuecomment-1731601603 +# also adding links until they ignored by its: nature +# https://github.com/codespell-project/codespell/issues/2243#issuecomment-1732019960 +ignore-words-list = "te, compiletime" [tool.ruff] line-length = 120 # Enable Pyflakes `E` and `F` codes by default. -select = [ +lint.select = [ "E", "W", # see: https://pypi.org/project/pycodestyle "F", # see: https://pypi.org/project/pyflakes - "D", # see: https://pypi.org/project/pydocstyle - "N", # see: https://pypi.org/project/pep8-naming + "S", # see: https://pypi.org/project/flake8-bandit + "RUF018", # see: https://docs.astral.sh/ruff/rules/assignment-in-assert ] -extend-select = [ +lint.extend-select = [ + "I", # see: isort "C4", # see: https://pypi.org/project/flake8-comprehensions - "PT", # see: https://pypi.org/project/flake8-pytest-style - "RET", # see: https://pypi.org/project/flake8-return "SIM", # see: https://pypi.org/project/flake8-simplify + "RET", # see: https://pypi.org/project/flake8-return + "PT", # see: https://pypi.org/project/flake8-pytest-style ] -ignore = [ +lint.ignore = [ "E731", # Do not assign a lambda expression, use a def + "S108", + "E203", # conflicts with black ] # Exclude a variety of commonly ignored directories. exclude = [ - ".eggs", ".git", - ".mypy_cache", - ".ruff_cache", - "__pypackages__", - "_build", - "build", - "dist", - "docs" + "docs", + "_notebooks" +] +lint.ignore-init-module-imports = true + +[tool.ruff.lint.per-file-ignores] +".actions/*" = ["S101", "S310"] +"setup.py" = ["S101"] +"examples/**" = [ + "S101", # Use of `assert` detected + "S113", # todo: Probable use of requests call without + "S104", # Possible binding to all interface + "F821", # Undefined name `...` + "S311", # Standard pseudo-random generators are not suitable for cryptographic purposes + "S501", # Probable use of `requests` call with `verify=False` disabling SSL certificate checks + "S108", # Probable insecure usage of temporary file or directory: "/tmp/data/MNIST" ] -ignore-init-module-imports = true +"src/**" = [ + "S101", # todo: Use of `assert` detected + "S105", "S106", "S107", # todo: Possible hardcoded password: ... + "S113", # todo: Probable use of requests call without timeout + "S301", # todo: `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue + "S324", # todo: Probable use of insecure hash functions in `hashlib` + "S403", # todo: `pickle`, `cPickle`, `dill`, and `shelve` modules are possibly insecure + "S404", # todo: `subprocess` module is possibly insecure + "S602", # todo: `subprocess` call with `shell=True` identified, security issue + "S603", # todo: `subprocess` call: check for execution of untrusted input + "S605", # todo: Starting a process with a shell: seems safe, but may be changed in the future; consider rewriting without `shell` + "S607", # todo: Starting a process with a partial executable path + "RET504", # todo:Unnecessary variable assignment before `return` statement + "RET503", +] +"src/lightning/data/**" = [ + "S310", # todo: Audit URL open for permitted schemes. Allowing use of `file:` or custom schemes is often unexpected. +] +"tests/**" = [ + "S101", # Use of `assert` detected + "S105", "S106", # todo: Possible hardcoded password: ... + "S301", # `pickle` and modules that wrap it can be unsafe when used to deserialize untrusted data, possible security issue + "S113", # todo: Probable use of requests call without timeout + "S311", # todo: Standard pseudo-random generators are not suitable for cryptographic purposes + "S108", # todo: Probable insecure usage of temporary file or directory: "/tmp/sys-customizations-sync" + "S202", # Uses of `tarfile.extractall()` + "S403", # `pickle`, `cPickle`, `dill`, and `shelve` modules are possibly insecure + "S404", # `subprocess` module is possibly insecure + "S602", # todo: `subprocess` call with `shell=True` identified, security issue + "S603", # todo: `subprocess` call: check for execution of untrusted input + "S605", # todo: Starting a process with a shell: seems safe, but may be changed in the future; consider rewriting without `shell` + "S607", # todo: Starting a process with a partial executable path + "RET504", # todo:Unnecessary variable assignment before `return` statement + "PT004", # todo: Fixture `tmpdir_unittest_fixture` does not return anything, add leading underscore + "PT011", # todo: `pytest.raises(ValueError)` is too broad, set the `match` parameter or use a more specific exception + "PT012", # todo: `pytest.raises()` block should contain a single simple statement + "PT019", # todo: Fixture `_` without value is injected as parameter, use `@pytest.mark.usefixtures` instead +] + +[tool.ruff.lint.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 -[tool.ruff.per-file-ignores] -"setup.py" = ["D100", "SIM115"] -"__about__.py" = ["D100"] -"__init__.py" = ["D100"] -[tool.ruff.pydocstyle] -# Use Google-style docstrings. -convention = "google" +[tool.mypy] +files = [ + "src/lightning", +] +# This section is for folders with "-" as they are not valid python modules +exclude = [ +] +install_types = "True" +non_interactive = "True" +disallow_untyped_defs = "True" +ignore_missing_imports = "True" +show_error_codes = "True" +warn_redundant_casts = "True" +warn_unused_configs = "True" +warn_unused_ignores = "True" +allow_redefinition = "True" +# disable this rule as the Trainer attributes are defined in the connectors, not in its __init__ +disable_error_code = "attr-defined" +# style choices +warn_no_return = "False" + +# Ignore mypy errors for these files +# TODO: the goal is for this to be empty +[[tool.mypy.overrides]] +# the list can be generated with: +# mypy --no-error-summary 2>&1 | tr ':' ' ' | awk '{print $1}' | sort | uniq | sed 's/\.py//g; s|src/||g; s|\/|\.|g' | xargs -I {} echo '"{}",' +module = [ +] +ignore_errors = "True" -#[tool.ruff.pycodestyle] -#ignore-overlong-task-comments = true -[tool.ruff.mccabe] -# Unlike Flake8, default to a complexity level of 10. -max-complexity = 10 +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "warnings", + "pass", + "rank_zero_warn", + "raise NotImplementedError", +] + + +[tool.pytest.ini_options] +norecursedirs = [ + ".git", + ".github", + "dist", + "build", + "docs", +] +addopts = [ + "--strict-markers", + "--doctest-modules", + "--color=yes", + "--disable-pytest-warnings", + "--ignore=legacy/checkpoints", +] +markers = [ + "cloud: Run the cloud tests for example", +] +filterwarnings = [ + "error::FutureWarning", +] +xfail_strict = true +junit_duration_report = "call" diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 8d740a95..c2eb5e36 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,2 +1,5 @@ lightning-utilities >=0.8.0, <0.10.0 -lightning-cloud == 0.5.64 # Must be pinned to ensure compatibility \ No newline at end of file +lightning-cloud == 0.5.64 # Must be pinned to ensure compatibility +# to be able to include also PL 2.0 and preserve `>` needed for CI min version bypass +torch >0.14.0, <=2.2.0 +filelock \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt index 1b85876c..155d2939 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,7 +1,9 @@ -coverage>=5.0 -pytest>=6.0 -pytest-cov -mypy==1.8.0 +coverage ==7.3.1 +pytest ==7.4.0 +pytest-cov ==4.1.0 +pytest-timeout ==2.1.0 +pytest-rerunfailures ==12.0 +pytest-random-order ==1.1.0 viztracer pandas pyarrow