diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..598b818b --- /dev/null +++ b/.coveragerc @@ -0,0 +1,4 @@ +[run] +omit = + tests/* + */__init__.py diff --git a/.editorconfig b/.editorconfig deleted file mode 100644 index 69f4298e..00000000 --- a/.editorconfig +++ /dev/null @@ -1,21 +0,0 @@ -# https://editorconfig.org - -root = true - -[*] -indent_style = space -indent_size = 4 -trim_trailing_whitespace = true -insert_final_newline = true -charset = utf-8 -end_of_line = lf - -[*.bat] -indent_style = tab -end_of_line = crlf - -[LICENSE] -insert_final_newline = false - -[Makefile] -indent_style = tab diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 8166c398..00000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length = 120 -ignore = E203, W503 diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 00000000..0aa82eef --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,38 @@ +[mypy] +# Adapted from this StackOverflow post: +# https://stackoverflow.com/questions/55944201/python-type-hinting-how-do-i-enforce-that-project-wide +python_version = 3.11 + +# This flag enhances the user feedback for error messages +pretty = True + +# Disallow dynamic typing +disallow_any_unimported = True +disallow_any_expr = True +disallow_any_decorated = True +disallow_any_generics = True +disallow_any_explicit = True +disallow_subclassing_any = True + +# Disallow untyped definitions and calls +disallow_untyped_calls = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True + +# None and optional handling +no_implicit_optional = True + +# Configuring warnings +warn_unused_ignores = True +warn_no_return = True +warn_return_any = True +warn_redundant_casts = True +warn_unreachable = True + +# Misc things +strict_equality = True + +# Config file +warn_unused_configs = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1a7fd4f6..e04836f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,12 +2,23 @@ # pre-commit run --all-files # Update this file: # pre-commit autoupdate -exclude: tests/test_aux_files/.* +# TODO remove when other files are updated to pass. NOTE: keep the first line +# as-is. +exclude: | + (?x)^( + tests/test_aux_files/.*| + anaconda_linter/.*| + docs/.*| + scripts/.*| + setup.py| + tests/.* + )$ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: check-added-large-files + exclude_types: ["json", "yaml"] - id: check-ast - id: fix-byte-order-marker - id: check-case-conflict @@ -21,31 +32,28 @@ repos: - id: end-of-file-fixer - id: mixed-line-ending - id: trailing-whitespace + # Notes: + # - By default, `black` uses a line-length of 88. + # - `isort` will conflict with `black`, if not properly configured - repo: https://github.com/PyCQA/isort rev: 5.12.0 hooks: - id: isort + args: ["--profile", "black", --line-length=120] - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.0 hooks: - - id: black - - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 + - id: black + args: [--line-length=120] + - repo: https://github.com/PyCQA/pylint + rev: v3.0.1 hooks: - - id: pyupgrade - args: [--py37-plus, --keep-runtime-typing] - - repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 + - id: pylint + args: ["--rcfile=.pylintrc"] - repo: https://github.com/lovesegfault/beautysh rev: v6.2.1 hooks: - id: beautysh - - repo: https://github.com/koalaman/shellcheck-precommit - rev: v0.9.0 - hooks: - - id: shellcheck - repo: https://github.com/python-jsonschema/check-jsonschema rev: 0.27.0 hooks: @@ -57,7 +65,26 @@ repos: hooks: - id: codespell args: [--write] - - repo: meta + - repo: local hooks: - - id: check-hooks-apply - - id: check-useless-excludes + # Run shellcheck without needing `docker` to run + - id: shellcheck + name: shellcheck + description: Test shell scripts with shellcheck + entry: shellcheck + language: python + types: [shell] + require_serial: true # shellcheck can detect sourcing this way + # Run tests through `make` to be consistent with manually run commands. + - id: pytest + language: system + name: pytest + always_run: true + pass_filenames: false + entry: make test + - id: pytest-cov + language: system + name: pytest-cov + always_run: true + pass_filenames: false + entry: make test-cov diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..ede36159 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,440 @@ +# This Pylint rcfile contains a best-effort configuration to uphold the +# best-practices and style described in the Google Python style guide: +# https://google.github.io/styleguide/pyguide.html +# +# Its canonical open-source location is: +# https://google.github.io/styleguide/pylintrc + +# This has been adapted for use with Anaconda. +# Changes will be marked with `Anaconda Delta:` + +# Anaconda Delta: use newer, more inclusive tag for "primary" section +[MAIN] + +# Files or directories to be skipped. They should be base names, not paths. +ignore=third_party + +# Files or directories matching the regex patterns are skipped. The regex +# matches against base names, not paths. +ignore-patterns= + +# Pickle collected data for later comparisons. +persistent=no + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +# Anaconda Delta: Added docparams extension +load-plugins=pylint.extensions.docparams + +# Use multiple processes to speed up Pylint. +# Anaconda Delta: bumped from 4 to 8 jobs +jobs=8 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +# Anaconda Delta: enabled stricter rules around docstrings +enable=missing-function-docstring, + missing-param-doc, + missing-raises-doc, + missing-any-param-doc, + missing-return-doc, + redundant-returns-doc, + empty-docstring + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=abstract-method, + apply-builtin, + arguments-differ, + attribute-defined-outside-init, + backtick, + bad-option-value, + basestring-builtin, + buffer-builtin, + c-extension-no-member, + consider-using-enumerate, + cmp-builtin, + cmp-method, + coerce-builtin, + coerce-method, + delslice-method, + div-method, + duplicate-code, + eq-without-hash, + execfile-builtin, + file-builtin, + filter-builtin-not-iterating, + fixme, + getslice-method, + global-statement, + hex-method, + idiv-method, + implicit-str-concat, + import-error, + import-self, + import-star-module-level, + inconsistent-return-statements, + input-builtin, + intern-builtin, + invalid-str-codec, + locally-disabled, + long-builtin, + long-suffix, + map-builtin-not-iterating, + misplaced-comparison-constant, + metaclass-assignment, + next-method-called, + next-method-defined, + no-absolute-import, + no-else-break, + no-else-continue, + no-else-raise, + no-else-return, + no-init, # added + no-member, + no-name-in-module, + no-self-use, + nonzero-method, + oct-method, + old-division, + old-ne-operator, + old-octal-literal, + old-raise-syntax, + parameter-unpacking, + print-statement, + raising-string, + range-builtin-not-iterating, + raw_input-builtin, + rdiv-method, + reduce-builtin, + relative-import, + reload-builtin, + round-builtin, + setslice-method, + signature-differs, + standarderror-builtin, + suppressed-message, + sys-max-int, + too-few-public-methods, + too-many-ancestors, + too-many-arguments, + too-many-boolean-expressions, + too-many-branches, + too-many-instance-attributes, + too-many-locals, + too-many-nested-blocks, + too-many-public-methods, + too-many-return-statements, + too-many-statements, + trailing-newlines, + unichr-builtin, + unicode-builtin, + unnecessary-pass, + unpacking-in-except, + useless-else-on-loop, + useless-object-inheritance, + useless-suppression, + using-cmp-argument, + wrong-import-order, + xrange-builtin, + zip-builtin-not-iterating, + + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[BASIC] + +# Good variable names which should always be accepted, separated by a comma +good-names=main,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl + +# Regular expression matching correct function names +function-rgx=^(?:(?PsetUp|tearDown|setUpModule|tearDownModule)|(?P_?[A-Z][a-zA-Z0-9]*)|(?P_?[a-z][a-z0-9_]*))$ + +# Regular expression matching correct variable names +variable-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct constant names +const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct attribute names +attr-rgx=^_{0,2}[a-z][a-z0-9_]*$ + +# Regular expression matching correct argument names +argument-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=^[a-z][a-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=^_?[A-Z][a-zA-Z0-9]*$ + +# Regular expression matching correct module names +module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$ + +# Regular expression matching correct method names +method-rgx=(?x)^(?:(?P_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P_{0,2}[a-z][a-z0-9_]*))$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +# Anaconda Delta: Require more docstrings +no-docstring-rgx=(test.*|.*test|.*Test)$ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=10 + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis. It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + + +[FORMAT] + +# Maximum number of characters on a single line. +# Anaconda Delta: a number of our team members asked for 120. +max-line-length=120 + +# TODO(https://github.com/PyCQA/pylint/issues/3352): Direct pylint to exempt +# lines made too long by directives to pytype. + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=(?x)( + ^\s*(\#\ )??$| + ^\s*(from\s+\S+\s+)?import\s+.+$) + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=yes + +# Maximum number of lines in a module +max-module-lines=99999 + +# Anaconda Delta: enforce 4 spaces, not 2 +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=TODO + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=yes + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_) + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging,absl.logging,tensorflow.io.logging + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub, + TERMIOS, + Bastion, + rexec, + sets + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant, absl + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict, + _fields, + _replace, + _source, + _make + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls, + class_ + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=builtins.StandardError, + builtins.Exception, + builtins.BaseException diff --git a/Makefile b/Makefile index 4346aca2..2e01be54 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,16 @@ -.PHONY: clean clean-build clean-pyc clean-test coverage dist docs help install +# The `.ONESHELL` and setting `SHELL` allows us to run commands that require +# `conda activate` +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -o pipefail -o errexit +CONDA_ACTIVATE = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate ; conda activate +# Resolve the path to `python3` and store it in a variable. Always invoke `python` using this variable. This prevents a +# nasty scenario where the GitHub Actions container fails to find `python` or `python3`, when running `make` commands. +PYTHON3 := $(shell which python3) #$(shell type python3 | awk '{ print $3 }') + +.PHONY: clean clean-cov clean-build clean-env clean-pyc clean-test dist docs help install environment release dev test test-debug test-cov pre-commit lint format analyze .DEFAULT_GOAL := help -SHELL := /bin/bash -o pipefail -o errexit - CONDA_ENV_NAME ?= anaconda-linter # Folder for all the build artifacts to be archived by CI. @@ -30,7 +38,19 @@ export PRINT_HELP_PYSCRIPT BROWSER := python -c "$$BROWSER_PYSCRIPT" -clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts +# For now, most tools only run on new files, not the entire project. +LINTER_FILES := tests/*.py +# Tracks all python files. This will eventually be used by the auto formatter, linter, and static analyzer. +ALL_PY_FILES := anaconda_linter/**/*.py scripts/*.py tests/*.py + +clean: clean-cov clean-build clean-env clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-cov: + @rm -rf .coverage + @rm -rf htmlcov + @rm -rf reports/{*.html,*.png,*.js,*.css,*.json} + @rm -rf pytest.xml + @rm -rf pytest-coverage.txt clean-build: ## remove build artifacts rm -fr build/ @@ -39,6 +59,9 @@ clean-build: ## remove build artifacts find . -name '*.egg-info' -exec rm -fr {} + find . -name '*.egg' -exec rm -f {} + +clean-env: ## remove conda environment + conda remove -y -n $(CONDA_ENV_NAME) --all + clean-pyc: ## remove Python file artifacts find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + @@ -52,33 +75,16 @@ clean-test: ## remove test and coverage artifacts rm -fr artifacts/ rm -fr .pytest_cache -coverage: ## check code coverage quickly with the default Python - coverage run --source anaconda_linter -m pytest - coverage report -m - coverage html - $(BROWSER) htmlcov/index.html - help: - @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + $(PYTHON3) -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +install: clean ## install the package to the active Python's site-packages + pip install . environment: ## Handles environment creation conda env create -f environment.yaml --name $(CONDA_ENV_NAME) --force conda run --name $(CONDA_ENV_NAME) pip install -e . -pre-commit: ## Runs pre-commit against files - pre-commit run --all-files - -test: ## Run tests - mkdir -p $(ARTIFACTS_PATH) - python -m pytest \ - -n auto \ - --junit-xml="$(ARTIFACTS_PATH)/test-report.xml" \ - --html="$(ARTIFACTS_PATH)/test-report.html" \ - --cov \ - --cov-report html --cov-report html:$(ARTIFACTS_PATH)/cov_html \ - --cov-report xml --cov-report xml:$(ARTIFACTS_PATH)/cobertura.xml \ - --show-capture=all -vv ./tests - release: dist ## package and upload a release twine upload dist/* @@ -86,8 +92,29 @@ dist: clean ## builds source and wheel package python -m build ls -l dist -install: clean ## install the package to the active Python's site-packages - pip install . - dev: clean ## install the package's development version - pip install -e . + conda env create -f environment.yaml --name $(CONDA_ENV_NAME) --force + conda run --name $(CONDA_ENV_NAME) pip install -e . + $(CONDA_ACTIVATE) percy && pre-commit install + +pre-commit: ## runs pre-commit against files. NOTE: older files are disabled in the pre-commit config. + pre-commit run --all-files + +test: ## runs test cases + $(PYTHON3) -m pytest -n auto --capture=no tests/ + +test-debug: ## runs test cases with debugging info enabled + $(PYTHON3) -m pytest -n auto -vv --capture=no tests/ + +test-cov: ## checks test coverage requirements + $(PYTHON3) -m pytest -n auto --cov-config=.coveragerc --cov=anaconda_linter tests/ --cov-fail-under=80 --cov-report term-missing + +lint: ## runs the linter against the project + pylint --rcfile=.pylintrc $(LINTER_FILES) + +format: ## runs the code auto-formatter + isort --profile black --line-length=120 $(LINTER_FILES) + black --line-length=120 $(LINTER_FILES) + +analyze: ## runs static analyzer on the project + mypy --config-file=.mypy.ini $(LINTER_FILES) diff --git a/environment.yaml b/environment.yaml index 97139c6f..cc7926bc 100644 --- a/environment.yaml +++ b/environment.yaml @@ -3,7 +3,7 @@ channels: - defaults dependencies: - - python + - python >=3.8 # build - setuptools - pip @@ -17,9 +17,14 @@ dependencies: - networkx - requests # test - - make - pytest - pre-commit + # developer tools + - black + - isort + - mypy + - pylint + - shellcheck # plug ins - pytest-cov - pytest-html diff --git a/py.typed b/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/pyproject.toml b/pyproject.toml index cb525b3c..60b621a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,7 @@ -[tool.black] -line-length = 100 -target-version = ['py37', 'py38', 'py39'] +# TODO Complete later +# [tool.setuptools.package-data] +# "anaconda-linter" = ["py.typed"] -[tool.isort] -profile = "black" -line_length = 100 -src_paths = ["anaconda_linter"] - -[tool.pytest.ini_options] -filterwarnings = [ - # Skip PendingDeprecationWarning from conda-build - "ignore:.*`conda\\.exports\\.memoized`.*:PendingDeprecationWarning", - "ignore:.*`conda\\._vendor\\.toolz`.*:PendingDeprecationWarning", -] - -[tool.coverage.run] -omit = [ - "anaconda_linter/run.py", -] +# [project] +# name = "anaconda-linter" +# version = "v0.1.1"