diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e7a2f58c..9d5771ac 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -41,19 +41,28 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install -r requirements-dev.txt + ./scripts/version.sh shell: bash - - name: Run integrity checks + - name: pycodestyle run: | - ./scripts/version.sh pycodestyle --format=pylint archivist examples functests unittests + shell: bash + - name: pylint + run: | python3 -m pylint archivist examples functests unittests - black archivist examples unittests functests + shell: bash + - name: black + run: | modified=$(git status -s | wc -l) + black archivist examples unittests functests if [ $modified -gt 0 ] then echo "there are $modified files that must be reformatted" exit 1 fi + shell: bash + - name: unittests.sh + run: | ./scripts/unittests.sh shell: bash - name: Run type-hint checks diff --git a/archivist/asset.py b/archivist/asset.py index 0e2ad473..3e76bbd5 100644 --- a/archivist/asset.py +++ b/archivist/asset.py @@ -1,6 +1,7 @@ """Asset data class """ + from contextlib import suppress diff --git a/archivist/assets.py b/archivist/assets.py index 9101cef5..209e5a95 100644 --- a/archivist/assets.py +++ b/archivist/assets.py @@ -21,7 +21,6 @@ """ - from copy import deepcopy from logging import getLogger from typing import TYPE_CHECKING, Any diff --git a/archivist/compliance.py b/archivist/compliance.py index f87644f8..27ec03e2 100644 --- a/archivist/compliance.py +++ b/archivist/compliance.py @@ -21,7 +21,6 @@ """ - from logging import getLogger from typing import TYPE_CHECKING, Any diff --git a/archivist/compliance_policies.py b/archivist/compliance_policies.py index 5e9cd653..b7220946 100644 --- a/archivist/compliance_policies.py +++ b/archivist/compliance_policies.py @@ -25,7 +25,6 @@ """ - from copy import deepcopy from logging import getLogger from typing import TYPE_CHECKING, Any, Union diff --git a/archivist/compliance_policy_requests.py b/archivist/compliance_policy_requests.py index 1518749f..30b0c6e9 100644 --- a/archivist/compliance_policy_requests.py +++ b/archivist/compliance_policy_requests.py @@ -4,7 +4,6 @@ """ - from dataclasses import asdict, dataclass from .compliance_policy_type import CompliancePolicyType diff --git a/archivist/composite.py b/archivist/composite.py index a417df16..a4f58cdd 100644 --- a/archivist/composite.py +++ b/archivist/composite.py @@ -21,7 +21,6 @@ """ - from logging import getLogger from typing import TYPE_CHECKING diff --git a/archivist/confirmer.py b/archivist/confirmer.py index a19d9d84..ee90db91 100644 --- a/archivist/confirmer.py +++ b/archivist/confirmer.py @@ -1,7 +1,6 @@ """assets confirmer interface """ - from copy import deepcopy from logging import getLogger from typing import TYPE_CHECKING, Any, Union, overload @@ -54,23 +53,27 @@ def __on_giveup_confirmation(details: "Details"): @overload -def _wait_for_confirmation(self: "_AssetsRestricted", identity: str) -> "Asset": - ... # pragma: no cover +def _wait_for_confirmation( + self: "_AssetsRestricted", identity: str +) -> "Asset": ... # pragma: no cover @overload -def _wait_for_confirmation(self: "_AssetsPublic", identity: str) -> "Asset": - ... # pragma: no cover +def _wait_for_confirmation( + self: "_AssetsPublic", identity: str +) -> "Asset": ... # pragma: no cover @overload -def _wait_for_confirmation(self: "_EventsRestricted", identity: str) -> "Event": - ... # pragma: no cover +def _wait_for_confirmation( + self: "_EventsRestricted", identity: str +) -> "Event": ... # pragma: no cover @overload -def _wait_for_confirmation(self: "_EventsPublic", identity: str) -> "Event": - ... # pragma: no cover +def _wait_for_confirmation( + self: "_EventsPublic", identity: str +) -> "Event": ... # pragma: no cover @backoff.on_predicate( diff --git a/archivist/dictmerge.py b/archivist/dictmerge.py index e17b9a69..02d36af8 100644 --- a/archivist/dictmerge.py +++ b/archivist/dictmerge.py @@ -1,7 +1,6 @@ """Archivist dict deep merge """ - from copy import deepcopy from typing import Any diff --git a/archivist/events.py b/archivist/events.py index ab6bfa59..cc4ab3ab 100644 --- a/archivist/events.py +++ b/archivist/events.py @@ -22,7 +22,6 @@ """ - from copy import deepcopy from logging import getLogger from typing import TYPE_CHECKING, Any diff --git a/archivist/headers.py b/archivist/headers.py index 8ae55075..da1bbb6e 100644 --- a/archivist/headers.py +++ b/archivist/headers.py @@ -3,7 +3,6 @@ Manage headers allowing for upper/lower/canonicalize form """ - from requests import models diff --git a/archivist/locations.py b/archivist/locations.py index 0d95eb3d..a25102a0 100644 --- a/archivist/locations.py +++ b/archivist/locations.py @@ -22,7 +22,6 @@ """ - from contextlib import suppress from copy import deepcopy from logging import getLogger diff --git a/archivist/notebooks/Create Event with Verified Domain.ipynb b/archivist/notebooks/Create Event with Verified Domain.ipynb index 712ef264..2b843075 100644 --- a/archivist/notebooks/Create Event with Verified Domain.ipynb +++ b/archivist/notebooks/Create Event with Verified Domain.ipynb @@ -226,7 +226,7 @@ " # their arc_display_type.\n", " # So a mistake here can result in asset data being\n", " # under- or over-shared.\n", - " \"some_custom_attribute\": \"value\" # You can add any custom value as long as\n", + " \"some_custom_attribute\": \"value\", # You can add any custom value as long as\n", " # it does not start with arc_\n", " }\n", " # The first argument is the attributes of the asset\n", diff --git a/archivist/notebooks/Initialization and Credentials.ipynb b/archivist/notebooks/Initialization and Credentials.ipynb index 5c823065..a7159419 100644 --- a/archivist/notebooks/Initialization and Credentials.ipynb +++ b/archivist/notebooks/Initialization and Credentials.ipynb @@ -43,8 +43,8 @@ ], "source": [ "# Install the datatrails-python package\n", - "%pip install --upgrade datatrails-archivist\n", - "%pip install --upgrade python-dotenv" + "%python3 -m pip install --upgrade datatrails-archivist\n", + "%python3 -m pip install --upgrade python-dotenv" ] }, { diff --git a/archivist/or_dict.py b/archivist/or_dict.py index bcea5bee..06f95c30 100644 --- a/archivist/or_dict.py +++ b/archivist/or_dict.py @@ -5,7 +5,6 @@ Dictionaries where key is always 'or' and value is a list of strings """ - from typing import Any diff --git a/archivist/runner.py b/archivist/runner.py index 3e76e0a7..b5dc9292 100644 --- a/archivist/runner.py +++ b/archivist/runner.py @@ -3,7 +3,6 @@ """ - from collections import defaultdict from functools import partialmethod from json import dumps as json_dumps diff --git a/archivist/subjects.py b/archivist/subjects.py index 31668f9f..f9b44e7d 100644 --- a/archivist/subjects.py +++ b/archivist/subjects.py @@ -21,7 +21,6 @@ """ - from base64 import b64decode from json import loads as json_loads from logging import getLogger diff --git a/archivist/subjects_confirmer.py b/archivist/subjects_confirmer.py index 059808f2..00fe57c1 100644 --- a/archivist/subjects_confirmer.py +++ b/archivist/subjects_confirmer.py @@ -1,7 +1,6 @@ """assets confirmer interface """ - from logging import getLogger from typing import TYPE_CHECKING diff --git a/archivist/tenancies.py b/archivist/tenancies.py index 68b9c422..47c70cff 100644 --- a/archivist/tenancies.py +++ b/archivist/tenancies.py @@ -21,7 +21,6 @@ """ - from logging import getLogger from typing import TYPE_CHECKING diff --git a/archivist/utils.py b/archivist/utils.py index cf759be0..083b92ce 100644 --- a/archivist/utils.py +++ b/archivist/utils.py @@ -1,7 +1,6 @@ """Some convenience stuff """ - from logging import getLogger from typing import TYPE_CHECKING diff --git a/docs/notebooks/requirements.txt b/docs/notebooks/requirements.txt index f975959e..75d837a6 100644 --- a/docs/notebooks/requirements.txt +++ b/docs/notebooks/requirements.txt @@ -1,9 +1,10 @@ # # jupyter notebooks -ipython~=8.12 +ipython~=8.12.3 +notebook~=7.0.8 jupyter~=1.0 -jupyterlab~=4.0 jupyter-console~=6.6 jupyter-contrib-nbextensions~=0.7 +jupyter-nbextensions-configurator~=0.6 python-dotenv[cli]~=1.0 diff --git a/examples/access_policies_filter.py b/examples/access_policies_filter.py index 40450d81..7b380f5b 100644 --- a/examples/access_policies_filter.py +++ b/examples/access_policies_filter.py @@ -7,6 +7,7 @@ with suitable properties and attributes. """ + from os import getenv from warnings import filterwarnings diff --git a/examples/compliance_policies_since.py b/examples/compliance_policies_since.py index 68af36da..ef4c728f 100644 --- a/examples/compliance_policies_since.py +++ b/examples/compliance_policies_since.py @@ -6,6 +6,7 @@ with suitable properties and attributes. """ + from json import dumps as json_dumps from os import getenv from time import sleep diff --git a/examples/create_asset.py b/examples/create_asset.py index 5f6aa64c..fb8f51f4 100644 --- a/examples/create_asset.py +++ b/examples/create_asset.py @@ -40,7 +40,7 @@ def create_asset(arch): # their arc_display_type. # So a mistake here can result in asset data being # under- or over-shared. - "some_custom_attribute": "value" # You can add any custom value as long as + "some_custom_attribute": "value", # You can add any custom value as long as # it does not start with arc_ } # diff --git a/examples/create_event.py b/examples/create_event.py index 0f9d856b..80394d49 100644 --- a/examples/create_event.py +++ b/examples/create_event.py @@ -12,6 +12,7 @@ In both cases the verified domain name is displayed. """ + from json import dumps as json_dumps from os import getenv from warnings import filterwarnings @@ -112,7 +113,7 @@ def create_asset(arch): # their arc_display_type. # So a mistake here can result in asset data being # under- or over-shared. - "some_custom_attribute": "value" # You can add any custom value as long as + "some_custom_attribute": "value", # You can add any custom value as long as # it does not start with arc_ } # The first argument is the attributes of the asset diff --git a/examples/get_asset.py b/examples/get_asset.py index 9c0a77e2..2e8b7daf 100644 --- a/examples/get_asset.py +++ b/examples/get_asset.py @@ -6,6 +6,7 @@ The main function would then call arch.assets.read_by_signature() to get one asset from the instance. """ + from os import getenv from warnings import filterwarnings diff --git a/examples/subject_create.py b/examples/subject_create.py index 107c4fe5..db1ae3f4 100644 --- a/examples/subject_create.py +++ b/examples/subject_create.py @@ -7,6 +7,7 @@ 'arch' is then called with arch.subjects.create_from_b64() and the subject will be created. """ + from os import getenv from warnings import filterwarnings diff --git a/examples/subjects_filter.py b/examples/subjects_filter.py index 3e12d356..e566ecea 100644 --- a/examples/subjects_filter.py +++ b/examples/subjects_filter.py @@ -6,6 +6,7 @@ attributes. """ + from os import getenv from warnings import filterwarnings diff --git a/functests/constants.py b/functests/constants.py index 64f2af37..945d252f 100644 --- a/functests/constants.py +++ b/functests/constants.py @@ -1,6 +1,7 @@ """ hide docstringss """ + # pylint: disable=missing-docstring import unittest diff --git a/functests/execapplications.py b/functests/execapplications.py index 30022a93..aab2713a 100644 --- a/functests/execapplications.py +++ b/functests/execapplications.py @@ -1,6 +1,7 @@ """ Test applications """ + from copy import deepcopy from json import dumps as json_dumps from os import getenv diff --git a/functests/execattachments.py b/functests/execattachments.py index 84b122e6..63ead67d 100644 --- a/functests/execattachments.py +++ b/functests/execattachments.py @@ -1,6 +1,7 @@ """ Tests the upload and download functionality of the SDK """ + from contextlib import suppress from filecmp import clear_cache, cmp from io import BytesIO diff --git a/functests/execcompliance_policies.py b/functests/execcompliance_policies.py index 8a86c83a..b91134a4 100644 --- a/functests/execcompliance_policies.py +++ b/functests/execcompliance_policies.py @@ -1,6 +1,7 @@ """ Test compliance policies """ + from json import dumps as json_dumps from os import getenv from time import sleep diff --git a/functests/execnotebooks.py b/functests/execnotebooks.py index 58f90291..6fc50acb 100644 --- a/functests/execnotebooks.py +++ b/functests/execnotebooks.py @@ -1,6 +1,7 @@ """ Test subjects """ + from os import getenv from unittest import skip diff --git a/functests/execsubjects.py b/functests/execsubjects.py index bf06412a..a56efc81 100644 --- a/functests/execsubjects.py +++ b/functests/execsubjects.py @@ -1,6 +1,7 @@ """ Test subjects """ + from json import dumps as json_dumps from os import getenv from uuid import uuid4 diff --git a/pyproject.toml b/pyproject.toml index 09a5d563..9c331fb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -563,6 +563,10 @@ redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", # this will eventually be the de facto linter and pylint,pycodestyle etc will be removed. # we can add a host of checkers in extend-select but for now lets keep it simple [tool.ruff] +show-fixes = true +target-version = "py37" + +[tool.ruff.lint] extend-select = [ "C4", "E", @@ -571,7 +575,4 @@ extend-select = [ "TCH", ] ignore = ["E501"] -show-fixes = true -target-version = "py37" - diff --git a/requirements-dev.txt b/requirements-dev.txt index 01222d41..fa7d06e5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,22 +2,22 @@ # code quality autopep8~=2.0 -black[jupyter]~=23.9 -coverage[toml]~=7.3 -pip-audit~=2.6 -pycodestyle~=2.10 +black[jupyter]~=24.1 +coverage[toml]~=7.4 +pip-audit~=2.7 +pycodestyle~=2.11 pylint~=3.0 pyright~=1.1 -ruff~=0.1 +ruff~=0.2 unittest-xml-reporting~=3.2 -testbook~=0.4 +testbook~=0.3 # analyze dependencies pipdeptree~=2.13 # uploading to pypi build~=1.0 -twine~=4.0 +twine~=5.0 # for sbom.xml file xq~=0.0 diff --git a/requirements.txt b/requirements.txt index eb1a6444..f19704e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ backoff~=2.2.1 certifi flatten-dict~=0.4 -iso8601~=2.0 +iso8601~=2.1 Jinja2~=3.1 pyaml-env~=1.2 requests~=2.31.0 diff --git a/scripts/builder.sh b/scripts/builder.sh index 499a2d89..5eb17f25 100755 --- a/scripts/builder.sh +++ b/scripts/builder.sh @@ -21,7 +21,10 @@ fi docker run \ --rm ${USE_TTY} \ -v $(pwd):/home/builder \ + -w /home/builder \ -u $(id -u):$(id -g) \ + -e USER=$(id -u -n) \ + -e HOME=/home/builder \ -e FUNCTEST \ -e UNITTEST \ -e DATATRAILS_UNIQUE_ID \ @@ -38,3 +41,5 @@ docker run \ -e GITHUB_REF \ datatrails-python-builder \ "$@" + + diff --git a/unittests/__init__.py b/unittests/__init__.py index 782379d1..1ada2a3b 100644 --- a/unittests/__init__.py +++ b/unittests/__init__.py @@ -1,6 +1,7 @@ """ Unit tests """ + import unittest # Hides Docstring diff --git a/unittests/testnotebooks.py b/unittests/testnotebooks.py index b3ff4e0d..04561dc3 100644 --- a/unittests/testnotebooks.py +++ b/unittests/testnotebooks.py @@ -1,6 +1,7 @@ """ Test notebooks """ + import os from unittest import TestCase diff --git a/unittests/testrunner.py b/unittests/testrunner.py index 4ed9d366..35c14a22 100644 --- a/unittests/testrunner.py +++ b/unittests/testrunner.py @@ -1,6 +1,7 @@ """ Test runner """ + from logging import getLogger from os import environ from unittest import TestCase diff --git a/unittests/testrunneractiomap.py b/unittests/testrunneractiomap.py index c8ef04f8..4ddf73cf 100644 --- a/unittests/testrunneractiomap.py +++ b/unittests/testrunneractiomap.py @@ -1,6 +1,7 @@ """ Test runner actionmap """ + from logging import getLogger from os import environ from unittest import TestCase diff --git a/unittests/testrunnerassets.py b/unittests/testrunnerassets.py index ebd4aae8..b723c0d6 100644 --- a/unittests/testrunnerassets.py +++ b/unittests/testrunnerassets.py @@ -1,6 +1,7 @@ """ Test runner assets """ + from logging import getLogger from os import environ from unittest import TestCase, mock diff --git a/unittests/testrunnercompliance.py b/unittests/testrunnercompliance.py index 3b609d5e..023aea45 100644 --- a/unittests/testrunnercompliance.py +++ b/unittests/testrunnercompliance.py @@ -1,6 +1,7 @@ """ Test runner compliance """ + from logging import getLogger from os import environ from unittest import TestCase, mock diff --git a/unittests/testrunnerlocation.py b/unittests/testrunnerlocation.py index b5ab1592..388ad7e3 100644 --- a/unittests/testrunnerlocation.py +++ b/unittests/testrunnerlocation.py @@ -1,6 +1,7 @@ """ Test runner location """ + from logging import getLogger from os import environ from unittest import TestCase, mock diff --git a/unittests/testrunnerstep.py b/unittests/testrunnerstep.py index d0b1d632..2dc93ab6 100644 --- a/unittests/testrunnerstep.py +++ b/unittests/testrunnerstep.py @@ -1,6 +1,7 @@ """ Test runner step """ + from logging import getLogger from os import environ from unittest import TestCase