global: add pre-commit with ruff

.github/workflows/push-master.yml

@@ -16,4 +16,3 @@ jobs:
     needs: [python2_tests, python3_tests]
     uses: ./.github/workflows/bump-and-publish.yml
     secrets: inherit
-

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: fix-byte-order-marker
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: [ --pytest-test-first ]
+        exclude: '^(?!factories/)'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.6
+    hooks:
+      - id: ruff
+        args: [ --fix ]

inspire_utils/config.py

@@ -29,8 +29,8 @@
 from __future__ import absolute_import, division, print_function
 
 import os
-import six
 
+import six
 
 DEFAULT_CONFIG_PATHS = (
     './var/inspirehep-instance/inspirehep.cfg',

inspire_utils/helpers.py

@@ -22,9 +22,8 @@
 
 from __future__ import absolute_import, division, print_function
 
-from lxml import etree
-
 import six
+from lxml import etree
 
 
 def force_list(data):

inspire_utils/name.py

@@ -23,16 +23,16 @@
 from __future__ import absolute_import, division, print_function
 
 import itertools
-from itertools import product, chain
 import re
+from itertools import chain, product
 
+import six
 from nameparser import HumanName
 from nameparser.config import Constants
-import six
 from unidecode import unidecode
 
-from .logging import getStackTraceLogger
-from .query import wrap_queries_in_bool_clauses_if_more_than_one
+from inspire_utils.logging import getStackTraceLogger
+from inspire_utils.query import wrap_queries_in_bool_clauses_if_more_than_one
 
 LOGGER = getStackTraceLogger(__name__)
 

inspire_utils/parsers/arxiv.py

@@ -23,22 +23,28 @@
 
 from __future__ import absolute_import, division, print_function
 
-from itertools import chain
 import re
+from itertools import chain
 
 import six
 from inspire_schemas.api import LiteratureBuilder
 from inspire_schemas.utils import classify_field, normalize_arxiv_category
-from ..dedupers import dedupe_list
-from ..helpers import maybe_int
 from pylatexenc.latex2text import (
     EnvironmentTextSpec,
     LatexNodes2Text,
     MacroTextSpec,
     get_default_latex_context_db,
 )
 
-from ..utils import CONFERENCE_WORDS, THESIS_WORDS, coll_cleanforthe, get_node, split_fullname
+from inspire_utils.dedupers import dedupe_list
+from inspire_utils.helpers import maybe_int
+from inspire_utils.utils import (
+    CONFERENCE_WORDS,
+    THESIS_WORDS,
+    coll_cleanforthe,
+    get_node,
+    split_fullname,
+)
 
 RE_CONFERENCE = re.compile(
     r'\b(%s)\b' % '|'.join(

inspire_utils/parsers/author_xml.py

@@ -29,10 +29,10 @@
 
 import re
 
+from inspire_schemas.api import LiteratureBuilder
 from scrapy.selector import Selector
 from six import binary_type
 from six.moves import zip
-from inspire_schemas.api import LiteratureBuilder
 
 from inspire_utils.name import normalize_name
 

inspire_utils/parsers/crossref.py

@@ -23,11 +23,13 @@
 from __future__ import absolute_import, division, print_function
 
 import itertools
+
 from inspire_schemas.api import LiteratureBuilder, ReferenceBuilder
-from ..date import PartialDate
-from ..helpers import force_list
-from ..record import get_value
-from ..dedupers import dedupe_list_of_dicts
+
+from inspire_utils.date import PartialDate
+from inspire_utils.dedupers import dedupe_list_of_dicts
+from inspire_utils.helpers import force_list
+from inspire_utils.record import get_value
 
 """Document types for the crossref objects have been extracted
 from the following link: https://api.crossref.org/v1/types

inspire_utils/parsers/elsevier.py

@@ -27,10 +27,10 @@
 
 import six
 from inspire_schemas.api import LiteratureBuilder, ReferenceBuilder
-from ..date import PartialDate
-from ..helpers import maybe_int, remove_tags
 
-from ..utils import get_node
+from inspire_utils.date import PartialDate
+from inspire_utils.helpers import maybe_int, remove_tags
+from inspire_utils.utils import get_node
 
 DOCTYPE_MAPPING = {
     "abs": "abstract",
@@ -669,13 +669,11 @@ def get_identifier(self):
         return self.dois[0]["doi"]
 
     def should_record_be_harvested(self):
-        if self.article_type in DOCTYPES_TO_HARVEST and all(
+        return self.article_type in DOCTYPES_TO_HARVEST and all(
             [
                 self.title,
                 self.journal_title,
                 self.journal_volume,
                 (self.artid or self.page_start),
             ]
-        ):
-            return True
-        return False
+        )

inspire_utils/parsers/jats.py

@@ -27,15 +27,13 @@
 import itertools
 
 import six
-
 from idutils import normalize_orcid
 from inspire_schemas.api import LiteratureBuilder, ReferenceBuilder
 from inspire_schemas.utils import split_page_artid
-from ..date import PartialDate
-from ..helpers import maybe_int, remove_tags
-
-from ..utils import get_node
 
+from inspire_utils.date import PartialDate
+from inspire_utils.helpers import maybe_int, remove_tags
+from inspire_utils.utils import get_node
 
 JOURNAL_TITLES_MAPPING = {
     "Physics": "APS Physics"
@@ -202,10 +200,7 @@ def dois(self):
 
     @property
     def document_type(self):
-        if self.is_conference_paper:
-            document_type = 'conference paper'
-        else:
-            document_type = 'article'
+        document_type = "conference paper" if self.is_conference_paper else "article"
 
         return document_type
 
@@ -608,7 +603,7 @@ def get_reference(self, ref_node):
                     builder.add_report_number
                 ),
                 ('./article-title/text()', builder.add_title),
-                ('../label/text()', lambda x: builder.set_label(x.strip('[].')))
+                ('../label/text()', lambda x, builder=builder: builder.set_label(x.strip('[].')))
             ]
 
             for xpath, field_handler in fields:

inspire_utils/record.py

@@ -26,7 +26,7 @@
 
 from six import string_types
 
-from .logging import getStackTraceLogger
+from inspire_utils.logging import getStackTraceLogger
 
 LOGGER = getStackTraceLogger(__name__)
 SPLIT_KEY_PATTERN = re.compile(r"\.|\[")

inspire_utils/urls.py

@@ -25,7 +25,7 @@
 from __future__ import absolute_import, division, print_function
 
 from six import text_type
-from six.moves.urllib.parse import urlsplit, urlunsplit, SplitResult
+from six.moves.urllib.parse import SplitResult, urlsplit, urlunsplit
 
 
 def ensure_scheme(url, default_scheme='http'):

inspire_utils/utils.py

@@ -3,9 +3,10 @@
     division,
     print_function,
 )
+
 import re
-from scrapy.selector import Selector
 
+from scrapy.selector import Selector
 
 RE_FOR_THE = re.compile(
     r'\b(?:for|on behalf of|representing)\b',

ruff.toml

@@ -0,0 +1,29 @@
+target-version = "py311"
+[lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[lint]
+select = [
+    # pycodestyle
+    "E",
+    # Pyflakes
+    "F",
+    # flake8-bugbear
+    "B",
+    # flake8-simplify
+    "SIM",
+    # isort
+    "I",
+    # flake8-tidy-imports
+    "TID",
+    # flake8-pytest-style
+    "PT",
+]
+ignore = ["B904", "E501"]
+
+
+[lint.pycodestyle]
+ignore-overlong-task-comments = true
+
+[lint.pydocstyle]
+convention = "google"

run-tests.sh

@@ -21,6 +21,4 @@
 # or submit itself to any jurisdiction.
 
 set -e
-
-flake8 inspire_utils tests
 py.test tests