WIP, Lots of delinting with pylint

.github/workflows/testing-and-coverage.yml

@@ -32,6 +32,6 @@ jobs:
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Run unit tests with pytest
       run: |
-        python -m pytest tests --cov=core --cov-report=xml
+        python -m pytest tests --cov=rail --cov-report=xml
     - name: Upload coverage report to codecov
       uses: codecov/codecov-action@v3

pyproject.toml

@@ -64,6 +64,7 @@ disable = [
     "abstract-method",
     "invalid-name",
     "too-many-statements",
+    "too-many-arguments",
     "missing-module-docstring",
     "missing-class-docstring",
     "missing-function-docstring",

src/rail/cli/commands.py

@@ -11,50 +11,50 @@ def cli() -> None:
 
 
 @cli.command()
-@options.outdir(default='docs')
+@options.outdir(default="docs")
 @options.clear_output()
 @options.dry_run()
 @options.inputs()
 @options.skip()
-def render_nb(outdir, clear_output, dry_run, inputs, skip, **kwargs):
+def render_nb(outdir, clear_output, dry_run, inputs, skip, **_kwargs):
     """Render jupyter notebooks"""
     return scripts.render_nb(outdir, clear_output, dry_run, inputs, skip)
 
 
 @cli.command()
-@options.outdir(default='..')
+@options.outdir(default="..")
 @options.git_mode()
 @options.dry_run()
 @options.package_file()
-def clone_source(outdir, git_mode, dry_run, package_file, **kwargs):
+def clone_source(outdir, git_mode, dry_run, package_file, **_kwargs):
     """Install packages from source"""
     scripts.clone_source(outdir, git_mode, dry_run, package_file)
     return 0
-   
-    
+
+
 @cli.command()
-@options.outdir(default='..')
+@options.outdir(default="..")
 @options.dry_run()
 @options.package_file()
-def update_source(outdir, dry_run, package_file, **kwargs):
+def update_source(outdir, dry_run, package_file, **_kwargs):
     """Update packages from source"""
     scripts.update_source(outdir, dry_run, package_file)
     return 0
-    
-    
+
+
 @cli.command()
-@options.outdir(default='..')
+@options.outdir(default="..")
 @options.dry_run()
 @options.from_source()
 @options.package_file()
-def install(outdir, dry_run, from_source, package_file, **kwargs):
+def install(outdir, dry_run, from_source, package_file, **_kwargs):
     """Install rail packages one by one, to be fault tolerant"""
     scripts.install(outdir, from_source, dry_run, package_file)
     return 0
-    
+
 
 @cli.command()
-@options.outdir(default='..')
+@options.outdir(default="..")
 @options.print_all()
 @options.print_packages()
 @options.print_namespaces()

src/rail/cli/options.py

@@ -1,6 +1,6 @@
 import enum
-from functools import partial, wraps
-from typing import Any, Callable, Type, TypeVar, cast
+from functools import partial
+from typing import Any, Type, TypeVar
 
 import click
 
@@ -20,12 +20,13 @@
     "package_file",
     "skip",
     "inputs",
-    "verbose_download"
+    "verbose_download",
 ]
 
 
 class GitMode(enum.Enum):
     """Choose git clone mode"""
+
     ssh = 0
     https = 1
     cli = 2
@@ -37,11 +38,11 @@ class GitMode(enum.Enum):
 class EnumChoice(click.Choice):
     """A version of click.Choice specialized for enum types"""
 
-    def __init__(self, enum: EnumType_co, case_sensitive: bool = True) -> None:
-        self._enum = enum
-        super().__init__(list(enum.__members__.keys()), case_sensitive=case_sensitive)
+    def __init__(self, the_enum: EnumType_co, case_sensitive: bool = True) -> None:
+        self._enum = the_enum
+        super().__init__(list(the_enum.__members__.keys()), case_sensitive=case_sensitive)
 
-    def convert(self, value: Any, param, ctx) -> EnumType_co:
+    def convert(self, value: Any, param, ctx) -> EnumType_co:  # pragma: no cover
         converted_str = super().convert(value, param, ctx)
         return self._enum.__members__[converted_str]
 
@@ -50,19 +51,23 @@ class PartialOption:
     """Wraps click.option with partial arguments for convenient reuse"""
 
     def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
-        self._partial = partial(click.option, *param_decls, cls=partial(click.Option), **kwargs)
+        self._partial = partial(
+            click.option, *param_decls, cls=partial(click.Option), **kwargs
+        )
 
-    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:  # pragma: no cover
         return self._partial(*args, **kwargs)
 
 
 class PartialArgument:
     """Wraps click.argument with partial arguments for convenient reuse"""
 
     def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
-        self._partial = partial(click.argument, *param_decls, cls=partial(click.Argument), **kwargs)
+        self._partial = partial(
+            click.argument, *param_decls, cls=partial(click.Argument), **kwargs
+        )
 
-    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:  # pragma: no cover
         return self._partial(*args, **kwargs)
 
 
@@ -94,7 +99,7 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any:
 git_mode = PartialOption(
     "--git-mode",
     type=EnumChoice(GitMode),
-    default='ssh',
+    default="ssh",
     help="Git Clone mode",
 )
 
@@ -148,20 +153,15 @@ def __call__(self, *args: Any, **kwargs: Any) -> Any:
     help="Skip files",
 )
 
-inputs = PartialArgument(
-    "inputs",
-    nargs=-1
-)
+inputs = PartialArgument("inputs", nargs=-1)
 
 verbose_download = PartialOption(
-    "-v",
-    "--verbose",
-    help="Verbose output when downloading",
-    is_flag=True
+    "-v", "--verbose", help="Verbose output when downloading", is_flag=True
 )
 
 bpz_demo_data = PartialOption(
     "--bpz-demo-data",
-    help="Download data that is explicitly only for use in the bpz demo and nowhere else (it is dummy data that will not make sense)",
-    is_flag=True
+    help="Download data that is explicitly only for use in the bpz demo and nowhere else"
+    "(it is dummy data that will not make sense)",
+    is_flag=True,
 )

src/rail/cli/scripts.py

@@ -1,6 +1,4 @@
-import sys
 import os
-import glob
 import yaml
 
 import rail.stages
@@ -9,8 +7,7 @@
 from rail.core.utils import RAILDIR
 
 
-def render_nb(outdir, clear_output, dry_run, inputs, skip, **kwargs):
-
+def render_nb(outdir, clear_output, dry_run, inputs, skip, **_kwargs):
     command = "jupyter nbconvert"
     options = "--to html"
 
@@ -19,10 +16,10 @@ def render_nb(outdir, clear_output, dry_run, inputs, skip, **kwargs):
     for nb_file in inputs:
         if nb_file in skip:
             continue
-        subdir = os.path.dirname(nb_file).split('/')[-1]
+        subdir = os.path.dirname(nb_file).split("/")[-1]
         basename = os.path.splitext(os.path.basename(nb_file))[0]
-        outfile = os.path.join('..', '..', outdir, f"{subdir}/{basename}.html")
-        relpath = os.path.join(outdir, f'{subdir}')
+        outfile = os.path.join("..", "..", outdir, f"{subdir}/{basename}.html")
+        relpath = os.path.join(outdir, f"{subdir}")
 
         try:
             print(relpath)
@@ -45,23 +42,22 @@ def render_nb(outdir, clear_output, dry_run, inputs, skip, **kwargs):
     failed_notebooks = []
     for key, val in status.items():
         print(f"{key} {val}")
-        if val != 0:
+        if val != 0:  # pragma: no cover
             failed_notebooks.append(key)
 
-    if failed_notebooks:
+    if failed_notebooks:  # pragma: no cover
         raise ValueError(f"The following notebooks failed {str(failed_notebooks)}")
-
 
-def clone_source(outdir, git_mode, dry_run, package_file):
 
-    with open(package_file) as pfile:
+def clone_source(outdir, git_mode, dry_run, package_file):  # pragma: no cover
+    with open(package_file, encoding='utf-8') as pfile:
         package_dict = yaml.safe_load(pfile)
 
-    for key, val in package_dict.items():
+    for key, _val in package_dict.items():
         if os.path.exists(f"{outdir}/{key}"):
             print(f"Skipping existing {outdir}/{key}")
             continue
-            
+
         if git_mode == GitMode.ssh:
             com_line = f"git clone https://github.com/LSSTDESC/{key}.git {outdir}/{key}"
         elif git_mode == GitMode.https:
@@ -74,71 +70,67 @@ def clone_source(outdir, git_mode, dry_run, package_file):
         else:
             os.system(com_line)
 
-
-def update_source(outdir, dry_run, package_file):
 
-    with open(package_file) as pfile:
+def update_source(outdir, dry_run, package_file):
+    with open(package_file, encoding='utf-8') as pfile:
         package_dict = yaml.safe_load(pfile)
 
-    currentpath = os.path.abspath('.')    
-    for key, val in package_dict.items():
+    currentpath = os.path.abspath(".")
+    for key, _val in package_dict.items():
         abspath = os.path.abspath(f"{outdir}/{key}")
 
-        if os.path.exists(f"{outdir}/{key}") is not True:
+        if os.path.exists(f"{outdir}/{key}") is not True:  # pragma: no cover
             print(f"Package {outdir}/{key} does not exist!")
-            continue            
-                
+            continue
+
         com_line = f"cd {abspath} && git pull && cd {currentpath}"
 
         if dry_run:
             print(com_line)
-        else:
+        else:  # pragma: no cover
             os.system(com_line)
-
 
-def install(outdir, from_source, dry_run, package_file):
 
-    with open(package_file) as pfile:
+def install(outdir, from_source, dry_run, package_file):
+    with open(package_file, encoding='utf-8') as pfile:
         package_dict = yaml.safe_load(pfile)
 
     for key, val in package_dict.items():
-
         if not from_source:
             com_line = f"pip install {val}"
-        else:            
-            if not os.path.exists(f"{outdir}/{key}"):
+        else:
+            if not os.path.exists(f"{outdir}/{key}"):  # pragma: no cover
                 print(f"Skipping missing {outdir}/{key}")
                 continue
             com_line = f"pip install -e {outdir}/{key}"
-        
+
         if dry_run:
             print(com_line)
-        else:
+        else:  # pragma: no cover
             os.system(com_line)
 
 
 def info(**kwargs):
-
     rail.stages.import_and_attach_all()
 
-    print_all = kwargs.get('print_all', False)
-    if kwargs.get('print_packages') or print_all:
+    print_all = kwargs.get("print_all", False)
+    if kwargs.get("print_packages") or print_all:
         print("======= Printing RAIL packages ==============")
         RailEnv.print_rail_packages()
         print("\n\n")
-    if kwargs.get('print_namespaces') or print_all:
+    if kwargs.get("print_namespaces") or print_all:
         print("======= Printing RAIL namespaces ==============")
         RailEnv.print_rail_namespaces()
         print("\n\n")
-    if kwargs.get('print_modules') or print_all:
+    if kwargs.get("print_modules") or print_all:
         print("======= Printing RAIL modules ==============")
         RailEnv.print_rail_modules()
         print("\n\n")
-    if kwargs.get('print_tree') or print_all:
+    if kwargs.get("print_tree") or print_all:
         print("======= Printing RAIL source tree ==============")
         RailEnv.print_rail_namespace_tree()
         print("\n\n")
-    if kwargs.get('print_stages') or print_all:
+    if kwargs.get("print_stages") or print_all:
         print("======= Printing RAIL stages ==============")
         RailEnv.print_rail_stage_dict()
         print("\n\n")
@@ -168,7 +160,7 @@ def get_data(verbose, **kwargs):  # pragma: no cover
 
     data_files = standard_data_files
     if kwargs.get("bpz_demo_data"):
-        # The bpz demo data is quarantined into its own flag, as it contains some 
+        # The bpz demo data is quarantined into its own flag, as it contains some
         # non-physical features that would add systematics if run on any real data.
         # This data should NOT be used for any science with real data!
         data_files = bpz_data_files

src/rail/core/__init__.py

@@ -1,23 +1,27 @@
 """Core code for RAIL"""
 import pkgutil
+import os
 import setuptools
 import rail
-import os
+
+from .stage import RailPipeline, RailStage
+
+# from .utilPhotometry import PhotormetryManipulator, HyperbolicSmoothing, HyperbolicMagnitudes
+from .util_stages import ColumnMapper, RowSelector, TableConverter
+from .introspection import RailEnv
+from .point_estimation import PointEstimationMixin
+
 
 def find_version():
     """Find the version"""
     # setuptools_scm should install a
     # file _version alongside this one.
-    from . import _version
+    from . import _version  # pylint: disable=import-outside-toplevel
+
     return _version.version
 
+
 try:
     __version__ = find_version()
-except ImportError: # pragma: no cover
+except ImportError:  # pragma: no cover
     __version__ = "unknown"
-
-from .stage import RailPipeline, RailStage
-#from .utilPhotometry import PhotormetryManipulator, HyperbolicSmoothing, HyperbolicMagnitudes
-from .util_stages import ColumnMapper, RowSelector, TableConverter
-from .introspection import RailEnv
-from .point_estimation import PointEstimationMixin