Skip to content

Commit

Permalink
feat!: much better typing
Browse files Browse the repository at this point in the history
  • Loading branch information
@kiyoon
kiyoon committed Oct 18, 2024
1 parent 0292164 commit 31970a1
Show file tree
Hide file tree
Showing 12 changed files with 14,276 additions and 145 deletions.
112 changes: 112 additions & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
[tool.pytest.ini_options]
addopts = "--cov=reduce_binary" # CHANGE (name of the importing module name)
testpaths = ["tests"]

[tool.ruff]
src = ["python/src"] # for ruff isort
namespace-packages = ["tools", "scripts"] # for INP rule, suppress on these directories

[tool.ruff.lint]
# OPTIONALLY ADD MORE LATER
select = [
# flake8
"E",
"F",
"W",
"B", # Bugbear
"D", # Docstring
"D213", # Multi-line docstring summary should start at the second line (replace D212)
"N", # Naming
"C4", # flake8-comprehensions
"UP", # pyupgrade
"SIM", # simplify
"RUF", # ruff-specific
"RET501", # return
"RET502", # return
"RET503", # return
"PTH", # path
"NPY", # numpy
"PD", # pandas
"PYI", # type stubs for pyright/pylance
"PT", # pytest
"PIE", #
"LOG", # logging
"COM818", # comma misplaced
"COM819", # comma
"DTZ", # datetime
"YTT",
"ASYNC",
"FBT", # boolean trap
"A", # Shadowing python builtins
"EXE", # executable (shebang)
"FA", # future annotations
"ISC", # Implicit string concatenation
"ICN", # Import convention
"INP", # Implicit namespace package (no __init__.py)
"Q", # Quotes
"RSE", # raise
"SLOT", # __slots__
"PL", # Pylint
"TRY", # try
"FAST", # FastAPI
"AIR", # airflow
"DOC", # docstring

# Not important
"T10", # debug statements
"T20", # print statements
]

ignore = [
"E402", # Module level import not at top of file
"W293", # Blank line contains whitespace
"W291", # Trailing whitespace
"D10", # Missing docstring in public module / function / etc.
"D200", # One-line docstring should fit on one line with quotes
"D212", # Multi-line docstring summary should start at the first line
"D417", # require documentation for every function parameter.
"D401", # require an imperative mood for all docstrings.
"DOC201", # missing Return field in docstring
"PTH123", # Path.open should be used instead of built-in open
"PT006", # Pytest parameterize style
"N812", # Lowercase `functional` imported as non-lowercase `F` (import torch.nn.functional as F)
"NPY002", # legacy numpy random
"UP017", # datetime.timezone.utc -> datetime.UTC
"SIM108", # use ternary operator instead of if-else
"TRY003", # long message in except
]

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.ruff.lint.pycodestyle]
# Black or ruff will enforce line length to be 88, except for docstrings and comments.
# We set it to 120 so we have more space for docstrings and comments.
max-line-length = 120

[tool.ruff.lint.isort]
# combine-as-imports = true
known-third-party = ["wandb"]

## Uncomment this if you want to use Python < 3.10
required-imports = [
"from __future__ import annotations",
]

[tool.ruff.lint.flake8-tidy-imports]
# Ban certain modules from being imported at module level, instead requiring
# that they're imported lazily (e.g., within a function definition, if TYPE_CHECKING, etc.)
# NOTE: Ruff code TID is currently disabled, so this settings doesn't do anything.
banned-module-level-imports = ["torch", "tensorflow"]

[tool.ruff.lint.pylint]
max-args = 10
max-bool-expr = 10
max-statements = 100

[tool.pyright]
include = ["python/src"]

typeCheckingMode = "standard"
useLibraryCodeForTypes = true
autoImportCompletions = true
114 changes: 0 additions & 114 deletions python/pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,117 +32,3 @@ keywords = ["biology"]
[tool.setuptools.packages.find]
where = ["src"]

[tool.pytest.ini_options]
addopts = "--cov=ml_project" # CHANGE (name of the importing module name)
testpaths = ["tests"]

[tool.ruff]
src = ["src"] # for ruff isort
namespace-packages = ["tools", "scripts"] # for INP rule, suppress on these directories

[tool.ruff.lint]
# OPTIONALLY ADD MORE LATER
select = [
# flake8
"E",
"F",
"W",
"B", # Bugbear
"D", # Docstring
"D213", # Multi-line docstring summary should start at the second line (replace D212)
"N", # Naming
"C4", # flake8-comprehensions
"UP", # pyupgrade
"SIM", # simplify
"RUF", # ruff-specific
"RET501", # return
"RET502", # return
"RET503", # return
"PTH", # path
"NPY", # numpy
"PD", # pandas
"PYI", # type stubs for pyright/pylance
"PT", # pytest
"PIE", #
"LOG", # logging
"COM818", # comma misplaced
"COM819", # comma
"DTZ", # datetime
"YTT",
"ASYNC",
"FBT", # boolean trap
"A", # Shadowing python builtins
"EXE", # executable (shebang)
"FA", # future annotations
"ISC", # Implicit string concatenation
"ICN", # Import convention
"INP", # Implicit namespace package (no __init__.py)
"Q", # Quotes
"RSE", # raise
"SLOT", # __slots__
"PL", # Pylint
"TRY", # try
"FAST", # FastAPI
"AIR", # airflow
"DOC", # docstring

# Not important
"T10", # debug statements
"T20", # print statements
]

ignore = [
"E402", # Module level import not at top of file
"W293", # Blank line contains whitespace
"W291", # Trailing whitespace
"D10", # Missing docstring in public module / function / etc.
"D200", # One-line docstring should fit on one line with quotes
"D212", # Multi-line docstring summary should start at the first line
"D417", # require documentation for every function parameter.
"D401", # require an imperative mood for all docstrings.
"DOC201", # missing Return field in docstring
"PTH123", # Path.open should be used instead of built-in open
"PT006", # Pytest parameterize style
"N812", # Lowercase `functional` imported as non-lowercase `F` (import torch.nn.functional as F)
"NPY002", # legacy numpy random
"UP017", # datetime.timezone.utc -> datetime.UTC
"SIM108", # use ternary operator instead of if-else
"TRY003", # long message in except
]

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.ruff.lint.pycodestyle]
# Black or ruff will enforce line length to be 88, except for docstrings and comments.
# We set it to 120 so we have more space for docstrings and comments.
max-line-length = 120

[tool.ruff.lint.isort]
# combine-as-imports = true
known-third-party = ["wandb"]

## Uncomment this if you want to use Python < 3.10
# required-imports = [
# "from __future__ import annotations",
# ]

[tool.ruff.lint.flake8-tidy-imports]
# Ban certain modules from being imported at module level, instead requiring
# that they're imported lazily (e.g., within a function definition, if TYPE_CHECKING, etc.)
# NOTE: Ruff code TID is currently disabled, so this settings doesn't do anything.
banned-module-level-imports = ["torch", "tensorflow"]

[tool.ruff.lint.pylint]
max-args = 10
max-bool-expr = 10
max-statements = 100

[tool.pyright]
include = ["src"]

typeCheckingMode = "standard"
useLibraryCodeForTypes = true
autoImportCompletions = true

# pythonPlatform = "Linux"
6 changes: 4 additions & 2 deletions python/src/reduce_binary/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from .executable import REDUCE_BIN_PATH, reduce
from __future__ import annotations

from .executable import REDUCE_BIN_PATH, popen_reduce, run_reduce

__version__ = "0.0.0"

__all__ = ["REDUCE_BIN_PATH", "reduce"]
__all__ = ["REDUCE_BIN_PATH", "run_reduce", "popen_reduce"]
103 changes: 83 additions & 20 deletions python/src/reduce_binary/executable.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,31 @@

import os
import subprocess
from functools import wraps
from os import PathLike
from pathlib import Path
from typing import Any, Literal, overload
from typing import (
TYPE_CHECKING,
Any,
Callable,
Generic,
TypeVar,
overload,
)

# https://stackoverflow.com/questions/59717828/copy-type-signature-from-another-function
# Needed to make subprocess.Popen or subprocess.run wrappers without modifying the signature
F = TypeVar("F", bound=Callable[..., Any])


# NOTE: it will return None and make the resulting function uncallable.
# Only use in TYPE_CHECKING block
class CopySignature(Generic[F]):
def __init__(self, target: F) -> None:
pass

def __call__(self, wrapped: Callable[..., Any]) -> F: ...


BIN_DIR = Path(__file__).parent / "bin"
if os.name == "nt":
Expand All @@ -12,28 +35,68 @@
REDUCE_BIN_PATH = BIN_DIR / "reduce"


@overload
def reduce(
*args: str, return_completed_process: Literal[True], **kwargs: Any
) -> subprocess.CompletedProcess[str | bytes]: ...
# @CopySignature(subprocess.run)
# @overload
# def run_reduce(*args, **kwargs): ...
#
#
# # NOTE: you can't change the function signature if you're copying it.
# # Thus, it is not possible to pass None as the first argument (or it will result in type error)
# # Thus, we treat an empty string ("") as no argument
# def run_reduce(*args, **kwargs):
# if len(args) > 0:
# cmd_args = args[0]
# args = args[1:]
# else:
# cmd_args = None
#
# if cmd_args is None or (isinstance(cmd_args, str) and cmd_args == ""):
# return subprocess.run(REDUCE_BIN_PATH, *args, **kwargs)
# elif isinstance(cmd_args, (str, bytes, PathLike)):
# return subprocess.run([REDUCE_BIN_PATH, cmd_args], *args, **kwargs)
# return subprocess.run([REDUCE_BIN_PATH, *cmd_args], *args, **kwargs)


if TYPE_CHECKING:

@overload
def reduce(
*args: str, return_completed_process: Literal[False] = ..., **kwargs: Any
) -> int: ...
@CopySignature(subprocess.run)
def run_reduce(
*args,
**kwargs,
): ...

@CopySignature(subprocess.Popen)
def popen_reduce(
*args,
**kwargs,
): ...
else:

def run_reduce(*args, **kwargs):
if len(args) > 0:
cmd_args = args[0]
args = args[1:]
else:
cmd_args = None

@overload
def reduce(
*args: str, return_completed_process: bool = False, **kwargs: Any
) -> int | subprocess.CompletedProcess[str | bytes]: ...
if cmd_args is None or (isinstance(cmd_args, str) and cmd_args == ""):
return subprocess.run(REDUCE_BIN_PATH, *args, **kwargs)
elif isinstance(cmd_args, (str, bytes, PathLike)):
return subprocess.run([REDUCE_BIN_PATH, cmd_args], *args, **kwargs)
return subprocess.run([REDUCE_BIN_PATH, *cmd_args], *args, **kwargs)

def popen_reduce(
*args,
**kwargs,
):
if len(args) > 0:
cmd_args = args[0]
args = args[1:]
else:
cmd_args = None

def reduce(
*args: str, return_completed_process: bool = False, **kwargs: Any
) -> int | subprocess.CompletedProcess[str | bytes]:
complete_process = subprocess.run([REDUCE_BIN_PATH, *args], **kwargs)
if return_completed_process:
return complete_process
return complete_process.returncode
if cmd_args is None or (isinstance(cmd_args, str) and cmd_args == ""):
return subprocess.Popen(REDUCE_BIN_PATH, *args, **kwargs)
elif isinstance(cmd_args, (str, bytes, PathLike)):
return subprocess.Popen([REDUCE_BIN_PATH, cmd_args], *args, **kwargs)
return subprocess.Popen([REDUCE_BIN_PATH, *cmd_args], *args, **kwargs)
Loading

0 comments on commit 31970a1

Please sign in to comment.