diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 53c0302..b3223ee 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,12 +10,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 + - uses: astral-sh/ruff-action@v1 with: args: 'format --check' - - uses: chartboost/ruff-action@v1 + - uses: astral-sh/ruff-action@v1 with: - args: 'check' + args: 'check --ignore D --ignore DOC' check: runs-on: ubuntu-latest diff --git a/async_utils/_cpython_stuff.py b/async_utils/_cpython_stuff.py index 3a76189..7dd0aba 100644 --- a/async_utils/_cpython_stuff.py +++ b/async_utils/_cpython_stuff.py @@ -19,10 +19,6 @@ class _HashedSeq(list[Any]): - """This class guarantees that hash() will be called no more than once - per element. This is important because the lru_cache() will hash - the key multiple times on a cache miss.""" - __slots__ = ("hashvalue",) def __init__( @@ -45,16 +41,6 @@ def make_key( type: type[type] = type, # noqa: A002 len: Callable[[Sized], int] = len, # noqa: A002 ) -> Hashable: - """Make a cache key from optionally typed positional and keyword arguments - The key is constructed in a way that is flat as possible rather than - as a nested structure that would take more memory. - If there is only a single argument and its data type is known to cache - its hash value, then that argument is returned without a wrapper. This - saves space and improves lookup speed.""" - # All of code below relies on kwds preserving the order input by the user. - # Formerly, we sorted() the kwds before looping. The new way is *much* - # faster; however, it means that f(x=1, y=2) will now be treated as a - # distinct call from f(y=2, x=1) which will be cached separately. key: tuple[Any, ...] = args if kwds: key += kwd_mark diff --git a/async_utils/bg_loop.py b/async_utils/bg_loop.py index 7055a02..7cce3cc 100644 --- a/async_utils/bg_loop.py +++ b/async_utils/bg_loop.py @@ -36,19 +36,20 @@ def stop(self) -> None: self._loop.call_soon_threadsafe(self._loop.stop) def schedule(self, coro: _FutureLike[_T]) -> Future[_T]: - """Schedule a coroutine to run on the wrapped event loop""" + """Schedule a coroutine to run on the wrapped event loop.""" return asyncio.run_coroutine_threadsafe(coro, self._loop) async def run(self, coro: _FutureLike[_T]) -> _T: - """Schedule a coroutine to run on the background loop, - awaiting it finishing.""" - + """Schedule and await a coroutine to run on the background loop.""" future = asyncio.run_coroutine_threadsafe(coro, self._loop) return await asyncio.wrap_future(future) def run_forever( - loop: asyncio.AbstractEventLoop, use_eager_task_factory: bool, / + loop: asyncio.AbstractEventLoop, + /, + *, + use_eager_task_factory: bool = True, ) -> None: asyncio.set_event_loop(loop) if use_eager_task_factory: @@ -69,13 +70,11 @@ def run_forever( for task in tasks: try: if (exc := task.exception()) is not None: - loop.call_exception_handler( - { - "message": "Unhandled exception in task during shutdown.", - "exception": exc, - "task": task, - } - ) + loop.call_exception_handler({ + "message": "Unhandled exception in task during shutdown.", + "exception": exc, + "task": task, + }) except (asyncio.InvalidStateError, asyncio.CancelledError): pass @@ -87,11 +86,14 @@ def run_forever( def threaded_loop( *, use_eager_task_factory: bool = True ) -> Generator[LoopWrapper, None, None]: - """Starts an event loop on a background thread, + """Create and use a managed event loop in a backround thread. + + Starts an event loop on a background thread, and yields an object with scheduling methods for interacting with the loop. - loop is scheduled for shutdown, and thread is joined at contextmanager exit""" + loop is scheduled for shutdown, and thread is joined at contextmanager exit + """ loop = asyncio.new_event_loop() thread = None try: diff --git a/async_utils/bg_tasks.py b/async_utils/bg_tasks.py index 84a4ee3..051244c 100644 --- a/async_utils/bg_tasks.py +++ b/async_utils/bg_tasks.py @@ -27,11 +27,11 @@ class BGTasks: - """An intentionally dumber task group""" + """An intentionally dumber task group.""" def __init__(self, exit_timeout: float | None) -> None: self._tasks: set[asyncio.Task[Any]] = set() - self._exit_timeout: float | None = exit_timeout + self._etime: float | None = exit_timeout def create_task( self, @@ -39,7 +39,7 @@ def create_task( *, name: str | None = None, context: Context | None = None, - ) -> Any: + ) -> asyncio.Task[_T]: t = asyncio.create_task(coro) self._tasks.add(t) t.add_done_callback(self._tasks.discard) @@ -48,11 +48,9 @@ def create_task( async def __aenter__(self: Self) -> Self: return self - async def __aexit__(self, *_dont_care: Any): + async def __aexit__(self, *_dont_care: object): while tsks := self._tasks.copy(): - _done, _pending = await asyncio.wait( - tsks, timeout=self._exit_timeout - ) + _done, _pending = await asyncio.wait(tsks, timeout=self._etime) for task in _pending: task.cancel() await asyncio.sleep(0) diff --git a/async_utils/corofunc_cache.py b/async_utils/corofunc_cache.py index ce84cd3..248aac1 100644 --- a/async_utils/corofunc_cache.py +++ b/async_utils/corofunc_cache.py @@ -39,14 +39,16 @@ def corocache( """Decorator to cache coroutine functions. This is less powerful than the version in task_cache.py but may work better - for some cases where typing of libraries this interacts with is too restrictive. + for some cases where typing of libraries this interacts with is too + restrictive. Note: This uses the args and kwargs of the original coroutine function as a cache key. This includes instances (self) when wrapping methods. - Consider not wrapping instance methods, but what those methods call when feasible - in cases where this may matter. + Consider not wrapping instance methods, but what those methods call when + feasible in cases where this may matter. - The ordering of args and kwargs matters.""" + The ordering of args and kwargs matters. + """ def wrapper(coro: CoroLike[P, R]) -> CoroFunc[P, R]: internal_cache: dict[Hashable, asyncio.Future[R]] = {} @@ -88,16 +90,18 @@ def lrucorocache( """Decorator to cache coroutine functions. This is less powerful than the version in task_cache.py but may work better - for some cases where typing of libraries this interacts with is too restrictive. + for some cases where typing of libraries this interacts with is too + restrictive. Note: This uses the args and kwargs of the original coroutine function as a cache key. This includes instances (self) when wrapping methods. - Consider not wrapping instance methods, but what those methods call when feasible - in cases where this may matter. + Consider not wrapping instance methods, but what those methods call when + feasible in cases where this may matter. The ordering of args and kwargs matters. - cached results are evicted by LRU and ttl.""" + Cached results are evicted by LRU and ttl. + """ def wrapper(coro: CoroLike[P, R]) -> CoroFunc[P, R]: internal_cache: LRU[Hashable, asyncio.Future[R]] = LRU(maxsize) diff --git a/async_utils/gen_transform.py b/async_utils/gen_transform.py index 526b954..68402b9 100644 --- a/async_utils/gen_transform.py +++ b/async_utils/gen_transform.py @@ -26,8 +26,8 @@ class _PeekableQueue[T](asyncio.Queue[T]): - """This is for internal use only, tested on both 3.12 and 3.13 - This will be tested for 3.14 prior to 3.14's release.""" + # This is for internal use only, tested on both 3.12 and 3.13 + # This will be tested for 3.14 prior to 3.14's release. _get_loop: Callable[[], asyncio.AbstractEventLoop] # pyright: ignore[reportUninitializedInstanceVariable] _getters: deque[asyncio.Future[None]] # pyright: ignore[reportUninitializedInstanceVariable] @@ -71,8 +71,7 @@ def sync_to_async_gen( *args: P.args, **kwargs: P.kwargs, ) -> AsyncGenerator[YieldType]: - """Asynchronously iterate over a synchronous generator run in - background thread. + """Asynchronously iterate over a synchronous generator. The generator function and it's arguments must be threadsafe and will be iterated lazily. Generators which perform cpu intensive work while holding @@ -84,7 +83,8 @@ def sync_to_async_gen( If your generator is actually a synchronous coroutine, that's super cool, but rewrite is as a native coroutine or use it directly then, you don't need - what this function does.""" + what this function does. + """ # Provides backpressure, ensuring the underlying sync generator in a thread # is lazy If the user doesn't want laziness, then using this method makes # little sense, they could trivially exhaust the generator in a thread with diff --git a/async_utils/lockout.py b/async_utils/lockout.py index 8a4e196..061337d 100644 --- a/async_utils/lockout.py +++ b/async_utils/lockout.py @@ -23,7 +23,7 @@ class Lockout: - """Lock out an async resource for an amount of time + """Lock out an async resource for an amount of time. Resources may be locked out multiple times. @@ -79,7 +79,9 @@ async def __aexit__(self, *_dont_care: object) -> None: class FIFOLockout: - """A FIFO preserving version of Lockout. This has slightly more + """A FIFO preserving version of Lockout. + + This has slightly more overhead than the base Lockout class, which is not guaranteed to preserve FIFO, though happens to in the case of not being locked. diff --git a/async_utils/priority_sem.py b/async_utils/priority_sem.py index 7a00ba9..0088c9f 100644 --- a/async_utils/priority_sem.py +++ b/async_utils/priority_sem.py @@ -19,7 +19,7 @@ import contextvars import heapq import threading -from collections.abc import Callable +from collections.abc import Callable, Generator from contextlib import contextmanager from typing import Any, NamedTuple @@ -55,7 +55,8 @@ def __lt__(self, other: Any) -> bool: @contextmanager -def priority_context(priority: int): +def priority_context(priority: int) -> Generator[None, None, None]: + """Set the priority for all PrioritySemaphore use in this context.""" token = _priority.set(priority) try: yield None @@ -67,7 +68,8 @@ def priority_context(priority: int): class PrioritySemaphore: - """ + """A Semaphore with priority-based aquisition ordering. + Provides a semaphore with similar semantics as asyncio.Semaphore, but using an underlying priority. priority is shared within a context manager's logical scope, but the context can be nested safely. @@ -76,11 +78,11 @@ class PrioritySemaphore: context manager use: - sem = PrioritySemaphore(1) + >>> sem = PrioritySemaphore(1) + >>> with priority_ctx(10): + async with sem: + ... - with priority_ctx(10): - async with sem: - ... """ _loop: asyncio.AbstractEventLoop | None = None @@ -93,12 +95,14 @@ def _get_loop(self) -> asyncio.AbstractEventLoop: if self._loop is None: self._loop = loop if loop is not self._loop: - raise RuntimeError(f"{self!r} is bound to a different event loop") + msg = f"{self!r} is bound to a different event loop" + raise RuntimeError(msg) return loop def __init__(self, value: int = 1): if value < 0: - raise ValueError("Semaphore initial value must be >= 0") + msg = "Semaphore initial value must be >= 0" + raise ValueError(msg) self._waiters: list[PriorityWaiter] | None = None self._value: int = value @@ -120,9 +124,8 @@ def locked(self) -> bool: async def __aenter__(self): prio = _priority.get() await self.acquire(prio) - return - async def __aexit__(self, *dont_care: Any): + async def __aexit__(self, *dont_care: object): self.release() async def acquire(self, priority: int = _default) -> bool: @@ -174,6 +177,6 @@ def _maybe_wake(self) -> None: heapq.heappush(self._waiters, next_waiter) break - def release(self): + def release(self) -> None: self._value += 1 self._maybe_wake() diff --git a/async_utils/ratelimiter.py b/async_utils/ratelimiter.py index f1338ca..b8cd403 100644 --- a/async_utils/ratelimiter.py +++ b/async_utils/ratelimiter.py @@ -23,9 +23,12 @@ class RateLimiter: - """This is an asyncio specific ratelimit implementation which does not + """Asyncio-specific internal application ratelimiter. + + This is an asyncio specific ratelimit implementation which does not account for various networking effects / responses and - should only be used for internal limiting.""" + should only be used for internal limiting. + """ def __init__(self, rate_limit: int, period: float, granularity: float): self.rate_limit: int = rate_limit diff --git a/async_utils/scheduler.py b/async_utils/scheduler.py index 0d070b4..eb55305 100644 --- a/async_utils/scheduler.py +++ b/async_utils/scheduler.py @@ -73,7 +73,7 @@ async def __aenter__(self): return self - async def __aexit__(self, *_dont_care: Any): + async def __aexit__(self, *_dont_care: object): self.__closed = True def __aiter__(self): @@ -108,23 +108,22 @@ async def create_task( await self.__tqueue.put(t) return t.cancel_token - async def cancel_task(self, cancel_token: CancelationToken, /) -> bool: - """Returns if the task with that CancelationToken. Cancelling an - already cancelled task is allowed and has no additional effect.""" + async def cancel_task(self, cancel_token: CancelationToken, /) -> None: + """Cancel a task. + + Canceling an already canceled task is not an error + """ async with self.__l: try: task = self.__tasks[cancel_token] task.canceled = True except KeyError: pass - else: - return True - return False - def close(self): - """Closes the scheduler without waiting""" + def close(self) -> None: + """Closes the scheduler without waiting.""" self.__closed = True - async def join(self): - """Waits for the scheduler's internal queue to be empty""" + async def join(self) -> None: + """Waits for the scheduler's internal queue to be empty.""" await self.__tqueue.join() diff --git a/async_utils/sig_service.py b/async_utils/sig_service.py index cf4055a..f1a2180 100644 --- a/async_utils/sig_service.py +++ b/async_utils/sig_service.py @@ -42,9 +42,12 @@ class SpecialExit(enum.IntEnum): class SignalService: - """Meant for graceful signal handling where the main thread is only used + """Helper for signal handling. + + Meant for graceful signal handling where the main thread is only used for signal handling. - This should be paired with event loops being run in threads.""" + This should be paired with event loops being run in threads. + """ def __init__( self, @@ -59,16 +62,16 @@ def __init__( def get_send_socket(self) -> socket.socket: return self.cs - def add_startup(self, job: StartStopCall): + def add_startup(self, job: StartStopCall) -> None: self._startup.append(job) - def add_signal_cb(self, cb: SignalCallback): + def add_signal_cb(self, cb: SignalCallback) -> None: self._cbs.append(cb) - def add_join(self, join: StartStopCall): + def add_join(self, join: StartStopCall) -> None: self._joins.append(join) - def run(self): + def run(self) -> None: signal.set_wakeup_fd(self.cs.fileno()) original_handlers: list[_HANDLER] = [] @@ -92,5 +95,5 @@ def run(self): for join in self._joins: join() - for sig, original in zip(actual, original_handlers): + for sig, original in zip(actual, original_handlers, strict=True): signal.signal(sig, original) diff --git a/async_utils/task_cache.py b/async_utils/task_cache.py index bbeb436..daf582b 100644 --- a/async_utils/task_cache.py +++ b/async_utils/task_cache.py @@ -37,7 +37,8 @@ def taskcache( ttl: float | None = None, ) -> Callable[[TaskCoroFunc[P, R]], TaskFunc[P, R]]: - """ + """Async caching via decorator. + Decorator to modify coroutine functions to instead act as functions returning cached tasks. @@ -92,7 +93,8 @@ def _lru_evict( def lrutaskcache( ttl: float | None = None, maxsize: int = 1024 ) -> Callable[[TaskCoroFunc[P, R]], TaskFunc[P, R]]: - """ + """Async caching via decorator. + Decorator to modify coroutine functions to instead act as functions returning cached tasks. diff --git a/async_utils/waterfall.py b/async_utils/waterfall.py index cbfa8d6..8dbafa0 100644 --- a/async_utils/waterfall.py +++ b/async_utils/waterfall.py @@ -24,9 +24,7 @@ class Waterfall[T]: - """ - Class for batch event scheduling based on recurring intervals, - with a quanity threshold which overrides the interval. + """Batch event scheduling based on recurring quantity-interval pairs. Initial intended was batching of simple db writes with an acceptable tolerance for lost writes, @@ -52,7 +50,7 @@ def __init__( self.task: asyncio.Task[None] | None = None self._alive: bool = False - def start(self): + def start(self) -> None: if self.task is not None: msg = "Already Running" raise RuntimeError(msg) @@ -61,22 +59,22 @@ def start(self): self.task = asyncio.create_task(self._loop()) @overload - def stop(self, wait: Literal[True]) -> Coroutine[Any, Any, None]: + def stop(self, *, wait: Literal[True]) -> Coroutine[Any, Any, None]: pass @overload - def stop(self, wait: Literal[False]) -> None: + def stop(self, *, wait: Literal[False]) -> None: pass @overload - def stop(self, wait: bool = False) -> Coroutine[Any, Any, None] | None: + def stop(self, *, wait: bool = False) -> Coroutine[Any, Any, None] | None: pass - def stop(self, wait: bool = False) -> Coroutine[Any, Any, None] | None: + def stop(self, *, wait: bool = False) -> Coroutine[Any, Any, None] | None: self._alive = False return self.queue.join() if wait else None - def put(self, item: T): + def put(self, item: T) -> None: if not self._alive: msg = "Can't put something in a non-running Waterfall." raise RuntimeError(msg) diff --git a/pyproject.toml b/pyproject.toml index 424dcf3..50782a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ reportUnnecessaryTypeIgnoreComment = "warning" line-length = 80 target-version = "py312" +preview = true [tool.ruff.format] line-ending = "lf" @@ -32,36 +33,47 @@ line-ending = "lf" [tool.ruff.lint] select = [ - "F", "E", "I", "UP", "YTT", "ANN", "S", "BLE", "B", "A", "COM", "C4", "DTZ", - "EM", "ISC", "G", "INP", "PIE", "T20", "Q003", "RSE", "RET", "SIM", "TID", "PTH", - "ERA", "PD", "PLC", "PLE", "PLW", "TRY", "NPY", "RUF" + "A", "ANN", "ASYNC", "B", "BLE", "C4", "COM", "D", "DOC", "DTZ", "E", + "EM", "ERA", "F", "FA", "FBT", "FURB", "G", "I", "INP", "ISC", "NPY", + "PD", "PERF", "PGH", "PIE", "PLC", "PLE", "PLR", "PLW", "PTH", "PYI", + "Q", "Q003", "RET", "RSE", "RUF", "S", "SIM", "SLOT", "T20", "TC", "TID", + "TRY", "UP", "YTT" ] -extend-ignore = [ +ignore = [ + "ANN202", # implied return fine sometimes + "ANN204", # special method return types + "ANN401", # Any is the correct type in some cases + "ASYNC116", # Long sleeps are fine + "B901", # I'm aware of how generators as coroutines work + "C90", # mccabe complexity memes + "COM812", # ruff format suggested + "D105", # documenting magic methods is often dumb. + "E501", # ruff format suggested + "FBT003", # Wrong end to enforce this on. "G002", # erroneous issue with %-logging when logging can be confiured for % logging - "S101", # use of assert here is a known quantity, blame typing memes + "ISC001", # ruff format suggested + "PLC0105", # no, I don't like co naming style for typevars + "PLR0912", # too many branches + "PLR0913", # number of function arguments + "PLR0915", # too many statements.... in an async entrypoint handling graceful shutdown... + "PLR0917", # too many positional arguments "PLR2004", # Magic value comparison, may remove later + "RUF001", # ambiguous characters not something I want to enforce here. + "S101", # use of assert here is a known quantity, blame typing memes + "S311", # Yes, I know that standard pseudo-random generators are not suitable for cryptographic purposes "SIM105", # supressable exception, I'm not paying the overhead of contextlib.supress for stylistic choices. - "C90", # mccabe complexity memes - "ANN201", # return types - "ANN204", # special method return types - "ANN401", - "EM101", - "EM102", - "TRY003", - "PLR0913", # number of function arguments + "TC003", # I prefer to avoid if TYPE_CHECKING "UP007", # "Use | For Union" doesn't account for typevar tuple unpacking. - "PTH123", # `open()` should be replaced by `Path.open()` - "PLR", # more complexity things - "COM812", # trailing commmas - "ERA001", # commented out code - "E731", # No, I think I'll keep my lambdas when I feel they are the right call - "B905", # zip without explicit strict= - "COM819", # reccomended by ruff when using ruff format - "E501", # reccomended by ruff when using ruff format - "ISC001", # reccomended by ruff when using ruff format - "Q003", # reccomended by ruff when using ruff format + "UP031", # No, I like % formatting more for some things... ] +unfixable = [ + "E501", # line length handled in other ways by ruff format + "ERA", # Don't delete commented out code +] + +[tool.ruff.lint.pydocstyle] +convention = "google" [tool.ruff.lint.flake8-tidy-imports.banned-api] # https://discuss.python.org/t/problems-with-typeis/55410/6 diff --git a/setup.cfg b/setup.cfg index 4764f64..e38cb62 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,7 @@ license_files = [options] packages = find_namespace: -python_requires = >=3.12.0 +python_requires = >=3.12.0,<3.14 include_package_data = True [options.package_data]