From 4df2730c6367fb81bc1d5d3b017ff806ea59be99 Mon Sep 17 00:00:00 2001 From: Hironori Yamamoto Date: Sun, 22 Sep 2024 21:26:41 +0900 Subject: [PATCH] fix: fix mypy errors on Python3.12 --- luigi/configuration/cfg_parser.py | 2 +- luigi/configuration/toml_parser.py | 8 +++++--- luigi/db_task_history.py | 6 +++--- luigi/freezing.py | 2 +- luigi/parameter.py | 2 +- luigi/scheduler.py | 2 +- luigi/task.py | 5 +++-- luigi/task_register.py | 9 +++++---- tox.ini | 3 +++ 9 files changed, 23 insertions(+), 16 deletions(-) diff --git a/luigi/configuration/cfg_parser.py b/luigi/configuration/cfg_parser.py index abca6d713a..ceb51e99ae 100644 --- a/luigi/configuration/cfg_parser.py +++ b/luigi/configuration/cfg_parser.py @@ -117,7 +117,7 @@ def before_write(self, parser, section, option, value): class LuigiConfigParser(BaseParser, ConfigParser): NO_DEFAULT = object() enabled = True - optionxform = str + optionxform = str # type: ignore _instance = None _config_paths = [ '/etc/luigi/client.cfg', # Deprecated old-style global luigi config diff --git a/luigi/configuration/toml_parser.py b/luigi/configuration/toml_parser.py index 683d8b5c4b..0d977e6ecc 100644 --- a/luigi/configuration/toml_parser.py +++ b/luigi/configuration/toml_parser.py @@ -16,11 +16,13 @@ # import os.path from configparser import ConfigParser +from typing import Any, Dict try: import toml + toml_enabled = True except ImportError: - toml = False + toml_enabled = False from .base_parser import BaseParser from ..freezing import recursively_freeze @@ -28,8 +30,8 @@ class LuigiTomlParser(BaseParser, ConfigParser): NO_DEFAULT = object() - enabled = bool(toml) - data = dict() + enabled = bool(toml_enabled) + data: Dict[str, Any] = dict() _instance = None _config_paths = [ '/etc/luigi/luigi.toml', diff --git a/luigi/db_task_history.py b/luigi/db_task_history.py index e1eabcb7d1..69dc65d63c 100644 --- a/luigi/db_task_history.py +++ b/luigi/db_task_history.py @@ -188,7 +188,7 @@ def find_task_by_id(self, id, session=None): return session.query(TaskRecord).get(id) -class TaskParameter(Base): +class TaskParameter(Base): # type: ignore """ Table to track luigi.Parameter()s of a Task. """ @@ -201,7 +201,7 @@ def __repr__(self): return "TaskParameter(task_id=%d, name=%s, value=%s)" % (self.task_id, self.name, self.value) -class TaskEvent(Base): +class TaskEvent(Base): # type: ignore """ Table to track when a task is scheduled, starts, finishes, and fails. """ @@ -215,7 +215,7 @@ def __repr__(self): return "TaskEvent(task_id=%s, event_name=%s, ts=%s" % (self.task_id, self.event_name, self.ts) -class TaskRecord(Base): +class TaskRecord(Base): # type: ignore """ Base table to track information about a luigi.Task. diff --git a/luigi/freezing.py b/luigi/freezing.py index 2f0a4b49f6..037e574b92 100644 --- a/luigi/freezing.py +++ b/luigi/freezing.py @@ -8,7 +8,7 @@ try: from collections.abc import Mapping except ImportError: - from collections import Mapping + from collections import Mapping # type: ignore import operator import functools diff --git a/luigi/parameter.py b/luigi/parameter.py index f7f137a6d1..83b21ff24b 100644 --- a/luigi/parameter.py +++ b/luigi/parameter.py @@ -1603,4 +1603,4 @@ def normalize(self, x): class OptionalPathParameter(OptionalParameter, PathParameter): """Class to parse optional path parameters.""" - expected_type = (str, Path) + expected_type = (str, Path) # type: ignore diff --git a/luigi/scheduler.py b/luigi/scheduler.py index 10d67a10af..81532d3c96 100644 --- a/luigi/scheduler.py +++ b/luigi/scheduler.py @@ -84,7 +84,7 @@ "disable_hard_timeout", "disable_window", ] -RetryPolicy = collections.namedtuple("RetryPolicy", _retry_policy_fields) +RetryPolicy = collections.namedtuple("RetryPolicy", _retry_policy_fields) # type: ignore def _get_empty_retry_policy(): diff --git a/luigi/task.py b/luigi/task.py index 4d3a312884..3482f1bf26 100644 --- a/luigi/task.py +++ b/luigi/task.py @@ -29,6 +29,7 @@ import re import copy import functools +from typing import Any, Dict import luigi @@ -170,7 +171,7 @@ class MyTask(luigi.Task): """ - _event_callbacks = {} + _event_callbacks: Dict[Any, Any] = {} #: Priority of the task: the scheduler should favor available #: tasks with higher priority values first. @@ -180,7 +181,7 @@ class MyTask(luigi.Task): #: Resources used by the task. Should be formatted like {"scp": 1} to indicate that the #: task requires 1 unit of the scp resource. - resources = {} + resources: Dict[str, Any] = {} #: Number of seconds after which to time out the run function. #: No timeout if set to 0. diff --git a/luigi/task_register.py b/luigi/task_register.py index f5e0acdd32..b104be31ad 100644 --- a/luigi/task_register.py +++ b/luigi/task_register.py @@ -19,8 +19,9 @@ """ import abc - import logging +from typing import Any, Dict + logger = logging.getLogger('luigi-interface') @@ -46,9 +47,9 @@ class Register(abc.ABCMeta): same object. 2. Keep track of all subclasses of :py:class:`Task` and expose them. """ - __instance_cache = {} - _default_namespace_dict = {} - _reg = [] + __instance_cache: Dict[str, Any] = {} + _default_namespace_dict: Dict[str, Any] = {} + _reg: list[Any] = [] AMBIGUOUS_CLASS = object() # Placeholder denoting an error """If this value is returned by :py:meth:`_get_reg` then there is an ambiguous task name (two :py:class:`Task` have the same name). This denotes diff --git a/tox.ini b/tox.ini index f9e93d6889..0e3f3e0f5d 100644 --- a/tox.ini +++ b/tox.ini @@ -62,6 +62,9 @@ deps = dropbox: dropbox>=11.0.0 jsonschema mypy + types-toml + types-python-dateutil + types-requests passenv = USER JAVA_HOME POSTGRES_USER DATAPROC_TEST_PROJECT_ID GCS_TEST_PROJECT_ID GCS_TEST_BUCKET GOOGLE_APPLICATION_CREDENTIALS TRAVIS_BUILD_ID TRAVIS TRAVIS_BRANCH TRAVIS_JOB_NUMBER TRAVIS_PULL_REQUEST TRAVIS_JOB_ID TRAVIS_REPO_SLUG TRAVIS_COMMIT CI DROPBOX_APP_TOKEN DOCKERHUB_TOKEN GITHUB_ACTIONS OVERRIDE_SKIP_CI_TESTS setenv =