From 4364d52690cd73dfd207320956ba420892cb8919 Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Thu, 24 Aug 2023 14:53:40 +0800 Subject: [PATCH] Fix UT. --- mmengine/config/utils.py | 288 +-------------------------- mmengine/testing/runner_test_case.py | 2 +- tests/test_config/test_lazy.py | 153 +------------- 3 files changed, 9 insertions(+), 434 deletions(-) diff --git a/mmengine/config/utils.py b/mmengine/config/utils.py index 36f07466b2..05cc83d194 100644 --- a/mmengine/config/utils.py +++ b/mmengine/config/utils.py @@ -4,9 +4,8 @@ import re import sys import warnings -from collections import defaultdict from importlib.util import find_spec -from typing import List, Optional, Tuple, Union +from typing import Tuple from mmengine.fileio import load from mmengine.utils import check_file_exist @@ -180,288 +179,3 @@ def _is_builtin_module(module_name: str) -> bool: return False else: return True - - -class ImportTransformer(ast.NodeTransformer): - """Convert the import syntax to the assignment of - :class:`mmengine.config.LazyObject` and preload the base variable before - parsing the configuration file. - - Since you are already looking at this part of the code, I believe you must - be interested in the mechanism of the ``lazy_import`` feature of - :class:`Config`. In this docstring, we will dive deeper into its - principles. - - Most of OpenMMLab users maybe bothered with that: - - * In most of popular IDEs, they cannot navigate to the source code in - configuration file - * In most of popular IDEs, they cannot jump to the base file in current - configuration file, which is much painful when the inheritance - relationship is complex. - - In order to solve this problem, we introduce the ``lazy_import`` mode. - - A very intuitive idea for solving this problem is to import the module - corresponding to the "type" field using the ``import`` syntax. Similarly, - we can also ``import`` base file. - - However, this approach has a significant drawback. It requires triggering - the import logic to parse the configuration file, which can be - time-consuming. Additionally, it implies downloading numerous dependencies - solely for the purpose of parsing the configuration file. - However, it's possible that only a portion of the config will actually be - used. For instance, the package used in the ``train_pipeline`` may not - be necessary for an evaluation task. Forcing users to download these - unused packages is not a desirable solution. - - To avoid this problem, we introduce :class:`mmengine.config.LazyObject` and - :class:`mmengine.config.LazyAttr`. Before we proceed with further - explanations, you may refer to the documentation of these two modules to - gain an understanding of their functionalities. - - Actually, one of the functions of ``ImportTransformer`` is to hack the - ``import`` syntax. It will replace the import syntax - (exclude import the base files) with the assignment of ``LazyObject``. - - As for the import syntax of the base file, we cannot lazy import it since - we're eager to merge the fields of current file and base files. Therefore, - another function of the ``ImportTransformer`` is to collaborate with - ``Config._parse_lazy_import`` to parse the base files. - - Args: - global_dict (dict): The global dict of the current configuration file. - If we divide ordinary Python syntax into two parts, namely the - import section and the non-import section (assuming a simple case - with imports at the beginning and the rest of the code following), - the variables generated by the import statements are stored in - global variables for subsequent code use. In this context, - the ``global_dict`` represents the global variables required when - executing the non-import code. ``global_dict`` will be filled - during visiting the parsed code. - base_dict (dict): All variables defined in base files. - - Examples: - >>> from mmengine.config import read_base - >>> - >>> - >>> with read_base(): - >>> from .._base_.default_runtime import * - >>> from .._base_.datasets.coco_detection import dataset - - In this case, the base_dict will be: - - Examples: - >>> base_dict = { - >>> '.._base_.default_runtime': ... - >>> '.._base_.datasets.coco_detection': dataset} - - and `global_dict` will be updated like this: - - Examples: - >>> global_dict.update(base_dict['.._base_.default_runtime']) # `import *` means update all data - >>> global_dict.update(dataset=base_dict['.._base_.datasets.coco_detection']['dataset']) # only update `dataset` - """ # noqa: E501 - - def __init__(self, - global_dict: dict, - base_dict: Optional[dict] = None, - filename: Optional[str] = None): - self.base_dict = base_dict if base_dict is not None else {} - self.global_dict = global_dict - # In Windows, the filename could be like this: - # "C:\\Users\\runneradmin\\AppData\\Local\\" - # Although it has been an raw string, ast.parse will firstly escape - # it as the executed code: - # "C:\Users\runneradmin\AppData\Local\\\" - # As you see, the `\U` will be treated as a part of - # the escape sequence during code parsing, leading to an - # parsing error - # Here we use `encode('unicode_escape').decode()` for double escaping - if isinstance(filename, str): - filename = filename.encode('unicode_escape').decode() - self.filename = filename - self.imported_obj: set = set() - super().__init__() - - def visit_ImportFrom( - self, node: ast.ImportFrom - ) -> Optional[Union[List[ast.Assign], ast.ImportFrom]]: - """Hack the ``from ... import ...`` syntax and update the global_dict. - - Examples: - >>> from mmdet.models import RetinaNet - - Will be parsed as: - - Examples: - >>> RetinaNet = lazyObject('mmdet.models', 'RetinaNet') - - ``global_dict`` will also be updated by ``base_dict`` as the - class docstring says. - - Args: - node (ast.AST): The node of the current import statement. - - Returns: - Optional[List[ast.Assign]]: There three cases: - - * If the node is a statement of importing base files. - None will be returned. - * If the node is a statement of importing a builtin module, - node will be directly returned - * Otherwise, it will return the assignment statements of - ``LazyObject``. - """ - # Built-in modules will not be parsed as LazyObject - module = f'{node.level*"."}{node.module}' - if _is_builtin_module(module): - # Make sure builtin module will be added into `self.imported_obj` - for alias in node.names: - if alias.asname is not None: - self.imported_obj.add(alias.asname) - elif alias.name == '*': - raise ConfigParsingError( - 'Cannot import * from non-base config') - else: - self.imported_obj.add(alias.name) - return node - - if module in self.base_dict: - for alias_node in node.names: - if alias_node.name == '*': - self.global_dict.update(self.base_dict[module]) - return None - if alias_node.asname is not None: - base_key = alias_node.asname - else: - base_key = alias_node.name - self.global_dict[base_key] = self.base_dict[module][ - alias_node.name] - return None - - nodes: List[ast.Assign] = [] - for alias_node in node.names: - # `ast.alias` has lineno attr after Python 3.10, - if hasattr(alias_node, 'lineno'): - lineno = alias_node.lineno - else: - lineno = node.lineno - if alias_node.name == '*': - # TODO: If users import * from a non-config module, it should - # fallback to import the real module and raise a warning to - # remind users the real module will be imported which will slow - # down the parsing speed. - raise ConfigParsingError( - 'Illegal syntax in config! `from xxx import *` is not ' - 'allowed to appear outside the `if base:` statement') - elif alias_node.asname is not None: - # case1: - # from mmengine.dataset import BaseDataset as Dataset -> - # Dataset = LazyObject('mmengine.dataset', 'BaseDataset') - code = f'{alias_node.asname} = LazyObject("{module}", "{alias_node.name}", "{self.filename}, line {lineno}")' # noqa: E501 - self.imported_obj.add(alias_node.asname) - else: - # case2: - # from mmengine.model import BaseModel - # BaseModel = LazyObject('mmengine.model', 'BaseModel') - code = f'{alias_node.name} = LazyObject("{module}", "{alias_node.name}", "{self.filename}, line {lineno}")' # noqa: E501 - self.imported_obj.add(alias_node.name) - try: - nodes.append(ast.parse(code).body[0]) # type: ignore - except Exception as e: - raise ConfigParsingError( - f'Cannot import {alias_node} from {module}' - '1. Cannot import * from 3rd party lib in the config ' - 'file\n' - '2. Please check if the module is a base config which ' - 'should be added to `_base_`\n') from e - return nodes - - def visit_Import(self, node) -> Union[ast.Assign, ast.Import]: - """Work with ``_gather_abs_import_lazyobj`` to hack the ``import ...`` - syntax. - - Examples: - >>> import mmcls.models - >>> import mmcls.datasets - >>> import mmcls - - Will be parsed as: - - Examples: - >>> # import mmcls.models; import mmcls.datasets; import mmcls - >>> mmcls = lazyObject(['mmcls', 'mmcls.datasets', 'mmcls.models']) - - Args: - node (ast.AST): The node of the current import statement. - - Returns: - ast.Assign: If the import statement is ``import ... as ...``, - ast.Assign will be returned, otherwise node will be directly - returned. - """ - # For absolute import like: `import mmdet.configs as configs`. - # It will be parsed as: - # configs = LazyObject('mmdet.configs') - # For absolute import like: - # `import mmdet.configs` - # `import mmdet.configs.default_runtime` - # This will be parsed as - # mmdet = LazyObject(['mmdet.configs.default_runtime', 'mmdet.configs]) - # However, visit_Import cannot gather other import information, so - # `_gather_abs_import_LazyObject` will gather all import information - # from the same module and construct the LazyObject. - alias_list = node.names - assert len(alias_list) == 1, ( - 'Illegal syntax in config! import multiple modules in one line is ' - 'not supported') - # TODO Support multiline import - alias = alias_list[0] - if alias.asname is not None: - self.imported_obj.add(alias.asname) - if _is_builtin_module(alias.name.split('.')[0]): - return node - return ast.parse( # type: ignore - f'{alias.asname} = LazyObject(' - f'"{alias.name}",' - f'location="{self.filename}, line {node.lineno}")').body[0] - return node - - -def _gather_abs_import_lazyobj(tree: ast.Module, - filename: Optional[str] = None): - """Experimental implementation of gathering absolute import information.""" - if isinstance(filename, str): - filename = filename.encode('unicode_escape').decode() - imported = defaultdict(list) - abs_imported = set() - new_body: List[ast.stmt] = [] - # module2node is used to get lineno when Python < 3.10 - module2node: dict = dict() - for node in tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - # Skip converting built-in module to LazyObject - if _is_builtin_module(alias.name): - new_body.append(node) - continue - module = alias.name.split('.')[0] - module2node.setdefault(module, node) - imported[module].append(alias) - continue - new_body.append(node) - - for key, value in imported.items(): - names = [_value.name for _value in value] - if hasattr(value[0], 'lineno'): - lineno = value[0].lineno - else: - lineno = module2node[key].lineno - lazy_module_assign = ast.parse( - f'{key} = LazyObject({names}, location="{filename}, line {lineno}")' # noqa: E501 - ) # noqa: E501 - abs_imported.add(key) - new_body.insert(0, lazy_module_assign.body[0]) - tree.body = new_body - return tree, abs_imported diff --git a/mmengine/testing/runner_test_case.py b/mmengine/testing/runner_test_case.py index f64594acef..1b77b4b948 100644 --- a/mmengine/testing/runner_test_case.py +++ b/mmengine/testing/runner_test_case.py @@ -133,7 +133,7 @@ def setUp(self) -> None: custom_hooks=[], env_cfg=dict(dist_cfg=dict(backend='nccl')), experiment_name='test1') - self.epoch_based_cfg = Config(epoch_based_cfg) + self.epoch_based_cfg = Config(epoch_based_cfg) # type: ignore # prepare iter based cfg. self.iter_based_cfg: Config = copy.deepcopy(self.epoch_based_cfg) diff --git a/tests/test_config/test_lazy.py b/tests/test_config/test_lazy.py index d69822814b..8e30cb9de9 100644 --- a/tests/test_config/test_lazy.py +++ b/tests/test_config/test_lazy.py @@ -1,122 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. -import ast import copy -import os -import os.path as osp from importlib import import_module -from importlib.util import find_spec from unittest import TestCase -import numpy -import numpy.compat -import numpy.linalg as linalg -from rich.progress import Progress - import mmengine -from mmengine.config import Config -from mmengine.config.lazy import LazyAttr, LazyObject -from mmengine.config.utils import ImportTransformer, _gather_abs_import_lazyobj -from mmengine.fileio import LocalBackend, PetrelBackend - - -class TestImportTransformer(TestCase): - - @classmethod - def setUpClass(cls) -> None: - cls.data_dir = osp.join( # type: ignore - osp.dirname(__file__), '..', 'data', 'config', - 'lazy_module_config') - super().setUpClass() - - def test_lazy_module(self): - cfg_path = osp.join(self.data_dir, 'test_ast_transform.py') - with open(cfg_path) as f: - codestr = f.read() - codeobj = ast.parse(codestr) - global_dict = { - 'LazyObject': LazyObject, - } - base_dict = { - '._base_.default_runtime': { - 'default_scope': 'test_config' - }, - '._base_.scheduler': { - 'val_cfg': {} - }, - } - codeobj = ImportTransformer(global_dict, base_dict).visit(codeobj) - codeobj, _ = _gather_abs_import_lazyobj(codeobj) - codeobj = ast.fix_missing_locations(codeobj) - - exec(compile(codeobj, cfg_path, mode='exec'), global_dict, global_dict) - # 1. absolute import - # 1.1 import module as LazyObject - lazy_numpy = global_dict['numpy'] - self.assertIsInstance(lazy_numpy, LazyObject) - - # 1.2 getattr as LazyAttr - self.assertIsInstance(lazy_numpy.linalg, LazyAttr) - self.assertIsInstance(lazy_numpy.compat, LazyAttr) - - # 1.3 Build module from LazyObject. amp and functional can be accessed - imported_numpy = lazy_numpy.build() - self.assertIs(imported_numpy.linalg, linalg) - self.assertIs(imported_numpy.compat, numpy.compat) - - # 1.4.1 Build module from LazyAttr - imported_linalg = lazy_numpy.linalg.build() - imported_compat = lazy_numpy.compat.build() - self.assertIs(imported_compat, numpy.compat) - self.assertIs(imported_linalg, linalg) - - # 1.4.2 build class method from LazyAttr - start = global_dict['start'] - self.assertEqual(start.module, 'rich.progress.Progress') - self.assertEqual(str(start), 'start') - self.assertIs(start.build(), Progress.start) - - # 1.5 import ... as, and build module from LazyObject - lazy_linalg = global_dict['linalg'] - self.assertIsInstance(lazy_linalg, LazyObject) - self.assertIs(lazy_linalg.build(), linalg) - self.assertIsInstance(lazy_linalg.norm, LazyAttr) - self.assertIs(lazy_linalg.norm.build(), linalg.norm) - - # 1.6 import built in module - imported_os = global_dict['os'] - self.assertIs(imported_os, os) - - # 2. Relative import - # 2.1 from ... import ... - lazy_local_backend = global_dict['local'] - self.assertIsInstance(lazy_local_backend, LazyObject) - self.assertIs(lazy_local_backend.build(), LocalBackend) - - # 2.2 from ... import ... as ... - lazy_petrel_backend = global_dict['PetrelBackend'] - self.assertIsInstance(lazy_petrel_backend, LazyObject) - self.assertIs(lazy_petrel_backend.build(), PetrelBackend) - - # 2.3 from ... import builtin module or obj from `mmengine.Config` - self.assertIs(global_dict['find_module'], find_spec) - self.assertIs(global_dict['Config'], Config) - - # 3 test import base config - # 3.1 simple from ... import and from ... import ... as - self.assertEqual(global_dict['scope'], 'test_config') - self.assertDictEqual(global_dict['val_cfg'], {}) - - # 4. Error catching - cfg_path = osp.join(self.data_dir, - 'test_ast_transform_error_catching1.py') - with open(cfg_path) as f: - codestr = f.read() - codeobj = ast.parse(codestr) - global_dict = {'LazyObject': LazyObject} - with self.assertRaisesRegex( - RuntimeError, - r'Illegal syntax in config! `from xxx import \*`'): - codeobj = ImportTransformer(global_dict).visit(codeobj) +from mmengine.config.lazy import LazyObject +from mmengine.fileio import LocalBackend class TestLazyObject(TestCase): @@ -126,14 +15,6 @@ def test_init(self): LazyObject('mmengine.fileio') LazyObject('mmengine.fileio', 'LocalBackend') - # module must be str - with self.assertRaises(TypeError): - LazyObject(1) - - # imported must be a sequence of string or None - with self.assertRaises(TypeError): - LazyObject('mmengine', ['error_type']) - def test_build(self): lazy_mmengine = LazyObject('mmengine') self.assertIs(lazy_mmengine.build(), mmengine) @@ -142,37 +23,19 @@ def test_build(self): self.assertIs(lazy_mmengine_fileio.build(), import_module('mmengine.fileio')) - lazy_local_backend = LazyObject('mmengine.fileio', 'LocalBackend') + lazy_local_backend = LazyObject('LocalBackend', + LazyObject('mmengine.fileio')) self.assertIs(lazy_local_backend.build(), LocalBackend) - # TODO: The commented test is required, we need to test the built - # LazyObject can access the `mmengine.dataset`. We need to clean the - # environment to make sure the `dataset` is not imported before, and - # it is triggered by lazy_mmengine.build(). However, if we simply - # pop the `mmengine.dataset` will lead to other tests failed, of which - # reason is still unknown. We need to figure out the reason and fix it - # in the latter - - # sys.modules.pop('mmengine.config') - # sys.modules.pop('mmengine.fileio') - # sys.modules.pop('mmengine') - # lazy_mmengine = LazyObject(['mmengine', 'mmengine.dataset']) - # self.assertIs(lazy_mmengine.build().dataset, - # import_module('mmengine.config')) copied = copy.deepcopy(lazy_local_backend) self.assertDictEqual(copied.__dict__, lazy_local_backend.__dict__) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): lazy_mmengine() with self.assertRaises(ImportError): LazyObject('unknown').build() - -class TestLazyAttr(TestCase): - # Since LazyAttr should only be built from LazyObect, we only test - # the build method here. - def test_build(self): lazy_mmengine = LazyObject('mmengine') local_backend = lazy_mmengine.fileio.LocalBackend self.assertIs(local_backend.build(), LocalBackend) @@ -180,10 +43,8 @@ def test_build(self): copied = copy.deepcopy(local_backend) self.assertDictEqual(copied.__dict__, local_backend.__dict__) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): local_backend() - with self.assertRaisesRegex( - ImportError, - 'Failed to import mmengine.fileio.LocalBackend.unknown'): + with self.assertRaisesRegex(ImportError, 'Failed to import'): local_backend.unknown.build()