[Improve] Use meta path instead of AST transform to implement new con…

…fig.

.pre-commit-config.yaml

@@ -55,7 +55,7 @@ repos:
         args: ["mmengine", "tests"]
       - id: remove-improper-eol-in-cn-docs
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v1.4.1
     hooks:
       - id: mypy
         exclude: "docs"

mmengine/config/__init__.py

@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
-from .config import Config, ConfigDict, DictAction, read_base
+from .config import Config, ConfigDict, DictAction
+from .new_config import read_base
 
 __all__ = ['Config', 'ConfigDict', 'DictAction', 'read_base']

mmengine/config/config.py

@@ -447,30 +447,8 @@ def fromfile(filename: Union[str, Path],
                 filename=filename,
                 env_variables=env_variables)
         else:
-            # Enable lazy import when parsing the config.
-            # Using try-except to make sure ``ConfigDict.lazy`` will be reset
-            # to False. See more details about lazy in the docstring of
-            # ConfigDict
-            ConfigDict.lazy = True
-            try:
-                cfg_dict, imported_names = Config._parse_lazy_import(filename)
-            except Exception as e:
-                raise e
-            finally:
-                ConfigDict.lazy = False
-
-            # delete builtin imported objects
-            for key, value in list(cfg_dict._to_lazy_dict().items()):
-                if isinstance(value, (types.FunctionType, types.ModuleType)):
-                    cfg_dict.pop(key)
-
-            # disable lazy import to get the real type. See more details about
-            # lazy in the docstring of ConfigDict
-            cfg = Config(
-                cfg_dict,
-                filename=filename,
-                format_python_code=format_python_code)
-            object.__setattr__(cfg, '_imported_names', imported_names)
+            from .new_config import Config as NewConfig
+            cfg = NewConfig.fromfile(filename, lazy_import=lazy_import)
             return cfg
 
     @staticmethod

mmengine/config/lazy.py

@@ -1,8 +1,9 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import importlib
-from typing import Any, Optional, Union
-
-from mmengine.utils import is_seq_of
+import re
+import sys
+from importlib.util import spec_from_loader
+from typing import Any
 
 
 class LazyObject:
@@ -38,184 +39,91 @@ class LazyObject:
             module statement happened.
     """
 
-    def __init__(self,
-                 module: Union[str, list, tuple],
-                 imported: Optional[str] = None,
-                 location: Optional[str] = None):
-        if not isinstance(module, str) and not is_seq_of(module, str):
-            raise TypeError('module should be `str`, `list`, or `tuple`'
-                            f'but got {type(module)}, this might be '
-                            'a bug of MMEngine, please report it to '
-                            'https://github.com/open-mmlab/mmengine/issues')
-        self._module: Union[str, list, tuple] = module
-
-        if not isinstance(imported, str) and imported is not None:
-            raise TypeError('imported should be `str` or None, but got '
-                            f'{type(imported)}, this might be '
-                            'a bug of MMEngine, please report it to '
-                            'https://github.com/open-mmlab/mmengine/issues')
-        self._imported = imported
-        self.location = location
+    def __init__(self, name: str, source: 'LazyObject' = None):
+        self.name = name
+        self.source = source
 
     def build(self) -> Any:
         """Return imported object.
 
         Returns:
             Any: Imported object
         """
-        if isinstance(self._module, str):
+        if self.source is not None:
+            module = self.source.build()
             try:
-                module = importlib.import_module(self._module)
-            except Exception as e:
-                raise type(e)(f'Failed to import {self._module} '
-                              f'in {self.location} for {e}')
-
-            if self._imported is not None:
-                if hasattr(module, self._imported):
-                    module = getattr(module, self._imported)
-                else:
-                    raise ImportError(
-                        f'Failed to import {self._imported} '
-                        f'from {self._module} in {self.location}')
-
-            return module
+                return getattr(module, self.name)
+            except AttributeError:
+                raise ImportError(
+                    f'Failed to import {self.name} from {self.source}')
         else:
-            # import xxx.xxx
-            # import xxx.yyy
-            # import xxx.zzz
-            # return imported xxx
             try:
-                for module in self._module:
-                    importlib.import_module(module)  # type: ignore
-                module_name = self._module[0].split('.')[0]
-                return importlib.import_module(module_name)
+                return importlib.import_module(self.name)
             except Exception as e:
-                raise type(e)(f'Failed to import {self.module} '
-                              f'in {self.location} for {e}')
-
-    @property
-    def module(self):
-        if isinstance(self._module, str):
-            return self._module
-        return self._module[0].split('.')[0]
-
-    def __call__(self, *args, **kwargs):
-        raise RuntimeError()
+                raise type(e)(f'Failed to import {self.name} for {e}')
 
     def __deepcopy__(self, memo):
-        return LazyObject(self._module, self._imported, self.location)
+        return LazyObject(self.name, self.source)
 
     def __getattr__(self, name):
-        # Cannot locate the line number of the getting attribute.
-        # Therefore only record the filename.
-        if self.location is not None:
-            location = self.location.split(', line')[0]
-        else:
-            location = self.location
-        return LazyAttr(name, self, location)
+        return LazyObject(name, self)
 
     def __str__(self) -> str:
-        if self._imported is not None:
-            return self._imported
-        return self.module
+        if self.source is not None:
+            return str(self.source) + '.' + self.name
+        return self.name
 
-    __repr__ = __str__
+    def __repr__(self) -> str:
+        return f"<Lazy '{str(self)}'>"
 
+    @property
+    def dump_str(self):
+        return f'<{str(self)}>'
 
-class LazyAttr:
-    """The attribute of the LazyObject.
+    @classmethod
+    def from_str(cls, string):
+        match_ = re.match(r'<([\w\.]+)>', string)
+        if match_ and '.' in match_.group(1):
+            source, _, name = match_.group(1).rpartition('.')
+            return cls(name, cls(source))
+        elif match_:
+            return cls(match_.group(1))
+        return None
 
-    When parsing the configuration file, the imported syntax will be
-    parsed as the assignment ``LazyObject``. During the subsequent parsing
-    process, users may reference the attributes of the LazyObject.
-    To ensure that these attributes also contain information needed to
-    reconstruct the attribute itself, LazyAttr was introduced.
 
-    Examples:
-        >>> models = LazyObject(['mmdet.models'])
-        >>> model = dict(type=models.RetinaNet)
-        >>> print(type(model['type']))  # <class 'mmengine.config.lazy.LazyAttr'>
-        >>> print(model['type'].build())  # <class 'mmdet.models.detectors.retinanet.RetinaNet'>
-    """  # noqa: E501
-
-    def __init__(self,
-                 name: str,
-                 source: Union['LazyObject', 'LazyAttr'],
-                 location=None):
-        self.name = name
-        self.source: Union[LazyAttr, LazyObject] = source
-
-        if isinstance(self.source, LazyObject):
-            if isinstance(self.source._module, str):
-                if self.source._imported is None:
-                    # source code:
-                    # from xxx.yyy import zzz
-                    # equivalent code:
-                    # zzz = LazyObject('xxx.yyy', 'zzz')
-                    # The source code of get attribute:
-                    # eee = zzz.eee
-                    # Then, `eee._module` should be "xxx.yyy.zzz"
-                    self._module = self.source._module
-                else:
-                    # source code:
-                    # import xxx.yyy as zzz
-                    # equivalent code:
-                    # zzz = LazyObject('xxx.yyy')
-                    # The source code of get attribute:
-                    # eee = zzz.eee
-                    # Then, `eee._module` should be "xxx.yyy"
-                    self._module = f'{self.source._module}.{self.source}'
-            else:
-                # The source code of LazyObject should be
-                # 1. import xxx.yyy
-                # 2. import xxx.zzz
-                # Equivalent to
-                # xxx = LazyObject(['xxx.yyy', 'xxx.zzz'])
-
-                # The source code of LazyAttr should be
-                # eee = xxx.eee
-                # Then, eee._module = xxx
-                self._module = str(self.source)
-        elif isinstance(self.source, LazyAttr):
-            # 1. import xxx
-            # 2. zzz = xxx.yyy.zzz
-
-            # Equivalent to:
-            # xxx = LazyObject('xxx')
-            # zzz = xxx.yyy.zzz
-            # zzz._module = xxx.yyy._module + zzz.name
-            self._module = f'{self.source._module}.{self.source.name}'
-        self.location = location
+LazyAttr = LazyObject
 
-    @property
-    def module(self):
-        return self._module
 
-    def __call__(self, *args, **kwargs: Any) -> Any:
-        raise RuntimeError()
+class LazyImportContext:
 
-    def __getattr__(self, name: str) -> 'LazyAttr':
-        return LazyAttr(name, self)
+    def __init__(self, enable=True):
+        self.enable = enable
 
-    def __deepcopy__(self, memo):
-        return LazyAttr(self.name, self.source)
+    def find_spec(self, fullname, path=None, target=None):
+        if not self.enable or 'mmengine.config' in fullname:
+            # avoid lazy import mmengine functions
+            return None
+        spec = spec_from_loader(fullname, self)
+        return spec
 
-    def build(self) -> Any:
-        """Return the attribute of the imported object.
+    def create_module(self, spec):
+        self.lazy_modules.append(spec.name)
+        return LazyObject(spec.name)
 
-        Returns:
-            Any: attribute of the imported object.
-        """
-        obj = self.source.build()
-        try:
-            return getattr(obj, self.name)
-        except AttributeError:
-            raise ImportError(f'Failed to import {self.module}.{self.name} in '
-                              f'{self.location}')
-        except ImportError as e:
-            raise e
+    @classmethod
+    def exec_module(self, module):
+        pass
 
-    def __str__(self) -> str:
-        return self.name
+    def __enter__(self):
+        # insert after FrozenImporter
+        index = sys.meta_path.index(importlib.machinery.FrozenImporter)
+        sys.meta_path.insert(index + 1, self)
+        self.lazy_modules = []
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.meta_path.remove(self)
+        for name in self.lazy_modules:
+            sys.modules.pop(name, None)
 
-    __repr__ = __str__
+    def __repr__(self):
+        return f'<LazyImportContext (enable={self.enable})>'