From 8d076bd815c77218afd4f620b04017a900b1522c Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Wed, 28 Jun 2023 16:40:49 +0800 Subject: [PATCH 01/10] [Improve] Use meta path instead of AST transform to implement new config. --- .pre-commit-config.yaml | 2 +- mmengine/config/__init__.py | 3 +- mmengine/config/config.py | 21 +- mmengine/config/lazy.py | 228 +++------ mmengine/config/new_config.py | 682 ++++++++++++++++++++++++++ mmengine/registry/registry.py | 23 +- mmengine/runner/runner.py | 3 +- mmengine/visualization/vis_backend.py | 3 +- tests/test_config/test_config.py | 12 +- 9 files changed, 771 insertions(+), 206 deletions(-) create mode 100644 mmengine/config/new_config.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b8da3b5f14..c9dae564d2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: args: ["mmengine", "tests"] - id: remove-improper-eol-in-cn-docs - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.812 + rev: v1.4.1 hooks: - id: mypy exclude: "docs" diff --git a/mmengine/config/__init__.py b/mmengine/config/__init__.py index 9a1bc47db4..71292d274c 100644 --- a/mmengine/config/__init__.py +++ b/mmengine/config/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. -from .config import Config, ConfigDict, DictAction, read_base +from .config import Config, ConfigDict, DictAction +from .new_config import read_base __all__ = ['Config', 'ConfigDict', 'DictAction', 'read_base'] diff --git a/mmengine/config/config.py b/mmengine/config/config.py index 33ae816f4d..ffd0948928 100644 --- a/mmengine/config/config.py +++ b/mmengine/config/config.py @@ -471,25 +471,8 @@ def fromfile(filename: Union[str, Path], env_variables=env_variables, ) else: - # Enable lazy import when parsing the config. - # Using try-except to make sure ``ConfigDict.lazy`` will be reset - # to False. See more details about lazy in the docstring of - # ConfigDict - ConfigDict.lazy = True - try: - cfg_dict, imported_names = Config._parse_lazy_import(filename) - except Exception as e: - raise e - finally: - # disable lazy import to get the real type. See more details - # about lazy in the docstring of ConfigDict - ConfigDict.lazy = False - - cfg = Config( - cfg_dict, - filename=filename, - format_python_code=format_python_code) - object.__setattr__(cfg, '_imported_names', imported_names) + from .new_config import Config as NewConfig + cfg = NewConfig.fromfile(filename, lazy_import=lazy_import) return cfg @staticmethod diff --git a/mmengine/config/lazy.py b/mmengine/config/lazy.py index e83cce7c89..3bb86aaddb 100644 --- a/mmengine/config/lazy.py +++ b/mmengine/config/lazy.py @@ -1,8 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. import importlib -from typing import Any, Optional, Union - -from mmengine.utils import is_seq_of +import re +import sys +from importlib.util import spec_from_loader +from typing import Any class LazyObject: @@ -38,24 +39,9 @@ class LazyObject: module statement happened. """ - def __init__(self, - module: Union[str, list, tuple], - imported: Optional[str] = None, - location: Optional[str] = None): - if not isinstance(module, str) and not is_seq_of(module, str): - raise TypeError('module should be `str`, `list`, or `tuple`' - f'but got {type(module)}, this might be ' - 'a bug of MMEngine, please report it to ' - 'https://github.com/open-mmlab/mmengine/issues') - self._module: Union[str, list, tuple] = module - - if not isinstance(imported, str) and imported is not None: - raise TypeError('imported should be `str` or None, but got ' - f'{type(imported)}, this might be ' - 'a bug of MMEngine, please report it to ' - 'https://github.com/open-mmlab/mmengine/issues') - self._imported = imported - self.location = location + def __init__(self, name: str, source: 'LazyObject' = None): + self.name = name + self.source = source def build(self) -> Any: """Return imported object. @@ -63,64 +49,49 @@ def build(self) -> Any: Returns: Any: Imported object """ - if isinstance(self._module, str): + if self.source is not None: + module = self.source.build() try: - module = importlib.import_module(self._module) - except Exception as e: - raise type(e)(f'Failed to import {self._module} ' - f'in {self.location} for {e}') - - if self._imported is not None: - if hasattr(module, self._imported): - module = getattr(module, self._imported) - else: - raise ImportError( - f'Failed to import {self._imported} ' - f'from {self._module} in {self.location}') - - return module + return getattr(module, self.name) + except AttributeError: + raise ImportError( + f'Failed to import {self.name} from {self.source}') else: - # import xxx.xxx - # import xxx.yyy - # import xxx.zzz - # return imported xxx try: - for module in self._module: - importlib.import_module(module) # type: ignore - module_name = self._module[0].split('.')[0] - return importlib.import_module(module_name) + return importlib.import_module(self.name) except Exception as e: - raise type(e)(f'Failed to import {self.module} ' - f'in {self.location} for {e}') - - @property - def module(self): - if isinstance(self._module, str): - return self._module - return self._module[0].split('.')[0] - - def __call__(self, *args, **kwargs): - raise RuntimeError() + raise type(e)(f'Failed to import {self.name} for {e}') def __deepcopy__(self, memo): - return LazyObject(self._module, self._imported, self.location) + return LazyObject(self.name, self.source) def __getattr__(self, name): - # Cannot locate the line number of the getting attribute. - # Therefore only record the filename. - if self.location is not None: - location = self.location.split(', line')[0] - else: - location = self.location - return LazyAttr(name, self, location) + return LazyObject(name, self) def __str__(self) -> str: - if self._imported is not None: - return self._imported - return self.module + if self.source is not None: + return str(self.source) + '.' + self.name + return self.name __repr__ = __str__ + def __repr__(self) -> str: + return f"" + + @property + def dump_str(self): + return f'<{str(self)}>' + + @classmethod + def from_str(cls, string): + match_ = re.match(r'<([\w\.]+)>', string) + if match_ and '.' in match_.group(1): + source, _, name = match_.group(1).rpartition('.') + return cls(name, cls(source)) + elif match_: + return cls(match_.group(1)) + return None + # `pickle.dump` will try to get the `__getstate__` and `__setstate__` # methods of the dumped object. If these two methods are not defined, # LazyObject will return a `__getstate__` LazyObject` or `__setstate__` @@ -132,110 +103,39 @@ def __setstate__(self, state): self.__dict__ = state -class LazyAttr: - """The attribute of the LazyObject. - - When parsing the configuration file, the imported syntax will be - parsed as the assignment ``LazyObject``. During the subsequent parsing - process, users may reference the attributes of the LazyObject. - To ensure that these attributes also contain information needed to - reconstruct the attribute itself, LazyAttr was introduced. - - Examples: - >>> models = LazyObject(['mmdet.models']) - >>> model = dict(type=models.RetinaNet) - >>> print(type(model['type'])) # - >>> print(model['type'].build()) # - """ # noqa: E501 - - def __init__(self, - name: str, - source: Union['LazyObject', 'LazyAttr'], - location=None): - self.name = name - self.source: Union[LazyAttr, LazyObject] = source - - if isinstance(self.source, LazyObject): - if isinstance(self.source._module, str): - if self.source._imported is None: - # source code: - # from xxx.yyy import zzz - # equivalent code: - # zzz = LazyObject('xxx.yyy', 'zzz') - # The source code of get attribute: - # eee = zzz.eee - # Then, `eee._module` should be "xxx.yyy.zzz" - self._module = self.source._module - else: - # source code: - # import xxx.yyy as zzz - # equivalent code: - # zzz = LazyObject('xxx.yyy') - # The source code of get attribute: - # eee = zzz.eee - # Then, `eee._module` should be "xxx.yyy" - self._module = f'{self.source._module}.{self.source}' - else: - # The source code of LazyObject should be - # 1. import xxx.yyy - # 2. import xxx.zzz - # Equivalent to - # xxx = LazyObject(['xxx.yyy', 'xxx.zzz']) - - # The source code of LazyAttr should be - # eee = xxx.eee - # Then, eee._module = xxx - self._module = str(self.source) - elif isinstance(self.source, LazyAttr): - # 1. import xxx - # 2. zzz = xxx.yyy.zzz - - # Equivalent to: - # xxx = LazyObject('xxx') - # zzz = xxx.yyy.zzz - # zzz._module = xxx.yyy._module + zzz.name - self._module = f'{self.source._module}.{self.source.name}' - self.location = location - - @property - def module(self): - return self._module +LazyAttr = LazyObject - def __call__(self, *args, **kwargs: Any) -> Any: - raise RuntimeError() - def __getattr__(self, name: str) -> 'LazyAttr': - return LazyAttr(name, self) +class LazyImportContext: - def __deepcopy__(self, memo): - return LazyAttr(self.name, self.source) + def __init__(self, enable=True): + self.enable = enable - def build(self) -> Any: - """Return the attribute of the imported object. + def find_spec(self, fullname, path=None, target=None): + if not self.enable or 'mmengine.config' in fullname: + # avoid lazy import mmengine functions + return None + spec = spec_from_loader(fullname, self) + return spec - Returns: - Any: attribute of the imported object. - """ - obj = self.source.build() - try: - return getattr(obj, self.name) - except AttributeError: - raise ImportError(f'Failed to import {self.module}.{self.name} in ' - f'{self.location}') - except ImportError as e: - raise e + def create_module(self, spec): + self.lazy_modules.append(spec.name) + return LazyObject(spec.name) - def __str__(self) -> str: - return self.name + @classmethod + def exec_module(self, module): + pass - __repr__ = __str__ + def __enter__(self): + # insert after FrozenImporter + index = sys.meta_path.index(importlib.machinery.FrozenImporter) + sys.meta_path.insert(index + 1, self) + self.lazy_modules = [] - # `pickle.dump` will try to get the `__getstate__` and `__setstate__` - # methods of the dumped object. If these two methods are not defined, - # LazyAttr will return a `__getstate__` LazyAttr` or `__setstate__` - # LazyAttr. - def __getstate__(self): - return self.__dict__ + def __exit__(self, exc_type, exc_val, exc_tb): + sys.meta_path.remove(self) + for name in self.lazy_modules: + sys.modules.pop(name, None) - def __setstate__(self, state): - self.__dict__ = state + def __repr__(self): + return f'' diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py new file mode 100644 index 0000000000..3f36c6fe11 --- /dev/null +++ b/mmengine/config/new_config.py @@ -0,0 +1,682 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import importlib +import inspect +import platform +import sys +from importlib.abc import Loader, MetaPathFinder +from importlib.machinery import PathFinder +from importlib.util import spec_from_loader +from pathlib import Path +from types import BuiltinFunctionType, FunctionType, ModuleType +from typing import Any, Optional, Tuple, Union + +from yapf.yapflib.yapf_api import FormatCode + +from mmengine.fileio import dump +from .config import ConfigDict +from .lazy import LazyImportContext, LazyObject + +DELETE_KEY = '_delete_' +RESERVED_KEYS = ['filename', 'text', 'pretty_text'] + +if platform.system() == 'Windows': + import regex as re +else: + import re # type: ignore + + +def format_inpsect(obj): + file = inspect.getsourcefile(obj) + lines, lineno = inspect.getsourcelines(obj) + msg = f'File "{file}", line {lineno}\n--> {lines[0]}' + return msg + + +def recover_lazy_field(cfg): + + if isinstance(cfg, dict): + for k, v in cfg.items(): + cfg[k] = recover_lazy_field(v) + return cfg + elif isinstance(cfg, (tuple, list)): + container_type = type(cfg) + cfg = list(cfg) + for i, v in enumerate(cfg): + cfg[i] = recover_lazy_field(v) + return container_type(cfg) + elif isinstance(cfg, str): + recover = LazyObject.from_str(cfg) + return recover if recover is not None else cfg + return cfg + + +class Config: + """A facility for config and config files. + + It supports common file formats as configs: python/json/yaml. + ``Config.fromfile`` can parse a dictionary from a config file, then + build a ``Config`` instance with the dictionary. + The interface is the same as a dict object and also allows access config + values as attributes. + + Args: + cfg_dict (dict, optional): A config dictionary. Defaults to None. + cfg_text (str, optional): Text of config. Defaults to None. + filename (str or Path, optional): Name of config file. + Defaults to None. + + Here is a simple example: + + Examples: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/username/projects/mmengine/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/username/projects/mmengine/tests/data/config/a.py] + :" + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + + You can find more advance usage in the `config tutorial`_. + + .. _config tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html + """ # noqa: E501 + _max_parent_depth = 4 + _parent_pkg = '_cfg_parent' + + def __init__(self, + cfg_dict: dict = None, + cfg_text: Optional[str] = None, + filename: Optional[Union[str, Path]] = None, + format_python_code: bool = True): + filename = str(filename) if isinstance(filename, Path) else filename + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError('cfg_dict must be a dict, but ' + f'got {type(cfg_dict)}') + + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f'{key} is reserved for config file') + self._sanity_check(cfg_dict) + + if not isinstance(cfg_dict, ConfigDict): + cfg_dict = ConfigDict(cfg_dict) + super().__setattr__('_cfg_dict', cfg_dict) + super().__setattr__('_filename', filename) + super().__setattr__('_format_python_code', format_python_code) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, encoding='utf-8') as f: + text = f.read() + else: + text = '' + super().__setattr__('_text', text) + + @staticmethod + def _sanity_check(cfg): + if isinstance(cfg, dict): + for v in cfg.values(): + Config._sanity_check(v) + elif isinstance(cfg, (tuple, list, set)): + for v in cfg: + Config._sanity_check(v) + elif isinstance(cfg, (type, FunctionType)): + if (Config._parent_pkg in cfg.__module__ + or '__main__' in cfg.__module__): + msg = ('You cannot use temporary functions ' + 'as the value of a field.\n\n') + msg += format_inpsect(cfg) + raise ValueError(msg) + + @staticmethod + def fromfile(filename: Union[str, Path], + lazy_import: Optional[bool] = None) -> 'Config': + """Build a Config instance from config file. + + Args: + filename (str or Path): Name of config file. + use_predefined_variables (bool, optional): Whether to use + predefined variables. Defaults to True. + import_custom_modules (bool, optional): Whether to support + importing custom modules in config. Defaults to None. + lazy_import (bool): Whether to load config in `lazy_import` mode. + If it is `None`, it will be deduced by the content of the + config file. Defaults to None. + + Returns: + Config: Config instance built from config file. + """ + # Enable lazy import when parsing the config. + # Using try-except to make sure ``ConfigDict.lazy`` will be reset + # to False. See more details about lazy in the docstring of + # ConfigDict + ConfigDict.lazy = lazy_import + try: + cfg_dict = Config._parse_lazy_import(filename) + finally: + ConfigDict.lazy = False + + for key, value in list(cfg_dict.to_dict().items()): + # Remove functions or modules + if isinstance(value, (LazyObject, ModuleType, FunctionType, type)): + cfg_dict.pop(key) + + # Recover dumped lazy object like '' from string + cfg_dict = recover_lazy_field(cfg_dict) + + cfg = Config(cfg_dict, filename=filename) + return cfg + + @staticmethod + def _parse_lazy_import(filename: Union[str, Path]) -> ConfigDict: + """Transform file to variables dictionary. + + Args: + filename (str): Name of config file. + + Returns: + Tuple[dict, dict]: ``cfg_dict`` and ``imported_names``. + + - cfg_dict (dict): Variables dictionary of parsed config. + - imported_names (set): Used to mark the names of + imported object. + """ + module = Config._get_config_module(filename) + module_dict = { + k: getattr(module, k) + for k in dir(module) if not k.startswith('__') + } + + return ConfigDict(module_dict) + + @staticmethod + def _get_config_module(filename: Union[str, Path], level=0): + file = Path(filename).absolute() + module_name = re.sub(r'\W|^(?=\d)', '_', file.stem) + parent_pkg = Config._parent_pkg + str(level) + fullname = '.'.join([parent_pkg] * Config._max_parent_depth + + [module_name]) + + # import config file as a module + with LazyImportContext(): + spec = importlib.util.spec_from_file_location(fullname, file) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + return module + + @staticmethod + def _dict_to_config_dict_lazy(cfg: dict): + """Recursively converts ``dict`` to :obj:`ConfigDict`. The only + difference between ``_dict_to_config_dict_lazy`` and + ``_dict_to_config_dict_lazy`` is that the former one does not consider + the scope, and will not trigger the building of ``LazyObject``. + + Args: + cfg (dict): Config dict. + + Returns: + ConfigDict: Converted dict. + """ + # Only the outer dict with key `type` should have the key `_scope_`. + if isinstance(cfg, dict): + cfg_dict = ConfigDict() + for key, value in cfg.items(): + cfg_dict[key] = Config._dict_to_config_dict_lazy(value) + return cfg_dict + if isinstance(cfg, (tuple, list)): + return type(cfg)( + Config._dict_to_config_dict_lazy(_cfg) for _cfg in cfg) + return cfg + + @staticmethod + def _merge_a_into_b(a: dict, + b: dict, + allow_list_keys: bool = False) -> dict: + """merge dict ``a`` into dict ``b`` (non-inplace). + + Values in ``a`` will overwrite ``b``. ``b`` is copied first to avoid + in-place modifications. + + Args: + a (dict): The source dict to be merged into ``b``. + b (dict): The origin dict to be fetch keys from ``a``. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in source ``a`` and will replace the element of the + corresponding index in b if b is a list. Defaults to False. + + Returns: + dict: The modified dict of ``b`` using ``a``. + + Examples: + # Normally merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # Delete b first and merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(_delete_=True, a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # b is a list + >>> Config._merge_a_into_b( + ... {'0': dict(a=2)}, [dict(a=1), dict(b=2)], True) + [{'a': 2}, {'b': 2}] + """ + b = b.copy() + for k, v in a.items(): + if allow_list_keys and k.isdigit() and isinstance(b, list): + k = int(k) + if len(b) <= k: + raise KeyError(f'Index {k} exceeds the length of list {b}') + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + elif isinstance(v, dict): + if k in b and not v.pop(DELETE_KEY, False): + allowed_types: Union[Tuple, type] = ( + dict, list) if allow_list_keys else dict + if not isinstance(b[k], allowed_types): + raise TypeError( + f'{k}={v} in child config cannot inherit from ' + f'base because {k} is a dict in the child config ' + f'but is of type {type(b[k])} in base config. ' + f'You may set `{DELETE_KEY}=True` to ignore the ' + f'base config.') + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + else: + b[k] = ConfigDict(v) + else: + b[k] = v + return b + + @property + def filename(self) -> str: + """get file name of config.""" + return self._filename + + @property + def text(self) -> str: + """get config text.""" + return self._text + + @property + def pretty_text(self) -> str: + """get formatted python config text.""" + + def _format_dict(input_dict): + use_mapping = not all(str(k).isidentifier() for k in input_dict) + + if use_mapping: + item_tmpl = '{k}: {v}' + else: + item_tmpl = '{k}={v}' + + items = [] + for k, v in input_dict.items(): + v_str = _format_basic_types(v) + k_str = _format_basic_types(k) if use_mapping else k + items.append(item_tmpl.format(k=k_str, v=v_str)) + items = ','.join(items) + + if use_mapping: + return '{' + items + '}' + else: + return f'dict({items})' + + def _format_list_tuple_set(input_container): + items = [] + + for item in input_container: + items.append(_format_basic_types(item)) + + if isinstance(input_container, tuple): + items = items + [''] if len(items) == 1 else items + return '(' + ','.join(items) + ')' + elif isinstance(input_container, list): + return '[' + ','.join(items) + ']' + elif isinstance(input_container, set): + return '{' + ','.join(items) + '}' + + def _format_basic_types(input_): + if isinstance(input_, str): + return repr(input_) + elif isinstance(input_, dict): + return _format_dict(input_) + elif isinstance(input_, (list, set, tuple)): + return _format_list_tuple_set(input_) + elif isinstance(input_, LazyObject): + return repr(input_.dump_str) + elif isinstance(input_, (type, FunctionType, BuiltinFunctionType)): + if Config._parent_pkg in input_.__module__: + # defined in the config file. + module = input_.__module__.rpartition('.')[-1] + else: + module = input_.__module__ + return repr('<' + module + '.' + input_.__name__ + '>') + elif isinstance(input_, ModuleType): + return repr(f'<{input_.__name__}>') + elif 'torch.dtype' in str(type(input_)): + return repr('<' + str(input_) + '>') + else: + return str(input_) + + cfg_dict = self._to_lazy_dict() + + items = [] + for k, v in cfg_dict.items(): + items.append(f'{k} = {_format_basic_types(v)}') + + text = '\n'.join(items) + if self._format_python_code: + # copied from setup.cfg + yapf_style = dict( + based_on_style='pep8', + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True) + try: + text, _ = FormatCode( + text, style_config=yapf_style, verify=True) + except: # noqa: E722 + raise SyntaxError('Failed to format the config file, please ' + f'check the syntax of: \n{text}') + + return text + + def __repr__(self): + return f'Config (path: {self.filename}): {self._cfg_dict.__repr__()}' + + def __len__(self): + return len(self._cfg_dict) + + def __getattr__(self, name: str) -> Any: + return getattr(self._cfg_dict, name) + + def __getitem__(self, name): + return self._cfg_dict.__getitem__(name) + + def __setattr__(self, name, value): + self._sanity_check(value) + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setattr__(name, value) + + def __setitem__(self, name, value): + self._sanity_check(value) + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setitem__(name, value) + + def __iter__(self): + return iter(self._cfg_dict) + + def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str], dict]: + return (self._cfg_dict, self._filename, self._text) + + def __deepcopy__(self, memo): + cls = self.__class__ + other = cls.__new__(cls) + memo[id(self)] = other + + for key, value in self.__dict__.items(): + super(Config, other).__setattr__(key, copy.deepcopy(value, memo)) + + return other + + def __copy__(self): + cls = self.__class__ + other = cls.__new__(cls) + other.__dict__.update(self.__dict__) + + return other + + def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], + dict]): + _cfg_dict, _filename, _text = state + super().__setattr__('_cfg_dict', _cfg_dict) + super().__setattr__('_filename', _filename) + super().__setattr__('_text', _text) + + def dump(self, file: Optional[Union[str, Path]] = None): + """Dump config to file or return config text. + + Args: + file (str or Path, optional): If not specified, then the object + is dumped to a str, otherwise to a file specified by the filename. + Defaults to None. + + Returns: + str or None: Config text. + """ + file = str(file) if isinstance(file, Path) else file + cfg_dict = super().__getattribute__('_cfg_dict').to_dict() + if file is None: + if self.filename is None or self.filename.endswith('.py'): + return self.pretty_text + else: + file_format = self.filename.split('.')[-1] + return dump(cfg_dict, file_format=file_format) + elif file.endswith('.py'): + with open(file, 'w', encoding='utf-8') as f: + f.write(self.pretty_text) + else: + file_format = file.split('.')[-1] + return dump(cfg_dict, file=file, file_format=file_format) + + def merge_from_dict(self, + options: dict, + allow_list_keys: bool = True) -> None: + """Merge list into cfg_dict. + + Merge the dict parsed by MultipleKVAction into this cfg. + + Args: + options (dict): dict of configs to merge from. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in ``options`` and will replace the element of the + corresponding index in the config if the config is a list. + Defaults to True. + + Examples: + >>> from mmengine import Config + >>> # Merge dictionary element + >>> options = {'model.backbone.depth': 50, 'model.backbone.with_cp': True} + >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet')))) + >>> cfg.merge_from_dict(options) + >>> cfg._cfg_dict + {'model': {'backbone': {'type': 'ResNet', 'depth': 50, 'with_cp': True}}} + >>> # Merge list element + >>> cfg = Config( + >>> dict(pipeline=[dict(type='LoadImage'), + >>> dict(type='LoadAnnotations')])) + >>> options = dict(pipeline={'0': dict(type='SelfLoadImage')}) + >>> cfg.merge_from_dict(options, allow_list_keys=True) + >>> cfg._cfg_dict + {'pipeline': [{'type': 'SelfLoadImage'}, {'type': 'LoadAnnotations'}]} + """ # noqa: E501 + option_cfg_dict: dict = {} + for full_key, v in options.items(): + d = option_cfg_dict + key_list = full_key.split('.') + for subkey in key_list[:-1]: + d.setdefault(subkey, ConfigDict()) + d = d[subkey] + subkey = key_list[-1] + d[subkey] = v + + cfg_dict = super().__getattribute__('_cfg_dict') + super().__setattr__( + '_cfg_dict', + Config._merge_a_into_b( + option_cfg_dict, cfg_dict, allow_list_keys=allow_list_keys)) + + def _to_lazy_dict(self, keep_imported: bool = False) -> dict: + """Convert config object to dictionary and filter the imported + object.""" + res = self._cfg_dict.to_dict() + + def filter_item(item): + _, v = item + if isinstance(v, (LazyObject, ModuleType, FunctionType, type)): + return False + if v is read_base: + return False + return True + + if keep_imported: + return res + else: + return dict(filter(filter_item, res.items())) + + def to_dict(self, keep_imported: bool = False): + """Convert all data in the config to a builtin ``dict``. + + Args: + keep_imported (bool): Whether to keep the imported field. + Defaults to False + + If you import third-party objects in the config file, all imported + objects will be converted to a string like ``torch.optim.SGD`` + """ + _cfg_dict = self._to_lazy_dict(keep_imported=keep_imported) + + def lazy2string(cfg_dict): + if isinstance(cfg_dict, dict): + return type(cfg_dict)( + {k: lazy2string(v) + for k, v in cfg_dict.items()}) + elif isinstance(cfg_dict, (tuple, list)): + return type(cfg_dict)(lazy2string(v) for v in cfg_dict) + elif isinstance(cfg_dict, LazyObject): + return str(cfg_dict) + else: + return cfg_dict + + return lazy2string(_cfg_dict) + + +class BaseConfigLoader(Loader): + + def __init__(self, filepath, level) -> None: + self.filepath = filepath + self.level = level + + def create_module(self, spec): + file = self.filepath + return Config._get_config_module(file, level=self.level) + + def exec_module(self, module): + for k in dir(module): + module.__dict__[k] = Config._dict_to_config_dict_lazy( + getattr(module, k)) + + +class ParentFolderLoader(Loader): + + @staticmethod + def create_module(spec): + return ModuleType(spec.name) + + @staticmethod + def exec_module(module): + pass + + +class BaseImportContext(MetaPathFinder): + + def find_spec(self, fullname, path=None, target=None): + """Try to find a spec for 'fullname' on sys.path or 'path'. + + The search is based on sys.path_hooks and sys.path_importer_cache. + """ + parent_pkg = Config._parent_pkg + str(self.level) + names = fullname.split('.') + + if names[-1] == parent_pkg: + self.base_modules.append(fullname) + # Create parent package + return spec_from_loader( + fullname, loader=ParentFolderLoader, is_package=True) + elif names[0] == parent_pkg: + self.base_modules.append(fullname) + # relative imported base package + filepath = self.root_path + for name in names: + if name == parent_pkg: + # Use parent to remove `..` at the end of the root path + filepath = filepath.parent + else: + filepath = filepath / name + if filepath.is_dir(): + # If a dir, create a package. + return spec_from_loader( + fullname, loader=ParentFolderLoader, is_package=True) + + pypath = filepath.with_suffix('.py') + + if not pypath.exists(): + raise ImportError(f'Not found base path {filepath.resolve()}') + return importlib.util.spec_from_loader( + fullname, BaseConfigLoader(pypath, self.level + 1)) + else: + # Absolute import + pkg = PathFinder.find_spec(names[0]) + if pkg and pkg.submodule_search_locations: + self.base_modules.append(fullname) + path = Path(pkg.submodule_search_locations[0]) + for name in names[1:]: + path = path / name + if path.is_dir(): + return spec_from_loader( + fullname, loader=ParentFolderLoader, is_package=True) + pypath = path.with_suffix('.py') + if not pypath.exists(): + raise ImportError(f'Not found base path {path.resolve()}') + return importlib.util.spec_from_loader( + fullname, BaseConfigLoader(pypath, self.level + 1)) + return None + + def __enter__(self): + # call from which file + stack = inspect.stack()[1] + file = inspect.getfile(stack[0]) + folder = Path(file).parent + self.root_path = folder.joinpath(*(['..'] * Config._max_parent_depth)) + + self.base_modules = [] + self.level = len( + [p for p in sys.meta_path if isinstance(p, BaseImportContext)]) + + # Disable enabled lazy loader during parsing base + self.lazy_importers = [] + for p in sys.meta_path: + if isinstance(p, LazyImportContext) and p.enable: + self.lazy_importers.append(p) + p.enable = False + + index = sys.meta_path.index(importlib.machinery.FrozenImporter) + sys.meta_path.insert(index + 1, self) + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.meta_path.remove(self) + for name in self.base_modules: + sys.modules.pop(name, None) + for p in self.lazy_importers: + p.enable = True + + def __repr__(self): + return f'' + + +read_base = BaseImportContext diff --git a/mmengine/registry/registry.py b/mmengine/registry/registry.py index 31fd44d827..94588c30a8 100644 --- a/mmengine/registry/registry.py +++ b/mmengine/registry/registry.py @@ -11,7 +11,8 @@ from rich.table import Table from mmengine.config.utils import MODULE2PACKAGE -from mmengine.utils import get_object_from_string, is_seq_of +from mmengine.config.lazy import LazyObject +from mmengine.utils import is_seq_of from .default_scope import DefaultScope @@ -442,6 +443,13 @@ def get(self, key: str) -> Optional[Type]: 'The key argument of `Registry.get` must be a str, ' f'got {type(key)}') + try: + obj_cls = LazyObject.from_str(key) + if obj_cls is not None: + return obj_cls.build() + except Exception: + raise RuntimeError(f'Failed to get {key}') + scope, real_key = self.split_scope_key(key) obj_cls = None registry_name = self.name @@ -495,18 +503,6 @@ def get(self, key: str) -> Optional[Type]: else: obj_cls = root.get(key) - if obj_cls is None: - # Actually, it's strange to implement this `try ... except` to - # get the object by its name in `Registry.get`. However, If we - # want to build the model using a configuration like - # `dict(type='mmengine.model.BaseModel')`, which can - # be dumped by lazy import config, we need this code snippet - # for `Registry.get` to work. - try: - obj_cls = get_object_from_string(key) - except Exception: - raise RuntimeError(f'Failed to get {key}') - if obj_cls is not None: # For some rare cases (e.g. obj_cls is a partial function), obj_cls # doesn't have `__name__`. Use default value to prevent error @@ -516,7 +512,6 @@ def get(self, key: str) -> Optional[Type]: f' registry in "{scope_name}"', logger='current', level=logging.DEBUG) - return obj_cls def _search_child(self, scope: str) -> Optional['Registry']: diff --git a/mmengine/runner/runner.py b/mmengine/runner/runner.py index 12830cf4ad..1811f59165 100644 --- a/mmengine/runner/runner.py +++ b/mmengine/runner/runner.py @@ -277,8 +277,9 @@ def __init__( # recursively copy the `cfg` because `self.cfg` will be modified # everywhere. + from mmengine.config.new_config import Config as NewConfig if cfg is not None: - if isinstance(cfg, Config): + if isinstance(cfg, (Config, NewConfig)): self.cfg = copy.deepcopy(cfg) elif isinstance(cfg, dict): self.cfg = Config(cfg) diff --git a/mmengine/visualization/vis_backend.py b/mmengine/visualization/vis_backend.py index a1d896ea5b..fa75f7e1b0 100644 --- a/mmengine/visualization/vis_backend.py +++ b/mmengine/visualization/vis_backend.py @@ -14,6 +14,7 @@ import torch from mmengine.config import Config +from mmengine.config.new_config import Config as NewConfig from mmengine.fileio import dump from mmengine.hooks.logger_hook import SUFFIX_TYPE from mmengine.logging import MMLogger, print_log @@ -235,7 +236,7 @@ def add_config(self, config: Config, **kwargs) -> None: Args: config (Config): The Config object """ - assert isinstance(config, Config) + assert isinstance(config, (Config, NewConfig)) config.dump(self._config_save_file) @force_init_env diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 474a7d7bf4..f12895110a 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -993,6 +993,8 @@ def _compare_dict(a, b): assert len(a) == len(b) for item_a, item_b in zip(a, b): _compare_dict(item_a, item_b) + elif isinstance(a, type): + assert a.__module__ + a.__name__ == str(b) else: assert str(a) == str(b) @@ -1015,7 +1017,7 @@ def _compare_dict(a, b): error_obj = tmp_path / 'error_obj.py' error_obj.write_text("""from mmengine.fileio import error_obj""") # match pattern should be double escaped - match = str(error_obj).encode('unicode_escape').decode() + match = 'Failed to import mmengine.fileio.error_obj' with pytest.raises(ImportError, match=match): cfg = Config.fromfile(str(error_obj)) cfg.error_obj @@ -1025,14 +1027,14 @@ def _compare_dict(a, b): import mmengine error_attr = mmengine.error_attr """) # noqa: E122 - match = str(error_attr).encode('unicode_escape').decode() - with pytest.raises(ImportError, match=match): + match = "module 'mmengine' has no attribute 'error_attr'" + with pytest.raises(AttributeError, match=match): cfg = Config.fromfile(str(error_attr)) cfg.error_attr error_module = tmp_path / 'error_module.py' error_module.write_text("""import error_module""") - match = str(error_module).encode('unicode_escape').decode() + match = 'Failed to import error_module' with pytest.raises(ImportError, match=match): cfg = Config.fromfile(str(error_module)) cfg.error_module @@ -1051,7 +1053,7 @@ def _compare_dict(a, b): lazy_import=False) # current lazy-import config, base text config - with pytest.raises(RuntimeError, match='_base_ ='): + with pytest.raises(AttributeError, match='item2'): Config.fromfile( osp.join(self.data_path, 'config/lazy_module_config/error_mix_using2.py')) From 1793ea445a719d6a437eedab44eee369b70eaa0e Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Tue, 11 Jul 2023 06:31:24 +0000 Subject: [PATCH 02/10] Split config to old version and new version --- .pre-commit-config.yaml | 2 +- mmengine/config/config.py | 930 ++------------------------ mmengine/config/lazy.py | 3 - mmengine/config/new_config.py | 350 +++------- mmengine/config/old_config.py | 679 +++++++++++++++++++ mmengine/runner/runner.py | 3 +- mmengine/visualization/vis_backend.py | 3 +- tests/test_config/test_config.py | 116 ++-- 8 files changed, 872 insertions(+), 1214 deletions(-) create mode 100644 mmengine/config/old_config.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c9dae564d2..b8da3b5f14 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: args: ["mmengine", "tests"] - id: remove-improper-eol-in-cn-docs - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.4.1 + rev: v0.812 hooks: - id: mypy exclude: "docs" diff --git a/mmengine/config/config.py b/mmengine/config/config.py index ffd0948928..438115ad1a 100644 --- a/mmengine/config/config.py +++ b/mmengine/config/config.py @@ -2,14 +2,9 @@ import ast import copy import os -import os.path as osp -import platform -import shutil -import sys import tempfile -import types -import uuid import warnings +from abc import ABCMeta, abstractmethod from argparse import Action, ArgumentParser, Namespace from collections import OrderedDict, abc from contextlib import contextmanager @@ -19,25 +14,13 @@ from addict import Dict from yapf.yapflib.yapf_api import FormatCode -from mmengine.fileio import dump, load +from mmengine.fileio import dump from mmengine.logging import print_log -from mmengine.utils import (check_file_exist, get_installed_path, - import_modules_from_strings, is_installed) -from .lazy import LazyAttr, LazyObject -from .utils import (ConfigParsingError, ImportTransformer, RemoveAssignFromAST, - _gather_abs_import_lazyobj, _get_external_cfg_base_path, - _get_external_cfg_path, _get_package_and_cfg_path, - _is_builtin_module) +from .lazy import LazyObject +from .utils import ConfigParsingError, _is_builtin_module BASE_KEY = '_base_' DELETE_KEY = '_delete_' -DEPRECATION_KEY = '_deprecation_' -RESERVED_KEYS = ['filename', 'text', 'pretty_text', 'env_variables'] - -if platform.system() == 'Windows': - import regex as re -else: - import re # type: ignore def _lazy2string(cfg_dict, dict_type=None): @@ -46,8 +29,8 @@ def _lazy2string(cfg_dict, dict_type=None): return dict_type({k: _lazy2string(v) for k, v in dict.items(cfg_dict)}) elif isinstance(cfg_dict, (tuple, list)): return type(cfg_dict)(_lazy2string(v) for v in cfg_dict) - elif isinstance(cfg_dict, (LazyAttr, LazyObject)): - return f'{cfg_dict.module}.{str(cfg_dict)}' + elif isinstance(cfg_dict, LazyObject): + return cfg_dict.dump_str else: return cfg_dict @@ -100,7 +83,7 @@ def __missing__(self, name): def __getattr__(self, name): try: value = super().__getattr__(name) - if isinstance(value, (LazyAttr, LazyObject)) and not self.lazy: + if isinstance(value, LazyObject) and not self.lazy: value = value.build() except KeyError: raise AttributeError(f"'{self.__class__.__name__}' object has no " @@ -206,7 +189,7 @@ def build_lazy(self, value: Any) -> Any: Returns: Any: The built value. """ - if isinstance(value, (LazyAttr, LazyObject)) and not self.lazy: + if isinstance(value, LazyObject) and not self.lazy: value = value.build() return value @@ -339,7 +322,7 @@ def add_args(parser: ArgumentParser, return parser -class Config: +class Config(metaclass=ABCMeta): """A facility for config and config files. It supports common file formats as configs: python/json/yaml. @@ -381,51 +364,18 @@ class Config: .. _config tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html """ # noqa: E501 - def __init__( - self, - cfg_dict: dict = None, - cfg_text: Optional[str] = None, - filename: Optional[Union[str, Path]] = None, - env_variables: Optional[dict] = None, - format_python_code: bool = True, - ): - filename = str(filename) if isinstance(filename, Path) else filename - if cfg_dict is None: - cfg_dict = dict() - elif not isinstance(cfg_dict, dict): - raise TypeError('cfg_dict must be a dict, but ' - f'got {type(cfg_dict)}') - for key in cfg_dict: - if key in RESERVED_KEYS: - raise KeyError(f'{key} is reserved for config file') - - if not isinstance(cfg_dict, ConfigDict): - cfg_dict = ConfigDict(cfg_dict) - super().__setattr__('_cfg_dict', cfg_dict) - super().__setattr__('_filename', filename) - super().__setattr__('_format_python_code', format_python_code) - if not hasattr(self, '_imported_names'): - super().__setattr__('_imported_names', set()) - - if cfg_text: - text = cfg_text - elif filename: - with open(filename, encoding='utf-8') as f: - text = f.read() - else: - text = '' - super().__setattr__('_text', text) - if env_variables is None: - env_variables = dict() - super().__setattr__('_env_variables', env_variables) + @classmethod + def __new__(cls, *args, **kwargs): + if cls is Config: + from .new_config import ConfigV2 + cls = ConfigV2 + return super().__new__(cls) @staticmethod def fromfile(filename: Union[str, Path], - use_predefined_variables: bool = True, - import_custom_modules: bool = True, - use_environment_variables: bool = True, lazy_import: Optional[bool] = None, - format_python_code: bool = True) -> 'Config': + format_python_code: bool = True, + **kwargs) -> 'Config': """Build a Config instance from config file. Args: @@ -444,36 +394,21 @@ def fromfile(filename: Union[str, Path], Config: Config instance built from config file. """ filename = str(filename) if isinstance(filename, Path) else filename - if lazy_import is False or \ - lazy_import is None and not Config._is_lazy_import(filename): - cfg_dict, cfg_text, env_variables = Config._file2dict( - filename, use_predefined_variables, use_environment_variables, - lazy_import) - if import_custom_modules and cfg_dict.get('custom_imports', None): - try: - import_modules_from_strings(**cfg_dict['custom_imports']) - except ImportError as e: - err_msg = ( - 'Failed to import custom modules from ' - f"{cfg_dict['custom_imports']}, the current sys.path " - 'is: ') - for p in sys.path: - err_msg += f'\n {p}' - err_msg += ( - '\nYou should set `PYTHONPATH` to make `sys.path` ' - 'include the directory which contains your custom ' - 'module') - raise ImportError(err_msg) from e - return Config( - cfg_dict, - cfg_text=cfg_text, + if lazy_import is None: + lazy_import = Config._is_lazy_import(filename) + + if not lazy_import: + from .old_config import ConfigV1 + return ConfigV1.fromfile( filename=filename, - env_variables=env_variables, - ) + format_python_code=format_python_code, + **kwargs) else: - from .new_config import Config as NewConfig - cfg = NewConfig.fromfile(filename, lazy_import=lazy_import) - return cfg + from .new_config import ConfigV2 + return ConfigV2.fromfile( + filename=filename, + format_python_code=format_python_code, + **kwargs) @staticmethod def fromstring(cfg_str: str, file_format: str) -> 'Config': @@ -511,80 +446,6 @@ def fromstring(cfg_str: str, file_format: str) -> 'Config': os.remove(temp_file.name) # manually delete the temporary file return cfg - @staticmethod - def _get_base_modules(nodes: list) -> list: - """Get base module name from parsed code. - - Args: - nodes (list): Parsed code of the config file. - - Returns: - list: Name of base modules. - """ - - def _get_base_module_from_with(with_nodes: list) -> list: - """Get base module name from if statement in python file. - - Args: - with_nodes (list): List of if statement. - - Returns: - list: Name of base modules. - """ - base_modules = [] - for node in with_nodes: - assert isinstance(node, ast.ImportFrom), ( - 'Illegal syntax in config file! Only ' - '`from ... import ...` could be implemented` in ' - 'with read_base()`') - assert node.module is not None, ( - 'Illegal syntax in config file! Syntax like ' - '`from . import xxx` is not allowed in `with read_base()`') - base_modules.append(node.level * '.' + node.module) - return base_modules - - for idx, node in enumerate(nodes): - if (isinstance(node, ast.Assign) - and isinstance(node.targets[0], ast.Name) - and node.targets[0].id == BASE_KEY): - raise ConfigParsingError( - 'The configuration file type in the inheritance chain ' - 'must match the current configuration file type, either ' - '"lazy_import" or non-"lazy_import". You got this error ' - f'since you use the syntax like `_base_ = "{node.targets[0].id}"` ' # noqa: E501 - 'in your config. You should use `with read_base(): ... to` ' # noqa: E501 - 'mark the inherited config file. See more information ' - 'in https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html' # noqa: E501 - ) - - if not isinstance(node, ast.With): - continue - - expr = node.items[0].context_expr - if (not isinstance(expr, ast.Call) - or not expr.func.id == 'read_base' or # type: ignore - len(node.items) > 1): - raise ConfigParsingError( - 'Only `read_base` context manager can be used in the ' - 'config') - - # The original code: - # ``` - # with read_base(): - # from .._base_.default_runtime import * - # ``` - # The processed code: - # ``` - # from .._base_.default_runtime import * - # ``` - # As you can see, the if statement is removed and the - # from ... import statement will be unindent - for nested_idx, nested_node in enumerate(node.body): - nodes.insert(idx + nested_idx + 1, nested_node) - nodes.pop(idx) - return _get_base_module_from_with(node.body) - return [] - @staticmethod def _validate_py_syntax(filename: str): """Validate syntax of python config. @@ -600,658 +461,6 @@ def _validate_py_syntax(filename: str): raise SyntaxError('There are syntax errors in config ' f'file {filename}: {e}') - @staticmethod - def _substitute_predefined_vars(filename: str, temp_config_name: str): - """Substitute predefined variables in config with actual values. - - Sometimes we want some variables in the config to be related to the - current path or file name, etc. - - Here is an example of a typical usage scenario. When training a model, - we define a working directory in the config that save the models and - logs. For different configs, we expect to define different working - directories. A common way for users is to use the config file name - directly as part of the working directory name, e.g. for the config - ``config_setting1.py``, the working directory is - ``. /work_dir/config_setting1``. - - This can be easily achieved using predefined variables, which can be - written in the config `config_setting1.py` as follows - - .. code-block:: python - - work_dir = '. /work_dir/{{ fileBasenameNoExtension }}' - - - Here `{{ fileBasenameNoExtension }}` indicates the file name of the - config (without the extension), and when the config class reads the - config file, it will automatically parse this double-bracketed string - to the corresponding actual value. - - .. code-block:: python - - cfg = Config.fromfile('. /config_setting1.py') - cfg.work_dir # ". /work_dir/config_setting1" - - - For details, Please refer to docs/zh_cn/advanced_tutorials/config.md . - - Args: - filename (str): Filename of config. - temp_config_name (str): Temporary filename to save substituted - config. - """ - file_dirname = osp.dirname(filename) - file_basename = osp.basename(filename) - file_basename_no_extension = osp.splitext(file_basename)[0] - file_extname = osp.splitext(filename)[1] - support_templates = dict( - fileDirname=file_dirname, - fileBasename=file_basename, - fileBasenameNoExtension=file_basename_no_extension, - fileExtname=file_extname) - with open(filename, encoding='utf-8') as f: - config_file = f.read() - for key, value in support_templates.items(): - regexp = r'\{\{\s*' + str(key) + r'\s*\}\}' - value = value.replace('\\', '/') - config_file = re.sub(regexp, value, config_file) - with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: - tmp_config_file.write(config_file) - - @staticmethod - def _substitute_env_variables(filename: str, temp_config_name: str): - """Substitute environment variables in config with actual values. - - Sometimes, we want to change some items in the config with environment - variables. For examples, we expect to change dataset root by setting - ``DATASET_ROOT=/dataset/root/path`` in the command line. This can be - easily achieved by writing lines in the config as follows - - .. code-block:: python - - data_root = '{{$DATASET_ROOT:/default/dataset}}/images' - - - Here, ``{{$DATASET_ROOT:/default/dataset}}`` indicates using the - environment variable ``DATASET_ROOT`` to replace the part between - ``{{}}``. If the ``DATASET_ROOT`` is not set, the default value - ``/default/dataset`` will be used. - - Environment variables not only can replace items in the string, they - can also substitute other types of data in config. In this situation, - we can write the config as below - - .. code-block:: python - - model = dict( - bbox_head = dict(num_classes={{'$NUM_CLASSES:80'}})) - - - For details, Please refer to docs/zh_cn/tutorials/config.md . - - Args: - filename (str): Filename of config. - temp_config_name (str): Temporary filename to save substituted - config. - """ - with open(filename, encoding='utf-8') as f: - config_file = f.read() - regexp = r'\{\{[\'\"]?\s*\$(\w+)\s*\:\s*(\S*?)\s*[\'\"]?\}\}' - keys = re.findall(regexp, config_file) - env_variables = dict() - for var_name, value in keys: - regexp = r'\{\{[\'\"]?\s*\$' + var_name + r'\s*\:\s*' \ - + value + r'\s*[\'\"]?\}\}' - if var_name in os.environ: - value = os.environ[var_name] - env_variables[var_name] = value - print_log( - f'Using env variable `{var_name}` with value of ' - f'{value} to replace item in config.', - logger='current') - if not value: - raise KeyError(f'`{var_name}` cannot be found in `os.environ`.' - f' Please set `{var_name}` in environment or ' - 'give a default value.') - config_file = re.sub(regexp, value, config_file) - - with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: - tmp_config_file.write(config_file) - return env_variables - - @staticmethod - def _pre_substitute_base_vars(filename: str, - temp_config_name: str) -> dict: - """Preceding step for substituting variables in base config with actual - value. - - Args: - filename (str): Filename of config. - temp_config_name (str): Temporary filename to save substituted - config. - - Returns: - dict: A dictionary contains variables in base config. - """ - with open(filename, encoding='utf-8') as f: - config_file = f.read() - base_var_dict = {} - regexp = r'\{\{\s*' + BASE_KEY + r'\.([\w\.]+)\s*\}\}' - base_vars = set(re.findall(regexp, config_file)) - for base_var in base_vars: - randstr = f'_{base_var}_{uuid.uuid4().hex.lower()[:6]}' - base_var_dict[randstr] = base_var - regexp = r'\{\{\s*' + BASE_KEY + r'\.' + base_var + r'\s*\}\}' - config_file = re.sub(regexp, f'"{randstr}"', config_file) - with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: - tmp_config_file.write(config_file) - return base_var_dict - - @staticmethod - def _substitute_base_vars(cfg: Any, base_var_dict: dict, - base_cfg: dict) -> Any: - """Substitute base variables from strings to their actual values. - - Args: - Any : Config dictionary. - base_var_dict (dict): A dictionary contains variables in base - config. - base_cfg (dict): Base config dictionary. - - Returns: - Any : A dictionary with origin base variables - substituted with actual values. - """ - cfg = copy.deepcopy(cfg) - - if isinstance(cfg, dict): - for k, v in cfg.items(): - if isinstance(v, str) and v in base_var_dict: - new_v = base_cfg - for new_k in base_var_dict[v].split('.'): - new_v = new_v[new_k] - cfg[k] = new_v - elif isinstance(v, (list, tuple, dict)): - cfg[k] = Config._substitute_base_vars( - v, base_var_dict, base_cfg) - elif isinstance(cfg, tuple): - cfg = tuple( - Config._substitute_base_vars(c, base_var_dict, base_cfg) - for c in cfg) - elif isinstance(cfg, list): - cfg = [ - Config._substitute_base_vars(c, base_var_dict, base_cfg) - for c in cfg - ] - elif isinstance(cfg, str) and cfg in base_var_dict: - new_v = base_cfg - for new_k in base_var_dict[cfg].split('.'): - new_v = new_v[new_k] - cfg = new_v - - return cfg - - @staticmethod - def _file2dict( - filename: str, - use_predefined_variables: bool = True, - use_environment_variables: bool = True, - lazy_import: Optional[bool] = None) -> Tuple[dict, str, dict]: - """Transform file to variables dictionary. - - Args: - filename (str): Name of config file. - use_predefined_variables (bool, optional): Whether to use - predefined variables. Defaults to True. - lazy_import (bool): Whether to load config in `lazy_import` mode. - If it is `None`, it will be deduced by the content of the - config file. Defaults to None. - - Returns: - Tuple[dict, str]: Variables dictionary and text of Config. - """ - if lazy_import is None and Config._is_lazy_import(filename): - raise RuntimeError( - 'The configuration file type in the inheritance chain ' - 'must match the current configuration file type, either ' - '"lazy_import" or non-"lazy_import". You got this error ' - 'since you use the syntax like `with read_base(): ...` ' - f'or import non-builtin module in {filename}. See more ' - 'information in https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html' # noqa: E501 - ) - - filename = osp.abspath(osp.expanduser(filename)) - check_file_exist(filename) - fileExtname = osp.splitext(filename)[1] - if fileExtname not in ['.py', '.json', '.yaml', '.yml']: - raise OSError('Only py/yml/yaml/json type are supported now!') - try: - with tempfile.TemporaryDirectory() as temp_config_dir: - temp_config_file = tempfile.NamedTemporaryFile( - dir=temp_config_dir, suffix=fileExtname, delete=False) - if platform.system() == 'Windows': - temp_config_file.close() - - # Substitute predefined variables - if use_predefined_variables: - Config._substitute_predefined_vars(filename, - temp_config_file.name) - else: - shutil.copyfile(filename, temp_config_file.name) - # Substitute environment variables - env_variables = dict() - if use_environment_variables: - env_variables = Config._substitute_env_variables( - temp_config_file.name, temp_config_file.name) - # Substitute base variables from placeholders to strings - base_var_dict = Config._pre_substitute_base_vars( - temp_config_file.name, temp_config_file.name) - - # Handle base files - base_cfg_dict = ConfigDict() - cfg_text_list = list() - for base_cfg_path in Config._get_base_files( - temp_config_file.name): - base_cfg_path, scope = Config._get_cfg_path( - base_cfg_path, filename) - _cfg_dict, _cfg_text, _env_variables = Config._file2dict( - filename=base_cfg_path, - use_predefined_variables=use_predefined_variables, - use_environment_variables=use_environment_variables, - lazy_import=lazy_import, - ) - cfg_text_list.append(_cfg_text) - env_variables.update(_env_variables) - duplicate_keys = base_cfg_dict.keys() & _cfg_dict.keys() - if len(duplicate_keys) > 0: - raise KeyError( - 'Duplicate key is not allowed among bases. ' - f'Duplicate keys: {duplicate_keys}') - - # _dict_to_config_dict will do the following things: - # 1. Recursively converts ``dict`` to :obj:`ConfigDict`. - # 2. Set `_scope_` for the outer dict variable for the base - # config. - # 3. Set `scope` attribute for each base variable. - # Different from `_scope_`， `scope` is not a key of base - # dict, `scope` attribute will be parsed to key `_scope_` - # by function `_parse_scope` only if the base variable is - # accessed by the current config. - _cfg_dict = Config._dict_to_config_dict(_cfg_dict, scope) - base_cfg_dict.update(_cfg_dict) - - if filename.endswith('.py'): - with open(temp_config_file.name, encoding='utf-8') as f: - parsed_codes = ast.parse(f.read()) - parsed_codes = RemoveAssignFromAST(BASE_KEY).visit( - parsed_codes) - codeobj = compile(parsed_codes, '', mode='exec') - # Support load global variable in nested function of the - # config. - global_locals_var = {BASE_KEY: base_cfg_dict} - ori_keys = set(global_locals_var.keys()) - eval(codeobj, global_locals_var, global_locals_var) - cfg_dict = { - key: value - for key, value in global_locals_var.items() - if (key not in ori_keys and not key.startswith('__')) - } - elif filename.endswith(('.yml', '.yaml', '.json')): - cfg_dict = load(temp_config_file.name) - # close temp file - for key, value in list(cfg_dict.items()): - if isinstance(value, - (types.FunctionType, types.ModuleType)): - cfg_dict.pop(key) - temp_config_file.close() - - # If the current config accesses a base variable of base - # configs, The ``scope`` attribute of corresponding variable - # will be converted to the `_scope_`. - Config._parse_scope(cfg_dict) - except Exception as e: - if osp.exists(temp_config_dir): - shutil.rmtree(temp_config_dir) - raise e - - # check deprecation information - if DEPRECATION_KEY in cfg_dict: - deprecation_info = cfg_dict.pop(DEPRECATION_KEY) - warning_msg = f'The config file {filename} will be deprecated ' \ - 'in the future.' - if 'expected' in deprecation_info: - warning_msg += f' Please use {deprecation_info["expected"]} ' \ - 'instead.' - if 'reference' in deprecation_info: - warning_msg += ' More information can be found at ' \ - f'{deprecation_info["reference"]}' - warnings.warn(warning_msg, DeprecationWarning) - - cfg_text = filename + '\n' - with open(filename, encoding='utf-8') as f: - # Setting encoding explicitly to resolve coding issue on windows - cfg_text += f.read() - - # Substitute base variables from strings to their actual values - cfg_dict = Config._substitute_base_vars(cfg_dict, base_var_dict, - base_cfg_dict) - cfg_dict.pop(BASE_KEY, None) - - cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict) - cfg_dict = { - k: v - for k, v in cfg_dict.items() if not k.startswith('__') - } - - # merge cfg_text - cfg_text_list.append(cfg_text) - cfg_text = '\n'.join(cfg_text_list) - - return cfg_dict, cfg_text, env_variables - - @staticmethod - def _parse_lazy_import(filename: str) -> Tuple[ConfigDict, set]: - """Transform file to variables dictionary. - - Args: - filename (str): Name of config file. - - Returns: - Tuple[dict, dict]: ``cfg_dict`` and ``imported_names``. - - - cfg_dict (dict): Variables dictionary of parsed config. - - imported_names (set): Used to mark the names of - imported object. - """ - # In lazy import mode, users can use the Python syntax `import` to - # implement inheritance between configuration files, which is easier - # for users to understand the hierarchical relationships between - # different configuration files. - - # Besides, users can also using `import` syntax to import corresponding - # module which will be filled in the `type` field. It means users - # can directly navigate to the source of the module in the - # configuration file by clicking the `type` field. - - # To avoid really importing the third party package like `torch` - # during import `type` object, we use `_parse_lazy_import` to parse the - # configuration file, which will not actually trigger the import - # process, but simply parse the imported `type`s as LazyObject objects. - - # The overall pipeline of _parse_lazy_import is: - # 1. Parse the base module from the config file. - # || - # \/ - # base_module = ['mmdet.configs.default_runtime'] - # || - # \/ - # 2. recursively parse the base module and gather imported objects to - # a dict. - # || - # \/ - # The base_dict will be: - # { - # 'mmdet.configs.default_runtime': {...} - # 'mmdet.configs.retinanet_r50_fpn_1x_coco': {...} - # ... - # }, each item in base_dict is a dict of `LazyObject` - # 3. parse the current config file filling the imported variable - # with the base_dict. - # - # 4. During the parsing process, all imported variable will be - # recorded in the `imported_names` set. These variables can be - # accessed, but will not be dumped by default. - - with open(filename, encoding='utf-8') as f: - global_dict = {'LazyObject': LazyObject, '__file__': filename} - base_dict = {} - - parsed_codes = ast.parse(f.read()) - # get the names of base modules, and remove the - # `with read_base():'` statement - base_modules = Config._get_base_modules(parsed_codes.body) - base_imported_names = set() - for base_module in base_modules: - # If base_module means a relative import, assuming the level is - # 2, which means the module is imported like - # "from ..a.b import c". we must ensure that c is an - # object `defined` in module b, and module b should not be a - # package including `__init__` file but a single python file. - level = len(re.match(r'\.*', base_module).group()) - if level > 0: - # Relative import - base_dir = osp.dirname(filename) - module_path = osp.join( - base_dir, *(['..'] * (level - 1)), - f'{base_module[level:].replace(".", "/")}.py') - else: - # Absolute import - module_list = base_module.split('.') - if len(module_list) == 1: - raise ConfigParsingError( - 'The imported configuration file should not be ' - f'an independent package {module_list[0]}. Here ' - 'is an example: ' - '`with read_base(): from mmdet.configs.retinanet_r50_fpn_1x_coco import *`' # noqa: E501 - ) - else: - package = module_list[0] - root_path = get_installed_path(package) - module_path = f'{osp.join(root_path, *module_list[1:])}.py' # noqa: E501 - if not osp.isfile(module_path): - raise ConfigParsingError( - f'{module_path} not found! It means that incorrect ' - 'module is defined in ' - f'`with read_base(): = from {base_module} import ...`, please ' # noqa: E501 - 'make sure the base config module is valid ' - 'and is consistent with the prior import ' - 'logic') - _base_cfg_dict, _base_imported_names = Config._parse_lazy_import( # noqa: E501 - module_path) - base_imported_names |= _base_imported_names - # The base_dict will be: - # { - # 'mmdet.configs.default_runtime': {...} - # 'mmdet.configs.retinanet_r50_fpn_1x_coco': {...} - # ... - # } - base_dict[base_module] = _base_cfg_dict - - # `base_dict` contains all the imported modules from `base_cfg`. - # In order to collect the specific imported module from `base_cfg` - # before parse the current file, we using AST Transform to - # transverse the imported module from base_cfg and merge then into - # the global dict. After the ast transformation, most of import - # syntax will be removed (except for the builtin import) and - # replaced with the `LazyObject` - transform = ImportTransformer( - global_dict=global_dict, - base_dict=base_dict, - filename=filename) - modified_code = transform.visit(parsed_codes) - modified_code, abs_imported = _gather_abs_import_lazyobj( - modified_code, filename=filename) - imported_names = transform.imported_obj | abs_imported - imported_names |= base_imported_names - modified_code = ast.fix_missing_locations(modified_code) - exec( - compile(modified_code, filename, mode='exec'), global_dict, - global_dict) - - ret: dict = {} - for key, value in global_dict.items(): - if key.startswith('__') or key in ['LazyObject']: - continue - ret[key] = value - # convert dict to ConfigDict - cfg_dict = Config._dict_to_config_dict_lazy(ret) - - return cfg_dict, imported_names - - @staticmethod - def _dict_to_config_dict_lazy(cfg: dict): - """Recursively converts ``dict`` to :obj:`ConfigDict`. The only - difference between ``_dict_to_config_dict_lazy`` and - ``_dict_to_config_dict_lazy`` is that the former one does not consider - the scope, and will not trigger the building of ``LazyObject``. - - Args: - cfg (dict): Config dict. - - Returns: - ConfigDict: Converted dict. - """ - # Only the outer dict with key `type` should have the key `_scope_`. - if isinstance(cfg, dict): - cfg_dict = ConfigDict() - for key, value in cfg.items(): - cfg_dict[key] = Config._dict_to_config_dict_lazy(value) - return cfg_dict - if isinstance(cfg, (tuple, list)): - return type(cfg)( - Config._dict_to_config_dict_lazy(_cfg) for _cfg in cfg) - return cfg - - @staticmethod - def _dict_to_config_dict(cfg: dict, - scope: Optional[str] = None, - has_scope=True): - """Recursively converts ``dict`` to :obj:`ConfigDict`. - - Args: - cfg (dict): Config dict. - scope (str, optional): Scope of instance. - has_scope (bool): Whether to add `_scope_` key to config dict. - - Returns: - ConfigDict: Converted dict. - """ - # Only the outer dict with key `type` should have the key `_scope_`. - if isinstance(cfg, dict): - if has_scope and 'type' in cfg: - has_scope = False - if scope is not None and cfg.get('_scope_', None) is None: - cfg._scope_ = scope # type: ignore - cfg = ConfigDict(cfg) - dict.__setattr__(cfg, 'scope', scope) - for key, value in cfg.items(): - cfg[key] = Config._dict_to_config_dict( - value, scope=scope, has_scope=has_scope) - elif isinstance(cfg, tuple): - cfg = tuple( - Config._dict_to_config_dict(_cfg, scope, has_scope=has_scope) - for _cfg in cfg) - elif isinstance(cfg, list): - cfg = [ - Config._dict_to_config_dict(_cfg, scope, has_scope=has_scope) - for _cfg in cfg - ] - return cfg - - @staticmethod - def _parse_scope(cfg: dict) -> None: - """Adds ``_scope_`` to :obj:`ConfigDict` instance, which means a base - variable. - - If the config dict already has the scope, scope will not be - overwritten. - - Args: - cfg (dict): Config needs to be parsed with scope. - """ - if isinstance(cfg, ConfigDict): - cfg._scope_ = cfg.scope - elif isinstance(cfg, (tuple, list)): - [Config._parse_scope(value) for value in cfg] - else: - return - - @staticmethod - def _get_base_files(filename: str) -> list: - """Get the base config file. - - Args: - filename (str): The config file. - - Raises: - TypeError: Name of config file. - - Returns: - list: A list of base config. - """ - file_format = osp.splitext(filename)[1] - if file_format == '.py': - Config._validate_py_syntax(filename) - with open(filename, encoding='utf-8') as f: - parsed_codes = ast.parse(f.read()).body - - def is_base_line(c): - return (isinstance(c, ast.Assign) - and isinstance(c.targets[0], ast.Name) - and c.targets[0].id == BASE_KEY) - - base_code = next((c for c in parsed_codes if is_base_line(c)), - None) - if base_code is not None: - base_code = ast.Expression( # type: ignore - body=base_code.value) # type: ignore - base_files = eval(compile(base_code, '', mode='eval')) - else: - base_files = [] - elif file_format in ('.yml', '.yaml', '.json'): - import mmengine - cfg_dict = mmengine.load(filename) - base_files = cfg_dict.get(BASE_KEY, []) - else: - raise TypeError('The config type should be py, json, yaml or ' - f'yml, but got {file_format}') - base_files = base_files if isinstance(base_files, - list) else [base_files] - return base_files - - @staticmethod - def _get_cfg_path(cfg_path: str, - filename: str) -> Tuple[str, Optional[str]]: - """Get the config path from the current or external package. - - Args: - cfg_path (str): Relative path of config. - filename (str): The config file being parsed. - - Returns: - Tuple[str, str or None]: Path and scope of config. If the config - is not an external config, the scope will be `None`. - """ - if '::' in cfg_path: - # `cfg_path` startswith '::' means an external config path. - # Get package name and relative config path. - scope = cfg_path.partition('::')[0] - package, cfg_path = _get_package_and_cfg_path(cfg_path) - - if not is_installed(package): - raise ModuleNotFoundError( - f'{package} is not installed, please install {package} ' - f'manually') - - # Get installed package path. - package_path = get_installed_path(package) - try: - # Get config path from meta file. - cfg_path = _get_external_cfg_path(package_path, cfg_path) - except ValueError: - # Since base config does not have a metafile, it should be - # concatenated with package path and relative config path. - cfg_path = _get_external_cfg_base_path(package_path, cfg_path) - except FileNotFoundError as e: - raise e - return cfg_path, scope - else: - # Get local config path. - cfg_dir = osp.dirname(filename) - cfg_path = osp.join(cfg_dir, cfg_path) - return cfg_path, None - @staticmethod def _merge_a_into_b(a: dict, b: dict, @@ -1325,19 +534,14 @@ def auto_argparser(description=None): return parser, cfg @property + @abstractmethod def filename(self) -> str: """get file name of config.""" - return self._filename @property + @abstractmethod def text(self) -> str: """get config text.""" - return self._text - - @property - def env_variables(self) -> dict: - """get used environment variables.""" - return self._env_variables @property def pretty_text(self) -> str: @@ -1478,13 +682,9 @@ def __setitem__(self, name, value): def __iter__(self): return iter(self._cfg_dict) - def __getstate__( - self - ) -> Tuple[dict, Optional[str], Optional[str], dict, bool, set]: - state = (self._cfg_dict, self._filename, self._text, - self._env_variables, self._format_python_code, - self._imported_names) - return state + @abstractmethod + def __getstate__(self) -> Tuple: + pass def __deepcopy__(self, memo): cls = self.__class__ @@ -1506,14 +706,9 @@ def __copy__(self): copy = __copy__ - def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], - dict, bool, set]): - super().__setattr__('_cfg_dict', state[0]) - super().__setattr__('_filename', state[1]) - super().__setattr__('_text', state[2]) - super().__setattr__('_env_variables', state[3]) - super().__setattr__('_format_python_code', state[4]) - super().__setattr__('_imported_names', state[5]) + @abstractmethod + def __setstate__(self, state: Tuple): + pass def dump(self, file: Optional[Union[str, Path]] = None): """Dump config to file or return config text. @@ -1628,36 +823,9 @@ def _is_lazy_import(filename: str) -> bool: return True return False - def _to_lazy_dict(self, keep_imported: bool = False) -> dict: - """Convert config object to dictionary with lazy object, and filter the - imported object.""" - res = self._cfg_dict._to_lazy_dict() - if hasattr(self, '_imported_names') and not keep_imported: - res = { - key: value - for key, value in res.items() - if key not in self._imported_names - } - return res - - def to_dict(self, keep_imported: bool = False): - """Convert all data in the config to a builtin ``dict``. - - Args: - keep_imported (bool): Whether to keep the imported field. - Defaults to False - - If you import third-party objects in the config file, all imported - objects will be converted to a string like ``torch.optim.SGD`` - """ - cfg_dict = self._cfg_dict.to_dict() - if hasattr(self, '_imported_names') and not keep_imported: - cfg_dict = { - key: value - for key, value in cfg_dict.items() - if key not in self._imported_names - } - return cfg_dict + def to_dict(self): + """Convert all data in the config to a builtin ``dict``.""" + return self._cfg_dict.to_dict() class DictAction(Action): @@ -1773,19 +941,3 @@ def __call__(self, key, val = kv.split('=', maxsplit=1) options[key] = self._parse_iterable(val) setattr(namespace, self.dest, options) - - -@contextmanager -def read_base(): - """Context manager to mark the base config. - - The pure Python-style configuration file allows you to use the import - syntax. However, it is important to note that you need to import the base - configuration file within the context of ``read_base``, and import other - dependencies outside of it. - - You can see more usage of Python-style configuration in the `tutorial`_ - - .. _tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html#a-pure-python-style-configuration-file-beta - """ # noqa: E501 - yield diff --git a/mmengine/config/lazy.py b/mmengine/config/lazy.py index 3bb86aaddb..0331c4e5e9 100644 --- a/mmengine/config/lazy.py +++ b/mmengine/config/lazy.py @@ -103,9 +103,6 @@ def __setstate__(self, state): self.__dict__ = state -LazyAttr = LazyObject - - class LazyImportContext: def __init__(self, enable=True): diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py index 3f36c6fe11..e73387e105 100644 --- a/mmengine/config/new_config.py +++ b/mmengine/config/new_config.py @@ -1,5 +1,4 @@ # Copyright (c) OpenMMLab. All rights reserved. -import copy import importlib import inspect import platform @@ -9,15 +8,13 @@ from importlib.util import spec_from_loader from pathlib import Path from types import BuiltinFunctionType, FunctionType, ModuleType -from typing import Any, Optional, Tuple, Union +from typing import Optional, Tuple, Union from yapf.yapflib.yapf_api import FormatCode -from mmengine.fileio import dump -from .config import ConfigDict +from .config import Config, ConfigDict from .lazy import LazyImportContext, LazyObject -DELETE_KEY = '_delete_' RESERVED_KEYS = ['filename', 'text', 'pretty_text'] if platform.system() == 'Windows': @@ -51,7 +48,35 @@ def recover_lazy_field(cfg): return cfg -class Config: +def dump_extra_type(value): + if isinstance(value, LazyObject): + return value.dump_str + if isinstance(value, (type, FunctionType, BuiltinFunctionType)): + return '<' + value.__module__ + '.' + value.__name__ + '>' + if isinstance(value, ModuleType): + return f'<{value.__name__}>' + + typename = type(value).__module__ + type(value).__name__ + if typename == 'torch.dtype': + return '<' + str(value) + '>' + + return None + + +def filter_imports(item): + k, v = item + # If the name is the same as the function/type name, + # It should come from import instead of a field + if isinstance(v, (FunctionType, type)): + return v.__name__ != k + elif isinstance(v, LazyObject): + return v.name != k + elif isinstance(v, ModuleType): + return False + return True + + +class ConfigV2(Config): """A facility for config and config files. It supports common file formats as configs: python/json/yaml. @@ -108,13 +133,13 @@ def __init__(self, for key in cfg_dict: if key in RESERVED_KEYS: raise KeyError(f'{key} is reserved for config file') - self._sanity_check(cfg_dict) if not isinstance(cfg_dict, ConfigDict): cfg_dict = ConfigDict(cfg_dict) - super().__setattr__('_cfg_dict', cfg_dict) - super().__setattr__('_filename', filename) - super().__setattr__('_format_python_code', format_python_code) + super(Config, self).__setattr__('_cfg_dict', cfg_dict) + super(Config, self).__setattr__('_filename', filename) + super(Config, self).__setattr__('_format_python_code', + format_python_code) if cfg_text: text = cfg_text elif filename: @@ -122,18 +147,21 @@ def __init__(self, text = f.read() else: text = '' - super().__setattr__('_text', text) + + super(Config, self).__setattr__('_text', text) + + self._sanity_check(self._to_lazy_dict()) @staticmethod def _sanity_check(cfg): if isinstance(cfg, dict): for v in cfg.values(): - Config._sanity_check(v) + ConfigV2._sanity_check(v) elif isinstance(cfg, (tuple, list, set)): for v in cfg: - Config._sanity_check(v) + ConfigV2._sanity_check(v) elif isinstance(cfg, (type, FunctionType)): - if (Config._parent_pkg in cfg.__module__ + if (ConfigV2._parent_pkg in cfg.__module__ or '__main__' in cfg.__module__): msg = ('You cannot use temporary functions ' 'as the value of a field.\n\n') @@ -142,7 +170,8 @@ def _sanity_check(cfg): @staticmethod def fromfile(filename: Union[str, Path], - lazy_import: Optional[bool] = None) -> 'Config': + keep_imported: bool = False, + format_python_code: bool = True) -> 'ConfigV2': """Build a Config instance from config file. Args: @@ -162,51 +191,35 @@ def fromfile(filename: Union[str, Path], # Using try-except to make sure ``ConfigDict.lazy`` will be reset # to False. See more details about lazy in the docstring of # ConfigDict - ConfigDict.lazy = lazy_import + ConfigDict.lazy = True try: - cfg_dict = Config._parse_lazy_import(filename) + module = ConfigV2._get_config_module(filename) + module_dict = { + k: getattr(module, k) + for k in dir(module) if not k.startswith('__') + } + if not keep_imported: + module_dict = dict(filter(filter_imports, module_dict.items())) + + cfg_dict = ConfigDict(module_dict) + # Recover dumped lazy object like '' from string + cfg_dict = recover_lazy_field(cfg_dict) + + cfg = ConfigV2( + cfg_dict, + filename=filename, + format_python_code=format_python_code) finally: ConfigDict.lazy = False - for key, value in list(cfg_dict.to_dict().items()): - # Remove functions or modules - if isinstance(value, (LazyObject, ModuleType, FunctionType, type)): - cfg_dict.pop(key) - - # Recover dumped lazy object like '' from string - cfg_dict = recover_lazy_field(cfg_dict) - - cfg = Config(cfg_dict, filename=filename) return cfg - @staticmethod - def _parse_lazy_import(filename: Union[str, Path]) -> ConfigDict: - """Transform file to variables dictionary. - - Args: - filename (str): Name of config file. - - Returns: - Tuple[dict, dict]: ``cfg_dict`` and ``imported_names``. - - - cfg_dict (dict): Variables dictionary of parsed config. - - imported_names (set): Used to mark the names of - imported object. - """ - module = Config._get_config_module(filename) - module_dict = { - k: getattr(module, k) - for k in dir(module) if not k.startswith('__') - } - - return ConfigDict(module_dict) - @staticmethod def _get_config_module(filename: Union[str, Path], level=0): file = Path(filename).absolute() module_name = re.sub(r'\W|^(?=\d)', '_', file.stem) - parent_pkg = Config._parent_pkg + str(level) - fullname = '.'.join([parent_pkg] * Config._max_parent_depth + + parent_pkg = ConfigV2._parent_pkg + str(level) + fullname = '.'.join([parent_pkg] * ConfigV2._max_parent_depth + [module_name]) # import config file as a module @@ -234,73 +247,13 @@ def _dict_to_config_dict_lazy(cfg: dict): if isinstance(cfg, dict): cfg_dict = ConfigDict() for key, value in cfg.items(): - cfg_dict[key] = Config._dict_to_config_dict_lazy(value) + cfg_dict[key] = ConfigV2._dict_to_config_dict_lazy(value) return cfg_dict if isinstance(cfg, (tuple, list)): return type(cfg)( - Config._dict_to_config_dict_lazy(_cfg) for _cfg in cfg) + ConfigV2._dict_to_config_dict_lazy(_cfg) for _cfg in cfg) return cfg - @staticmethod - def _merge_a_into_b(a: dict, - b: dict, - allow_list_keys: bool = False) -> dict: - """merge dict ``a`` into dict ``b`` (non-inplace). - - Values in ``a`` will overwrite ``b``. ``b`` is copied first to avoid - in-place modifications. - - Args: - a (dict): The source dict to be merged into ``b``. - b (dict): The origin dict to be fetch keys from ``a``. - allow_list_keys (bool): If True, int string keys (e.g. '0', '1') - are allowed in source ``a`` and will replace the element of the - corresponding index in b if b is a list. Defaults to False. - - Returns: - dict: The modified dict of ``b`` using ``a``. - - Examples: - # Normally merge a into b. - >>> Config._merge_a_into_b( - ... dict(obj=dict(a=2)), dict(obj=dict(a=1))) - {'obj': {'a': 2}} - - # Delete b first and merge a into b. - >>> Config._merge_a_into_b( - ... dict(obj=dict(_delete_=True, a=2)), dict(obj=dict(a=1))) - {'obj': {'a': 2}} - - # b is a list - >>> Config._merge_a_into_b( - ... {'0': dict(a=2)}, [dict(a=1), dict(b=2)], True) - [{'a': 2}, {'b': 2}] - """ - b = b.copy() - for k, v in a.items(): - if allow_list_keys and k.isdigit() and isinstance(b, list): - k = int(k) - if len(b) <= k: - raise KeyError(f'Index {k} exceeds the length of list {b}') - b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) - elif isinstance(v, dict): - if k in b and not v.pop(DELETE_KEY, False): - allowed_types: Union[Tuple, type] = ( - dict, list) if allow_list_keys else dict - if not isinstance(b[k], allowed_types): - raise TypeError( - f'{k}={v} in child config cannot inherit from ' - f'base because {k} is a dict in the child config ' - f'but is of type {type(b[k])} in base config. ' - f'You may set `{DELETE_KEY}=True` to ignore the ' - f'base config.') - b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) - else: - b[k] = ConfigDict(v) - else: - b[k] = v - return b - @property def filename(self) -> str: """get file name of config.""" @@ -356,21 +309,12 @@ def _format_basic_types(input_): return _format_dict(input_) elif isinstance(input_, (list, set, tuple)): return _format_list_tuple_set(input_) - elif isinstance(input_, LazyObject): - return repr(input_.dump_str) - elif isinstance(input_, (type, FunctionType, BuiltinFunctionType)): - if Config._parent_pkg in input_.__module__: - # defined in the config file. - module = input_.__module__.rpartition('.')[-1] - else: - module = input_.__module__ - return repr('<' + module + '.' + input_.__name__ + '>') - elif isinstance(input_, ModuleType): - return repr(f'<{input_.__name__}>') - elif 'torch.dtype' in str(type(input_)): - return repr('<' + str(input_) + '>') else: - return str(input_) + dump_str = dump_extra_type(input_) + if dump_str is not None: + return repr(dump_str) + else: + return str(input_) cfg_dict = self._to_lazy_dict() @@ -394,150 +338,24 @@ def _format_basic_types(input_): return text - def __repr__(self): - return f'Config (path: {self.filename}): {self._cfg_dict.__repr__()}' - - def __len__(self): - return len(self._cfg_dict) - - def __getattr__(self, name: str) -> Any: - return getattr(self._cfg_dict, name) - - def __getitem__(self, name): - return self._cfg_dict.__getitem__(name) - - def __setattr__(self, name, value): - self._sanity_check(value) - if isinstance(value, dict): - value = ConfigDict(value) - self._cfg_dict.__setattr__(name, value) - - def __setitem__(self, name, value): - self._sanity_check(value) - if isinstance(value, dict): - value = ConfigDict(value) - self._cfg_dict.__setitem__(name, value) - - def __iter__(self): - return iter(self._cfg_dict) - - def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str], dict]: + def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str]]: return (self._cfg_dict, self._filename, self._text) - def __deepcopy__(self, memo): - cls = self.__class__ - other = cls.__new__(cls) - memo[id(self)] = other - - for key, value in self.__dict__.items(): - super(Config, other).__setattr__(key, copy.deepcopy(value, memo)) - - return other - - def __copy__(self): - cls = self.__class__ - other = cls.__new__(cls) - other.__dict__.update(self.__dict__) - - return other - - def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], - dict]): + def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str]]): _cfg_dict, _filename, _text = state - super().__setattr__('_cfg_dict', _cfg_dict) - super().__setattr__('_filename', _filename) - super().__setattr__('_text', _text) - - def dump(self, file: Optional[Union[str, Path]] = None): - """Dump config to file or return config text. - - Args: - file (str or Path, optional): If not specified, then the object - is dumped to a str, otherwise to a file specified by the filename. - Defaults to None. - - Returns: - str or None: Config text. - """ - file = str(file) if isinstance(file, Path) else file - cfg_dict = super().__getattribute__('_cfg_dict').to_dict() - if file is None: - if self.filename is None or self.filename.endswith('.py'): - return self.pretty_text - else: - file_format = self.filename.split('.')[-1] - return dump(cfg_dict, file_format=file_format) - elif file.endswith('.py'): - with open(file, 'w', encoding='utf-8') as f: - f.write(self.pretty_text) - else: - file_format = file.split('.')[-1] - return dump(cfg_dict, file=file, file_format=file_format) - - def merge_from_dict(self, - options: dict, - allow_list_keys: bool = True) -> None: - """Merge list into cfg_dict. - - Merge the dict parsed by MultipleKVAction into this cfg. - - Args: - options (dict): dict of configs to merge from. - allow_list_keys (bool): If True, int string keys (e.g. '0', '1') - are allowed in ``options`` and will replace the element of the - corresponding index in the config if the config is a list. - Defaults to True. - - Examples: - >>> from mmengine import Config - >>> # Merge dictionary element - >>> options = {'model.backbone.depth': 50, 'model.backbone.with_cp': True} - >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet')))) - >>> cfg.merge_from_dict(options) - >>> cfg._cfg_dict - {'model': {'backbone': {'type': 'ResNet', 'depth': 50, 'with_cp': True}}} - >>> # Merge list element - >>> cfg = Config( - >>> dict(pipeline=[dict(type='LoadImage'), - >>> dict(type='LoadAnnotations')])) - >>> options = dict(pipeline={'0': dict(type='SelfLoadImage')}) - >>> cfg.merge_from_dict(options, allow_list_keys=True) - >>> cfg._cfg_dict - {'pipeline': [{'type': 'SelfLoadImage'}, {'type': 'LoadAnnotations'}]} - """ # noqa: E501 - option_cfg_dict: dict = {} - for full_key, v in options.items(): - d = option_cfg_dict - key_list = full_key.split('.') - for subkey in key_list[:-1]: - d.setdefault(subkey, ConfigDict()) - d = d[subkey] - subkey = key_list[-1] - d[subkey] = v - - cfg_dict = super().__getattribute__('_cfg_dict') - super().__setattr__( - '_cfg_dict', - Config._merge_a_into_b( - option_cfg_dict, cfg_dict, allow_list_keys=allow_list_keys)) + super(Config, self).__setattr__('_cfg_dict', _cfg_dict) + super(Config, self).__setattr__('_filename', _filename) + super(Config, self).__setattr__('_text', _text) def _to_lazy_dict(self, keep_imported: bool = False) -> dict: """Convert config object to dictionary and filter the imported object.""" - res = self._cfg_dict.to_dict() - - def filter_item(item): - _, v = item - if isinstance(v, (LazyObject, ModuleType, FunctionType, type)): - return False - if v is read_base: - return False - return True + res = self._cfg_dict._to_lazy_dict() if keep_imported: return res else: - return dict(filter(filter_item, res.items())) + return dict(filter(filter_imports, res.items())) def to_dict(self, keep_imported: bool = False): """Convert all data in the config to a builtin ``dict``. @@ -558,10 +376,9 @@ def lazy2string(cfg_dict): for k, v in cfg_dict.items()}) elif isinstance(cfg_dict, (tuple, list)): return type(cfg_dict)(lazy2string(v) for v in cfg_dict) - elif isinstance(cfg_dict, LazyObject): - return str(cfg_dict) else: - return cfg_dict + dump_str = dump_extra_type(cfg_dict) + return dump_str if dump_str is not None else cfg_dict return lazy2string(_cfg_dict) @@ -574,11 +391,11 @@ def __init__(self, filepath, level) -> None: def create_module(self, spec): file = self.filepath - return Config._get_config_module(file, level=self.level) + return ConfigV2._get_config_module(file, level=self.level) def exec_module(self, module): for k in dir(module): - module.__dict__[k] = Config._dict_to_config_dict_lazy( + module.__dict__[k] = ConfigV2._dict_to_config_dict_lazy( getattr(module, k)) @@ -600,7 +417,7 @@ def find_spec(self, fullname, path=None, target=None): The search is based on sys.path_hooks and sys.path_importer_cache. """ - parent_pkg = Config._parent_pkg + str(self.level) + parent_pkg = ConfigV2._parent_pkg + str(self.level) names = fullname.split('.') if names[-1] == parent_pkg: @@ -652,7 +469,8 @@ def __enter__(self): stack = inspect.stack()[1] file = inspect.getfile(stack[0]) folder = Path(file).parent - self.root_path = folder.joinpath(*(['..'] * Config._max_parent_depth)) + self.root_path = folder.joinpath(*(['..'] * + ConfigV2._max_parent_depth)) self.base_modules = [] self.level = len( diff --git a/mmengine/config/old_config.py b/mmengine/config/old_config.py new file mode 100644 index 0000000000..9a568794d8 --- /dev/null +++ b/mmengine/config/old_config.py @@ -0,0 +1,679 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import ast +import copy +import os +import os.path as osp +import platform +import shutil +import sys +import tempfile +import types +import uuid +import warnings +from pathlib import Path +from typing import Any, Optional, Tuple, Union + +from mmengine.fileio import load +from mmengine.logging import print_log +from mmengine.utils import (check_file_exist, get_installed_path, + import_modules_from_strings, is_installed) +from .config import BASE_KEY, Config, ConfigDict +from .utils import (ConfigParsingError, RemoveAssignFromAST, + _get_external_cfg_base_path, _get_external_cfg_path, + _get_package_and_cfg_path) + +DEPRECATION_KEY = '_deprecation_' +RESERVED_KEYS = ['filename', 'text', 'pretty_text', 'env_variables'] + +if platform.system() == 'Windows': + import regex as re +else: + import re # type: ignore + + +class ConfigV1(Config): + """A facility for config and config files. + + It supports common file formats as configs: python/json/yaml. + ``ConfigV1.fromfile`` can parse a dictionary from a config file, then + build a ``ConfigV1`` instance with the dictionary. + The interface is the same as a dict object and also allows access config + values as attributes. + + Args: + cfg_dict (dict, optional): A config dictionary. Defaults to None. + cfg_text (str, optional): Text of config. Defaults to None. + filename (str or Path, optional): Name of config file. + Defaults to None. + format_python_code (bool): Whether to format Python code by yapf. + Defaults to True. + + Here is a simple example: + + Examples: + >>> cfg = ConfigV1(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = ConfigV1.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/username/projects/mmengine/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/username/projects/mmengine/tests/data/config/a.py] + :" + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + + You can find more advance usage in the `config tutorial`_. + + .. _config tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html + """ # noqa: E501 + + def __init__(self, + cfg_dict: dict = None, + cfg_text: Optional[str] = None, + filename: Optional[Union[str, Path]] = None, + env_variables: Optional[dict] = None, + format_python_code: bool = True): + filename = str(filename) if isinstance(filename, Path) else filename + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError('cfg_dict must be a dict, but ' + f'got {type(cfg_dict)}') + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f'{key} is reserved for config file') + + if not isinstance(cfg_dict, ConfigDict): + cfg_dict = ConfigDict(cfg_dict) + super(Config, self).__setattr__('_cfg_dict', cfg_dict) + super(Config, self).__setattr__('_filename', filename) + super(Config, self).__setattr__('_format_python_code', + format_python_code) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, encoding='utf-8') as f: + text = f.read() + else: + text = '' + super(Config, self).__setattr__('_text', text) + if env_variables is None: + env_variables = dict() + super(Config, self).__setattr__('_env_variables', env_variables) + + @staticmethod + def fromfile(filename: Union[str, Path], + use_predefined_variables: bool = True, + import_custom_modules: bool = True, + use_environment_variables: bool = True, + format_python_code: bool = True) -> 'ConfigV1': + """Build a Config instance from config file. + + Args: + filename (str or Path): Name of config file. + use_predefined_variables (bool, optional): Whether to use + predefined variables. Defaults to True. + import_custom_modules (bool, optional): Whether to support + importing custom modules in config. Defaults to None. + format_python_code (bool): Whether to format Python code by yapf. + Defaults to True. + + Returns: + ConfigV1: Config instance built from config file. + """ + filename = str(filename) if isinstance(filename, Path) else filename + cfg_dict, cfg_text, env_variables = ConfigV1._file2dict( + filename, use_predefined_variables, use_environment_variables) + if import_custom_modules and cfg_dict.get('custom_imports', None): + try: + import_modules_from_strings(**cfg_dict['custom_imports']) + except ImportError as e: + err_msg = ( + 'Failed to import custom modules from ' + f"{cfg_dict['custom_imports']}, the current sys.path " + 'is: ') + for p in sys.path: + err_msg += f'\n {p}' + err_msg += ('\nYou should set `PYTHONPATH` to make `sys.path` ' + 'include the directory which contains your custom ' + 'module') + raise ImportError(err_msg) from e + return ConfigV1( + cfg_dict, + cfg_text=cfg_text, + filename=filename, + env_variables=env_variables, + format_python_code=format_python_code) + + @staticmethod + def _validate_py_syntax(filename: str): + """Validate syntax of python config. + + Args: + filename (str): Filename of python config file. + """ + with open(filename, encoding='utf-8') as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError('There are syntax errors in config ' + f'file {filename}: {e}') + + @staticmethod + def _substitute_predefined_vars(filename: str, temp_config_name: str): + """Substitute predefined variables in config with actual values. + + Sometimes we want some variables in the config to be related to the + current path or file name, etc. + + Here is an example of a typical usage scenario. When training a model, + we define a working directory in the config that save the models and + logs. For different configs, we expect to define different working + directories. A common way for users is to use the config file name + directly as part of the working directory name, e.g. for the config + ``config_setting1.py``, the working directory is + ``. /work_dir/config_setting1``. + + This can be easily achieved using predefined variables, which can be + written in the config `config_setting1.py` as follows + + .. code-block:: python + + work_dir = '. /work_dir/{{ fileBasenameNoExtension }}' + + + Here `{{ fileBasenameNoExtension }}` indicates the file name of the + config (without the extension), and when the config class reads the + config file, it will automatically parse this double-bracketed string + to the corresponding actual value. + + .. code-block:: python + + cfg = ConfigV1.fromfile('. /config_setting1.py') + cfg.work_dir # ". /work_dir/config_setting1" + + + For details, Please refer to docs/zh_cn/advanced_tutorials/config.md . + + Args: + filename (str): Filename of config. + temp_config_name (str): Temporary filename to save substituted + config. + """ + file_dirname = osp.dirname(filename) + file_basename = osp.basename(filename) + file_basename_no_extension = osp.splitext(file_basename)[0] + file_extname = osp.splitext(filename)[1] + support_templates = dict( + fileDirname=file_dirname, + fileBasename=file_basename, + fileBasenameNoExtension=file_basename_no_extension, + fileExtname=file_extname) + with open(filename, encoding='utf-8') as f: + config_file = f.read() + for key, value in support_templates.items(): + regexp = r'\{\{\s*' + str(key) + r'\s*\}\}' + value = value.replace('\\', '/') + config_file = re.sub(regexp, value, config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + + @staticmethod + def _substitute_env_variables(filename: str, temp_config_name: str): + """Substitute environment variables in config with actual values. + + Sometimes, we want to change some items in the config with environment + variables. For examples, we expect to change dataset root by setting + ``DATASET_ROOT=/dataset/root/path`` in the command line. This can be + easily achieved by writing lines in the config as follows + + .. code-block:: python + + data_root = '{{$DATASET_ROOT:/default/dataset}}/images' + + + Here, ``{{$DATASET_ROOT:/default/dataset}}`` indicates using the + environment variable ``DATASET_ROOT`` to replace the part between + ``{{}}``. If the ``DATASET_ROOT`` is not set, the default value + ``/default/dataset`` will be used. + + Environment variables not only can replace items in the string, they + can also substitute other types of data in config. In this situation, + we can write the config as below + + .. code-block:: python + + model = dict( + bbox_head = dict(num_classes={{'$NUM_CLASSES:80'}})) + + + For details, Please refer to docs/zh_cn/tutorials/config.md . + + Args: + filename (str): Filename of config. + temp_config_name (str): Temporary filename to save substituted + config. + """ + with open(filename, encoding='utf-8') as f: + config_file = f.read() + regexp = r'\{\{[\'\"]?\s*\$(\w+)\s*\:\s*(\S*?)\s*[\'\"]?\}\}' + keys = re.findall(regexp, config_file) + env_variables = dict() + for var_name, value in keys: + regexp = r'\{\{[\'\"]?\s*\$' + var_name + r'\s*\:\s*' \ + + value + r'\s*[\'\"]?\}\}' + if var_name in os.environ: + value = os.environ[var_name] + env_variables[var_name] = value + print_log( + f'Using env variable `{var_name}` with value of ' + f'{value} to replace item in config.', + logger='current') + if not value: + raise KeyError(f'`{var_name}` cannot be found in `os.environ`.' + f' Please set `{var_name}` in environment or ' + 'give a default value.') + config_file = re.sub(regexp, value, config_file) + + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + return env_variables + + @staticmethod + def _pre_substitute_base_vars(filename: str, + temp_config_name: str) -> dict: + """Preceding step for substituting variables in base config with actual + value. + + Args: + filename (str): Filename of config. + temp_config_name (str): Temporary filename to save substituted + config. + + Returns: + dict: A dictionary contains variables in base config. + """ + with open(filename, encoding='utf-8') as f: + config_file = f.read() + base_var_dict = {} + regexp = r'\{\{\s*' + BASE_KEY + r'\.([\w\.]+)\s*\}\}' + base_vars = set(re.findall(regexp, config_file)) + for base_var in base_vars: + randstr = f'_{base_var}_{uuid.uuid4().hex.lower()[:6]}' + base_var_dict[randstr] = base_var + regexp = r'\{\{\s*' + BASE_KEY + r'\.' + base_var + r'\s*\}\}' + config_file = re.sub(regexp, f'"{randstr}"', config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + return base_var_dict + + @staticmethod + def _substitute_base_vars(cfg: Any, base_var_dict: dict, + base_cfg: dict) -> Any: + """Substitute base variables from strings to their actual values. + + Args: + Any : Config dictionary. + base_var_dict (dict): A dictionary contains variables in base + config. + base_cfg (dict): Base config dictionary. + + Returns: + Any : A dictionary with origin base variables + substituted with actual values. + """ + cfg = copy.deepcopy(cfg) + + if isinstance(cfg, dict): + for k, v in cfg.items(): + if isinstance(v, str) and v in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[v].split('.'): + new_v = new_v[new_k] + cfg[k] = new_v + elif isinstance(v, (list, tuple, dict)): + cfg[k] = ConfigV1._substitute_base_vars( + v, base_var_dict, base_cfg) + elif isinstance(cfg, tuple): + cfg = tuple( + ConfigV1._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg) + elif isinstance(cfg, list): + cfg = [ + ConfigV1._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg + ] + elif isinstance(cfg, str) and cfg in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[cfg].split('.'): + new_v = new_v[new_k] + cfg = new_v + + return cfg + + @staticmethod + def _file2dict( + filename: str, + use_predefined_variables: bool = True, + use_environment_variables: bool = True) -> Tuple[dict, str, dict]: + """Transform file to variables dictionary. + + Args: + filename (str): Name of config file. + use_predefined_variables (bool, optional): Whether to use + predefined variables. Defaults to True. + + Returns: + Tuple[dict, str]: Variables dictionary and text of config. + """ + filename = osp.abspath(osp.expanduser(filename)) + lazy_import = Config._is_lazy_import(filename) + if lazy_import: + raise ConfigParsingError( + 'The configuration file type in the inheritance chain ' + 'must match the current configuration file type, either ' + '"lazy_import" or non-"lazy_import". You got this error ' + 'since you use the syntax like `_base_ = ..."` ' # noqa: E501 + 'in your config. You should use `with read_base(): ... to` ' # noqa: E501 + 'mark the inherited config file. See more information ' + 'in https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html' # noqa: E501 + ) + + check_file_exist(filename) + fileExtname = osp.splitext(filename)[1] + if fileExtname not in ['.py', '.json', '.yaml', '.yml']: + raise OSError('Only py/yml/yaml/json type are supported now!') + try: + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile( + dir=temp_config_dir, suffix=fileExtname, delete=False) + if platform.system() == 'Windows': + temp_config_file.close() + + # Substitute predefined variables + if use_predefined_variables: + ConfigV1._substitute_predefined_vars( + filename, temp_config_file.name) + else: + shutil.copyfile(filename, temp_config_file.name) + # Substitute environment variables + env_variables = dict() + if use_environment_variables: + env_variables = ConfigV1._substitute_env_variables( + temp_config_file.name, temp_config_file.name) + # Substitute base variables from placeholders to strings + base_var_dict = ConfigV1._pre_substitute_base_vars( + temp_config_file.name, temp_config_file.name) + + # Handle base files + base_cfg_dict = ConfigDict() + cfg_text_list = list() + for base_cfg_path in ConfigV1._get_base_files( + temp_config_file.name): + base_cfg_path, scope = ConfigV1._get_cfg_path( + base_cfg_path, filename) + _cfg_dict, _cfg_text, _env_variables = ConfigV1._file2dict( + filename=base_cfg_path, + use_predefined_variables=use_predefined_variables, + use_environment_variables=use_environment_variables) + cfg_text_list.append(_cfg_text) + env_variables.update(_env_variables) + duplicate_keys = base_cfg_dict.keys() & _cfg_dict.keys() + if len(duplicate_keys) > 0: + raise KeyError( + 'Duplicate key is not allowed among bases. ' + f'Duplicate keys: {duplicate_keys}') + + # _dict_to_config_dict will do the following things: + # 1. Recursively converts ``dict`` to :obj:`ConfigDict`. + # 2. Set `_scope_` for the outer dict variable for the base + # config. + # 3. Set `scope` attribute for each base variable. + # Different from `_scope_`， `scope` is not a key of base + # dict, `scope` attribute will be parsed to key `_scope_` + # by function `_parse_scope` only if the base variable is + # accessed by the current config. + _cfg_dict = ConfigV1._dict_to_config_dict(_cfg_dict, scope) + base_cfg_dict.update(_cfg_dict) + + if filename.endswith('.py'): + with open(temp_config_file.name, encoding='utf-8') as f: + parsed_codes = ast.parse(f.read()) + parsed_codes = RemoveAssignFromAST(BASE_KEY).visit( + parsed_codes) + codeobj = compile(parsed_codes, '', mode='exec') + # Support load global variable in nested function of the + # config. + global_locals_var = {BASE_KEY: base_cfg_dict} + ori_keys = set(global_locals_var.keys()) + eval(codeobj, global_locals_var, global_locals_var) + cfg_dict = { + key: value + for key, value in global_locals_var.items() + if (key not in ori_keys and not key.startswith('__')) + } + elif filename.endswith(('.yml', '.yaml', '.json')): + cfg_dict = load(temp_config_file.name) + # close temp file + for key, value in list(cfg_dict.items()): + if isinstance(value, + (types.FunctionType, types.ModuleType)): + cfg_dict.pop(key) + temp_config_file.close() + + # If the current config accesses a base variable of base + # configs, The ``scope`` attribute of corresponding variable + # will be converted to the `_scope_`. + ConfigV1._parse_scope(cfg_dict) + except Exception as e: + if osp.exists(temp_config_dir): + shutil.rmtree(temp_config_dir) + raise e + + # check deprecation information + if DEPRECATION_KEY in cfg_dict: + deprecation_info = cfg_dict.pop(DEPRECATION_KEY) + warning_msg = f'The config file {filename} will be deprecated ' \ + 'in the future.' + if 'expected' in deprecation_info: + warning_msg += f' Please use {deprecation_info["expected"]} ' \ + 'instead.' + if 'reference' in deprecation_info: + warning_msg += ' More information can be found at ' \ + f'{deprecation_info["reference"]}' + warnings.warn(warning_msg, DeprecationWarning) + + cfg_text = filename + '\n' + with open(filename, encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + cfg_text += f.read() + + # Substitute base variables from strings to their actual values + cfg_dict = ConfigV1._substitute_base_vars(cfg_dict, base_var_dict, + base_cfg_dict) + cfg_dict.pop(BASE_KEY, None) + + cfg_dict = ConfigV1._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = { + k: v + for k, v in cfg_dict.items() if not k.startswith('__') + } + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = '\n'.join(cfg_text_list) + + return cfg_dict, cfg_text, env_variables + + @staticmethod + def _dict_to_config_dict(cfg: dict, + scope: Optional[str] = None, + has_scope=True): + """Recursively converts ``dict`` to :obj:`ConfigDict`. + + Args: + cfg (dict): Config dict. + scope (str, optional): Scope of instance. + has_scope (bool): Whether to add `_scope_` key to config dict. + + Returns: + ConfigDict: Converted dict. + """ + # Only the outer dict with key `type` should have the key `_scope_`. + if isinstance(cfg, dict): + if has_scope and 'type' in cfg: + has_scope = False + if scope is not None and cfg.get('_scope_', None) is None: + cfg._scope_ = scope # type: ignore + cfg = ConfigDict(cfg) + dict.__setattr__(cfg, 'scope', scope) + for key, value in cfg.items(): + cfg[key] = ConfigV1._dict_to_config_dict( + value, scope=scope, has_scope=has_scope) + elif isinstance(cfg, tuple): + cfg = tuple( + ConfigV1._dict_to_config_dict( + _cfg, scope, has_scope=has_scope) for _cfg in cfg) + elif isinstance(cfg, list): + cfg = [ + ConfigV1._dict_to_config_dict( + _cfg, scope, has_scope=has_scope) for _cfg in cfg + ] + return cfg + + @staticmethod + def _parse_scope(cfg: dict) -> None: + """Adds ``_scope_`` to :obj:`ConfigDict` instance, which means a base + variable. + + If the config dict already has the scope, scope will not be + overwritten. + + Args: + cfg (dict): Config needs to be parsed with scope. + """ + if isinstance(cfg, ConfigDict): + cfg._scope_ = cfg.scope + elif isinstance(cfg, (tuple, list)): + [ConfigV1._parse_scope(value) for value in cfg] + else: + return + + @staticmethod + def _get_base_files(filename: str) -> list: + """Get the base config file. + + Args: + filename (str): The config file. + + Raises: + TypeError: Name of config file. + + Returns: + list: A list of base config. + """ + file_format = osp.splitext(filename)[1] + if file_format == '.py': + ConfigV1._validate_py_syntax(filename) + with open(filename, encoding='utf-8') as f: + parsed_codes = ast.parse(f.read()).body + + def is_base_line(c): + return (isinstance(c, ast.Assign) + and isinstance(c.targets[0], ast.Name) + and c.targets[0].id == BASE_KEY) + + base_code = next((c for c in parsed_codes if is_base_line(c)), + None) + if base_code is not None: + base_code = ast.Expression( # type: ignore + body=base_code.value) # type: ignore + base_files = eval(compile(base_code, '', mode='eval')) + else: + base_files = [] + elif file_format in ('.yml', '.yaml', '.json'): + import mmengine + cfg_dict = mmengine.load(filename) + base_files = cfg_dict.get(BASE_KEY, []) + else: + raise TypeError('The config type should be py, json, yaml or ' + f'yml, but got {file_format}') + base_files = base_files if isinstance(base_files, + list) else [base_files] + return base_files + + @staticmethod + def _get_cfg_path(cfg_path: str, + filename: str) -> Tuple[str, Optional[str]]: + """Get the config path from the current or external package. + + Args: + cfg_path (str): Relative path of config. + filename (str): The config file being parsed. + + Returns: + Tuple[str, str or None]: Path and scope of config. If the config + is not an external config, the scope will be `None`. + """ + if '::' in cfg_path: + # `cfg_path` startswith '::' means an external config path. + # Get package name and relative config path. + scope = cfg_path.partition('::')[0] + package, cfg_path = _get_package_and_cfg_path(cfg_path) + + if not is_installed(package): + raise ModuleNotFoundError( + f'{package} is not installed, please install {package} ' + f'manually') + + # Get installed package path. + package_path = get_installed_path(package) + try: + # Get config path from meta file. + cfg_path = _get_external_cfg_path(package_path, cfg_path) + except ValueError: + # Since base config does not have a metafile, it should be + # concatenated with package path and relative config path. + cfg_path = _get_external_cfg_base_path(package_path, cfg_path) + except FileNotFoundError as e: + raise e + return cfg_path, scope + else: + # Get local config path. + cfg_dir = osp.dirname(filename) + cfg_path = osp.join(cfg_dir, cfg_path) + return cfg_path, None + + @property + def filename(self) -> str: + """get file name of config.""" + return self._filename + + @property + def text(self) -> str: + """get config text.""" + return self._text + + @property + def env_variables(self) -> dict: + """get used environment variables.""" + return self._env_variables + + def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str], dict]: + return (self._cfg_dict, self._filename, self._text, + self._env_variables) + + def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], + dict]): + _cfg_dict, _filename, _text, _env_variables = state + super(Config, self).__setattr__('_cfg_dict', _cfg_dict) + super(Config, self).__setattr__('_filename', _filename) + super(Config, self).__setattr__('_text', _text) + super(Config, self).__setattr__('_text', _env_variables) diff --git a/mmengine/runner/runner.py b/mmengine/runner/runner.py index 1811f59165..12830cf4ad 100644 --- a/mmengine/runner/runner.py +++ b/mmengine/runner/runner.py @@ -277,9 +277,8 @@ def __init__( # recursively copy the `cfg` because `self.cfg` will be modified # everywhere. - from mmengine.config.new_config import Config as NewConfig if cfg is not None: - if isinstance(cfg, (Config, NewConfig)): + if isinstance(cfg, Config): self.cfg = copy.deepcopy(cfg) elif isinstance(cfg, dict): self.cfg = Config(cfg) diff --git a/mmengine/visualization/vis_backend.py b/mmengine/visualization/vis_backend.py index fa75f7e1b0..a1d896ea5b 100644 --- a/mmengine/visualization/vis_backend.py +++ b/mmengine/visualization/vis_backend.py @@ -14,7 +14,6 @@ import torch from mmengine.config import Config -from mmengine.config.new_config import Config as NewConfig from mmengine.fileio import dump from mmengine.hooks.logger_hook import SUFFIX_TYPE from mmengine.logging import MMLogger, print_log @@ -236,7 +235,7 @@ def add_config(self, config: Config, **kwargs) -> None: Args: config (Config): The Config object """ - assert isinstance(config, (Config, NewConfig)) + assert isinstance(config, Config) config.dump(self._config_save_file) @force_init_env diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index f12895110a..697af422c9 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -17,6 +17,7 @@ import mmengine from mmengine import Config, ConfigDict, DictAction from mmengine.config.lazy import LazyObject +from mmengine.config.old_config import ConfigV1 from mmengine.fileio import dump, load from mmengine.registry import MODELS, DefaultScope, Registry from mmengine.utils import is_installed @@ -203,7 +204,7 @@ def test_auto_argparser(self): def test_dict_to_config_dict(self): cfg_dict = dict( a=1, b=dict(c=dict()), d=[dict(e=dict(f=(dict(g=1), [])))]) - cfg_dict = Config._dict_to_config_dict(cfg_dict) + cfg_dict = ConfigV1._dict_to_config_dict(cfg_dict) assert isinstance(cfg_dict, ConfigDict) assert isinstance(cfg_dict.a, int) assert isinstance(cfg_dict.b, ConfigDict) @@ -381,7 +382,7 @@ def test_substitute_predefined_vars(self, tmp_path): expected_text = expected_text.replace('\\', '/') with open(cfg, 'w') as f: f.write(cfg_text) - Config._substitute_predefined_vars(cfg, substituted_cfg) + ConfigV1._substitute_predefined_vars(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == expected_text @@ -394,10 +395,10 @@ def test_substitute_environment_vars(self, tmp_path): with open(cfg, 'w') as f: f.write(cfg_text) with pytest.raises(KeyError): - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) os.environ['A'] = 'text_A' - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'a=text_A\n' os.environ.pop('A') @@ -405,12 +406,12 @@ def test_substitute_environment_vars(self, tmp_path): cfg_text = 'b={{$B:80}}\n' with open(cfg, 'w') as f: f.write(cfg_text) - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'b=80\n' os.environ['B'] = '100' - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'b=100\n' os.environ.pop('B') @@ -418,7 +419,7 @@ def test_substitute_environment_vars(self, tmp_path): cfg_text = 'c={{"$C:80"}}\n' with open(cfg, 'w') as f: f.write(cfg_text) - Config._substitute_env_variables(cfg, substituted_cfg) + ConfigV1._substitute_env_variables(cfg, substituted_cfg) with open(substituted_cfg) as f: assert f.read() == 'c=80\n' @@ -426,7 +427,7 @@ def test_pre_substitute_base_vars(self, tmp_path): cfg_path = osp.join(self.data_path, 'config', 'py_config/test_pre_substitute_base_vars.py') tmp_cfg = tmp_path / 'tmp_cfg.py' - base_var_dict = Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + base_var_dict = ConfigV1._pre_substitute_base_vars(cfg_path, tmp_cfg) assert 'item6' in base_var_dict.values() assert 'item10' in base_var_dict.values() assert 'item11' in base_var_dict.values() @@ -440,7 +441,7 @@ def test_pre_substitute_base_vars(self, tmp_path): cfg_path = osp.join(self.data_path, 'config', 'json_config/test_base.json') tmp_cfg = tmp_path / 'tmp_cfg.json' - Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + ConfigV1._pre_substitute_base_vars(cfg_path, tmp_cfg) cfg_module_dict = load(tmp_cfg) assert cfg_module_dict['item9'].startswith('_item2') assert cfg_module_dict['item10'].startswith('_item7') @@ -448,7 +449,7 @@ def test_pre_substitute_base_vars(self, tmp_path): cfg_path = osp.join(self.data_path, 'config', 'yaml_config/test_base.yaml') tmp_cfg = tmp_path / 'tmp_cfg.yaml' - Config._pre_substitute_base_vars(cfg_path, tmp_cfg) + ConfigV1._pre_substitute_base_vars(cfg_path, tmp_cfg) cfg_module_dict = load(tmp_cfg) assert cfg_module_dict['item9'].startswith('_item2') assert cfg_module_dict['item10'].startswith('_item7') @@ -464,7 +465,7 @@ def test_substitute_base_vars(self): '_item2_.fswf': 'item2', '_item0_.12ed21wq': 'item0' } - cfg = Config._substitute_base_vars(cfg, base_var_dict, cfg_base) + cfg = ConfigV1._substitute_base_vars(cfg, base_var_dict, cfg_base) assert cfg['item4'] == cfg_base['item1'] assert cfg['item5']['item2'] == cfg_base['item2'] @@ -496,7 +497,7 @@ def test_get_cfg_path_local(self): filename = 'py_config/simple_config.py' filename = osp.join(self.data_path, 'config', filename) cfg_name = './base.py' - cfg_path, scope = Config._get_cfg_path(cfg_name, filename) + cfg_path, scope = ConfigV1._get_cfg_path(cfg_name, filename) assert scope is None osp.isfile(cfg_path) @@ -508,12 +509,12 @@ def test_get_cfg_path_external(self): filename = osp.join(self.data_path, 'config', filename) cfg_name = 'mmdet::faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py' - cfg_path, scope = Config._get_cfg_path(cfg_name, filename) + cfg_path, scope = ConfigV1._get_cfg_path(cfg_name, filename) assert scope == 'mmdet' osp.isfile(cfg_path) cfg_name = 'mmcls::cspnet/cspresnet50_8xb32_in1k.py' - cfg_path, scope = Config._get_cfg_path(cfg_name, filename) + cfg_path, scope = ConfigV1._get_cfg_path(cfg_name, filename) assert scope == 'mmcls' osp.isfile(cfg_path) @@ -524,7 +525,8 @@ def _simple_load(self): filename = f'{file_format}_config/{name}.{file_format}' cfg_file = osp.join(self.data_path, 'config', filename) - cfg_dict, cfg_text, env_variables = Config._file2dict(cfg_file) + cfg_dict, cfg_text, env_variables = ConfigV1._file2dict( + cfg_file) assert isinstance(cfg_text, str) assert isinstance(cfg_dict, dict) assert isinstance(env_variables, dict) @@ -547,9 +549,12 @@ def _predefined_vars(self): item2=path, item3='abc_test_predefined_var') - assert Config._file2dict(cfg_file)[0]['item1'] == cfg_dict_dst['item1'] - assert Config._file2dict(cfg_file)[0]['item2'] == cfg_dict_dst['item2'] - assert Config._file2dict(cfg_file)[0]['item3'] == cfg_dict_dst['item3'] + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == cfg_dict_dst['item1'] + assert ConfigV1._file2dict( + cfg_file)[0]['item2'] == cfg_dict_dst['item2'] + assert ConfigV1._file2dict( + cfg_file)[0]['item3'] == cfg_dict_dst['item3'] # test `use_predefined_variable=False` cfg_dict_ori = dict( @@ -557,28 +562,31 @@ def _predefined_vars(self): item2='{{ fileDirname}}', item3='abc_{{ fileBasenameNoExtension }}') - assert Config._file2dict(cfg_file, - False)[0]['item1'] == cfg_dict_ori['item1'] - assert Config._file2dict(cfg_file, - False)[0]['item2'] == cfg_dict_ori['item2'] - assert Config._file2dict(cfg_file, - False)[0]['item3'] == cfg_dict_ori['item3'] + assert ConfigV1._file2dict(cfg_file, + False)[0]['item1'] == cfg_dict_ori['item1'] + assert ConfigV1._file2dict(cfg_file, + False)[0]['item2'] == cfg_dict_ori['item2'] + assert ConfigV1._file2dict(cfg_file, + False)[0]['item3'] == cfg_dict_ori['item3'] # test test_predefined_var.yaml cfg_file = osp.join(self.data_path, 'config/yaml_config/test_predefined_var.yaml') # test `use_predefined_variable=False` - assert Config._file2dict(cfg_file, - False)[0]['item1'] == '{{ fileDirname }}' - assert Config._file2dict(cfg_file)[0]['item1'] == self._get_file_path( - osp.dirname(cfg_file)) + assert ConfigV1._file2dict(cfg_file, + False)[0]['item1'] == '{{ fileDirname }}' + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == self._get_file_path( + osp.dirname(cfg_file)) # test test_predefined_var.json cfg_file = osp.join(self.data_path, 'config/json_config/test_predefined_var.json') - assert Config.fromfile(cfg_file, False)['item1'] == '{{ fileDirname }}' + assert Config.fromfile( + cfg_file, + use_predefined_variables=False)['item1'] == '{{ fileDirname }}' assert Config.fromfile(cfg_file)['item1'] == self._get_file_path( osp.dirname(cfg_file)) @@ -588,20 +596,26 @@ def _environment_vars(self): 'config/py_config/test_environment_var.py') with pytest.raises(KeyError): - Config._file2dict(cfg_file) + ConfigV1._file2dict(cfg_file) os.environ['ITEM1'] = '60' cfg_dict_dst = dict(item1='60', item2='default_value', item3=80) - assert Config._file2dict(cfg_file)[0]['item1'] == cfg_dict_dst['item1'] - assert Config._file2dict(cfg_file)[0]['item2'] == cfg_dict_dst['item2'] - assert Config._file2dict(cfg_file)[0]['item3'] == cfg_dict_dst['item3'] + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == cfg_dict_dst['item1'] + assert ConfigV1._file2dict( + cfg_file)[0]['item2'] == cfg_dict_dst['item2'] + assert ConfigV1._file2dict( + cfg_file)[0]['item3'] == cfg_dict_dst['item3'] os.environ['ITEM2'] = 'new_value' os.environ['ITEM3'] = '50' cfg_dict_dst = dict(item1='60', item2='new_value', item3=50) - assert Config._file2dict(cfg_file)[0]['item1'] == cfg_dict_dst['item1'] - assert Config._file2dict(cfg_file)[0]['item2'] == cfg_dict_dst['item2'] - assert Config._file2dict(cfg_file)[0]['item3'] == cfg_dict_dst['item3'] + assert ConfigV1._file2dict( + cfg_file)[0]['item1'] == cfg_dict_dst['item1'] + assert ConfigV1._file2dict( + cfg_file)[0]['item2'] == cfg_dict_dst['item2'] + assert ConfigV1._file2dict( + cfg_file)[0]['item3'] == cfg_dict_dst['item3'] os.environ.pop('ITEM1') os.environ.pop('ITEM2') @@ -610,7 +624,7 @@ def _environment_vars(self): def _merge_from_base(self): cfg_file = osp.join(self.data_path, 'config/py_config/test_merge_from_base_single.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item1'] == [2, 3] assert cfg_dict['item2']['a'] == 1 @@ -626,7 +640,7 @@ def _merge_from_multiple_bases(self): cfg_file = osp.join( self.data_path, 'config/py_config/test_merge_from_multiple_bases.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] # cfg.fcfg_dictd assert cfg_dict['item1'] == [1, 2] @@ -649,7 +663,7 @@ def _base_variables(self): 'json_config/test_base.json', 'yaml_config/test_base.yaml' ]: cfg_file = osp.join(self.data_path, 'config', file) - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item1'] == [1, 2] assert cfg_dict['item2']['a'] == 0 @@ -670,7 +684,7 @@ def _base_variables(self): 'yaml_config/test_base_variables_nested.yaml' ]: cfg_file = osp.join(self.data_path, 'config', file) - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['base'] == '_base_.item8' assert cfg_dict['item1'] == [1, 2] @@ -705,7 +719,7 @@ def _base_variables(self): cfg_file = osp.join( self.data_path, 'config/py_config/test_pre_substitute_base_vars.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item21'] == 'test_base_variables.py' assert cfg_dict['item22'] == 'test_base_variables.py' @@ -779,7 +793,7 @@ def _base_variables(self): # Test use global variable in config function cfg_file = osp.join(self.data_path, 'config/py_config/test_py_function_global_var.py') - cfg = Config._file2dict(cfg_file)[0] + cfg = ConfigV1._file2dict(cfg_file)[0] assert cfg['item1'] == 1 assert cfg['item2'] == 2 @@ -787,7 +801,7 @@ def _base_variables(self): # config. cfg_file = osp.join(self.data_path, 'config/py_config/test_py_modify_key.py') - cfg = Config._file2dict(cfg_file)[0] + cfg = ConfigV1._file2dict(cfg_file)[0] assert cfg == dict(item1=dict(a=1)) # Simulate the case that the temporary directory includes `.`, etc. @@ -802,13 +816,13 @@ def __init__(self, *args, prefix='test.', **kwargs): PatchedTempDirectory): cfg_file = osp.join(self.data_path, 'config/py_config/test_py_modify_key.py') - cfg = Config._file2dict(cfg_file)[0] + cfg = ConfigV1._file2dict(cfg_file)[0] assert cfg == dict(item1=dict(a=1)) def _merge_recursive_bases(self): cfg_file = osp.join(self.data_path, 'config/py_config/test_merge_recursive_bases.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] assert cfg_dict['item1'] == [2, 3] assert cfg_dict['item2']['a'] == 1 @@ -818,7 +832,7 @@ def _merge_recursive_bases(self): def _merge_delete(self): cfg_file = osp.join(self.data_path, 'config/py_config/test_merge_delete.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] # cfg.field assert cfg_dict['item1'] == dict(a=0) assert cfg_dict['item2'] == dict(a=0, b=0) @@ -834,7 +848,7 @@ def _merge_intermediate_variable(self): cfg_file = osp.join( self.data_path, 'config/py_config/test_merge_intermediate_variable_child.py') - cfg_dict = Config._file2dict(cfg_file)[0] + cfg_dict = ConfigV1._file2dict(cfg_file)[0] # cfg.field assert cfg_dict['item1'] == [1, 2] assert cfg_dict['item2'] == dict(a=0) @@ -971,9 +985,9 @@ def test_lazy_import(self, tmp_path): cfg = Config.fromfile(lazy_import_cfg_path) cfg_dict = cfg.to_dict() assert (cfg_dict['train_dataloader']['dataset']['type'] == - 'mmengine.testing.runner_test_case.ToyDataset') + '') assert ( - cfg_dict['custom_hooks'][0]['type'] == 'mmengine.hooks.EMAHook') + cfg_dict['custom_hooks'][0]['type'] == '') # Dumped config dumped_cfg_path = tmp_path / 'test_dump_lazy.py' cfg.dump(dumped_cfg_path) @@ -1033,11 +1047,11 @@ def _compare_dict(a, b): cfg.error_attr error_module = tmp_path / 'error_module.py' - error_module.write_text("""import error_module""") + error_module.write_text("""import error_module;a=error_module""") match = 'Failed to import error_module' with pytest.raises(ImportError, match=match): cfg = Config.fromfile(str(error_module)) - cfg.error_module + cfg.a # lazy-import and non-lazy-import should not be used mixed. # current text config, base lazy-import config From 3e3e3e66f69c83a8a2316666a6eaf16f152ec91f Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Mon, 17 Jul 2023 16:09:27 +0800 Subject: [PATCH 03/10] Use mock import instead of dummy parent package to implement relative import --- mmengine/config/new_config.py | 199 +++++++++++++------------------ mmengine/config/old_config.py | 20 ++-- mmengine/registry/registry.py | 2 +- tests/test_config/test_config.py | 13 +- 4 files changed, 101 insertions(+), 133 deletions(-) diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py index e73387e105..2579734dd8 100644 --- a/mmengine/config/new_config.py +++ b/mmengine/config/new_config.py @@ -1,11 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. +import builtins import importlib import inspect +import os import platform import sys -from importlib.abc import Loader, MetaPathFinder from importlib.machinery import PathFinder -from importlib.util import spec_from_loader from pathlib import Path from types import BuiltinFunctionType, FunctionType, ModuleType from typing import Optional, Tuple, Union @@ -16,6 +16,7 @@ from .lazy import LazyImportContext, LazyObject RESERVED_KEYS = ['filename', 'text', 'pretty_text'] +_CFG_UID = 0 if platform.system() == 'Windows': import regex as re @@ -115,8 +116,7 @@ class ConfigV2(Config): .. _config tutorial: https://mmengine.readthedocs.io/en/latest/advanced_tutorials/config.html """ # noqa: E501 - _max_parent_depth = 4 - _parent_pkg = '_cfg_parent' + _pkg_prefix = '_mmengine_cfg' def __init__(self, cfg_dict: dict = None, @@ -161,7 +161,7 @@ def _sanity_check(cfg): for v in cfg: ConfigV2._sanity_check(v) elif isinstance(cfg, (type, FunctionType)): - if (ConfigV2._parent_pkg in cfg.__module__ + if (ConfigV2._pkg_prefix in cfg.__module__ or '__main__' in cfg.__module__): msg = ('You cannot use temporary functions ' 'as the value of a field.\n\n') @@ -211,22 +211,29 @@ def fromfile(filename: Union[str, Path], format_python_code=format_python_code) finally: ConfigDict.lazy = False + global _CFG_UID + _CFG_UID = 0 + for mod in list(sys.modules): + if mod.startswith(ConfigV2._pkg_prefix): + del sys.modules[mod] return cfg @staticmethod - def _get_config_module(filename: Union[str, Path], level=0): + def _get_config_module(filename: Union[str, Path]): file = Path(filename).absolute() module_name = re.sub(r'\W|^(?=\d)', '_', file.stem) - parent_pkg = ConfigV2._parent_pkg + str(level) - fullname = '.'.join([parent_pkg] * ConfigV2._max_parent_depth + - [module_name]) + global _CFG_UID + # Build a unique module name to avoid conflict. + fullname = f'{ConfigV2._pkg_prefix}{_CFG_UID}_{module_name}' + _CFG_UID += 1 # import config file as a module with LazyImportContext(): spec = importlib.util.spec_from_file_location(fullname, file) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + sys.modules[fullname] = module return module @@ -338,14 +345,16 @@ def _format_basic_types(input_): return text - def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str]]: - return (self._cfg_dict, self._filename, self._text) + def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str], bool]: + return (self._cfg_dict, self._filename, self._text, + self._format_python_code) - def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str]]): - _cfg_dict, _filename, _text = state - super(Config, self).__setattr__('_cfg_dict', _cfg_dict) - super(Config, self).__setattr__('_filename', _filename) - super(Config, self).__setattr__('_text', _text) + def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], + bool]): + super(Config, self).__setattr__('_cfg_dict', state[0]) + super(Config, self).__setattr__('_filename', state[1]) + super(Config, self).__setattr__('_text', state[2]) + super(Config, self).__setattr__('_format_python_code', state[3]) def _to_lazy_dict(self, keep_imported: bool = False) -> dict: """Convert config object to dictionary and filter the imported @@ -383,99 +392,9 @@ def lazy2string(cfg_dict): return lazy2string(_cfg_dict) -class BaseConfigLoader(Loader): - - def __init__(self, filepath, level) -> None: - self.filepath = filepath - self.level = level - - def create_module(self, spec): - file = self.filepath - return ConfigV2._get_config_module(file, level=self.level) - - def exec_module(self, module): - for k in dir(module): - module.__dict__[k] = ConfigV2._dict_to_config_dict_lazy( - getattr(module, k)) - - -class ParentFolderLoader(Loader): - - @staticmethod - def create_module(spec): - return ModuleType(spec.name) - - @staticmethod - def exec_module(module): - pass - - -class BaseImportContext(MetaPathFinder): - - def find_spec(self, fullname, path=None, target=None): - """Try to find a spec for 'fullname' on sys.path or 'path'. - - The search is based on sys.path_hooks and sys.path_importer_cache. - """ - parent_pkg = ConfigV2._parent_pkg + str(self.level) - names = fullname.split('.') - - if names[-1] == parent_pkg: - self.base_modules.append(fullname) - # Create parent package - return spec_from_loader( - fullname, loader=ParentFolderLoader, is_package=True) - elif names[0] == parent_pkg: - self.base_modules.append(fullname) - # relative imported base package - filepath = self.root_path - for name in names: - if name == parent_pkg: - # Use parent to remove `..` at the end of the root path - filepath = filepath.parent - else: - filepath = filepath / name - if filepath.is_dir(): - # If a dir, create a package. - return spec_from_loader( - fullname, loader=ParentFolderLoader, is_package=True) - - pypath = filepath.with_suffix('.py') - - if not pypath.exists(): - raise ImportError(f'Not found base path {filepath.resolve()}') - return importlib.util.spec_from_loader( - fullname, BaseConfigLoader(pypath, self.level + 1)) - else: - # Absolute import - pkg = PathFinder.find_spec(names[0]) - if pkg and pkg.submodule_search_locations: - self.base_modules.append(fullname) - path = Path(pkg.submodule_search_locations[0]) - for name in names[1:]: - path = path / name - if path.is_dir(): - return spec_from_loader( - fullname, loader=ParentFolderLoader, is_package=True) - pypath = path.with_suffix('.py') - if not pypath.exists(): - raise ImportError(f'Not found base path {path.resolve()}') - return importlib.util.spec_from_loader( - fullname, BaseConfigLoader(pypath, self.level + 1)) - return None +class BaseImportContext(): def __enter__(self): - # call from which file - stack = inspect.stack()[1] - file = inspect.getfile(stack[0]) - folder = Path(file).parent - self.root_path = folder.joinpath(*(['..'] * - ConfigV2._max_parent_depth)) - - self.base_modules = [] - self.level = len( - [p for p in sys.meta_path if isinstance(p, BaseImportContext)]) - # Disable enabled lazy loader during parsing base self.lazy_importers = [] for p in sys.meta_path: @@ -483,18 +402,70 @@ def __enter__(self): self.lazy_importers.append(p) p.enable = False - index = sys.meta_path.index(importlib.machinery.FrozenImporter) - sys.meta_path.insert(index + 1, self) + old_import = builtins.__import__ + + def new_import(name, globals=None, locals=None, fromlist=(), level=0): + cur_file = None + + # Try to import the base config source file + if level != 0 and globals is not None: + # For relative import path + if '__file__' in globals: + loc = Path(globals['__file__']).parent + else: + loc = Path(os.getcwd()) + cur_file = self.find_relative_file(loc, name, level - 1) + if not cur_file.exists(): + raise ImportError( + f'Cannot find the base config "{name}" from ' + f'{loc}: {cur_file} does not exist.') + elif level == 0: + # For absolute import path + pkg, _, mod = name.partition('.') + pkg = PathFinder.find_spec(pkg) + if mod and pkg.submodule_search_locations: + loc = Path(pkg.submodule_search_locations[0]) + cur_file = self.find_relative_file(loc, mod) + if not cur_file.exists(): + raise ImportError( + f'Cannot find the base config "{name}": ' + f'{cur_file} does not exist.') + + # Recover the original import during handle the base config file. + builtins.__import__ = old_import + + if cur_file is not None: + mod = ConfigV2._get_config_module(cur_file) + + for k in dir(mod): + mod.__dict__[k] = ConfigV2._dict_to_config_dict_lazy( + getattr(mod, k)) + else: + mod = old_import( + name, globals, locals, fromlist=fromlist, level=level) + + builtins.__import__ = new_import + + return mod + + self.old_import = old_import + builtins.__import__ = new_import def __exit__(self, exc_type, exc_val, exc_tb): - sys.meta_path.remove(self) - for name in self.base_modules: - sys.modules.pop(name, None) + builtins.__import__ = self.old_import for p in self.lazy_importers: p.enable = True - def __repr__(self): - return f'' + @staticmethod + def find_relative_file(loc: Path, relative_import_path, level=0): + if level > 0: + loc = loc.parents[level - 1] + names = relative_import_path.lstrip('.').split('.') + + for name in names: + loc = loc / name + + return loc.with_suffix('.py') read_base = BaseImportContext diff --git a/mmengine/config/old_config.py b/mmengine/config/old_config.py index 9a568794d8..8bdb7d9baa 100644 --- a/mmengine/config/old_config.py +++ b/mmengine/config/old_config.py @@ -666,14 +666,16 @@ def env_variables(self) -> dict: """get used environment variables.""" return self._env_variables - def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str], dict]: - return (self._cfg_dict, self._filename, self._text, - self._env_variables) + def __getstate__( + self) -> Tuple[dict, Optional[str], Optional[str], dict, bool]: + state = (self._cfg_dict, self._filename, self._text, + self._env_variables, self._format_python_code) + return state def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], - dict]): - _cfg_dict, _filename, _text, _env_variables = state - super(Config, self).__setattr__('_cfg_dict', _cfg_dict) - super(Config, self).__setattr__('_filename', _filename) - super(Config, self).__setattr__('_text', _text) - super(Config, self).__setattr__('_text', _env_variables) + dict, bool]): + super(Config, self).__setattr__('_cfg_dict', state[0]) + super(Config, self).__setattr__('_filename', state[1]) + super(Config, self).__setattr__('_text', state[2]) + super(Config, self).__setattr__('_env_variables', state[3]) + super(Config, self).__setattr__('_format_python_code', state[4]) diff --git a/mmengine/registry/registry.py b/mmengine/registry/registry.py index 94588c30a8..a389c23e0c 100644 --- a/mmengine/registry/registry.py +++ b/mmengine/registry/registry.py @@ -10,8 +10,8 @@ from rich.console import Console from rich.table import Table -from mmengine.config.utils import MODULE2PACKAGE from mmengine.config.lazy import LazyObject +from mmengine.config.utils import MODULE2PACKAGE from mmengine.utils import is_seq_of from .default_scope import DefaultScope diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 697af422c9..b8d8318a8c 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -986,8 +986,9 @@ def test_lazy_import(self, tmp_path): cfg_dict = cfg.to_dict() assert (cfg_dict['train_dataloader']['dataset']['type'] == '') - assert ( - cfg_dict['custom_hooks'][0]['type'] == '') + assert (cfg_dict['custom_hooks'][0]['type'] + in ('', + '')) # Dumped config dumped_cfg_path = tmp_path / 'test_dump_lazy.py' cfg.dump(dumped_cfg_path) @@ -1060,12 +1061,6 @@ def _compare_dict(a, b): osp.join(self.data_path, 'config/lazy_module_config/error_mix_using1.py')) - # Force to import in non-lazy-import mode - Config.fromfile( - osp.join(self.data_path, - 'config/lazy_module_config/error_mix_using1.py'), - lazy_import=False) - # current lazy-import config, base text config with pytest.raises(AttributeError, match='item2'): Config.fromfile( @@ -1088,7 +1083,7 @@ def _compare_dict(a, b): dumped_cfg = Config.fromfile(dumped_cfg_path) assert set(dumped_cfg.keys()) == { - 'path', 'name', 'suffix', 'chained', 'existed', 'cfgname' + 'path', 'name', 'suffix', 'chained', 'existed', 'cfgname', 'ex' } assert dumped_cfg.to_dict() == cfg.to_dict() From 5edffa7f2898b494899f4ed1591fa0c161cc5b5b Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Thu, 24 Aug 2023 11:29:19 +0800 Subject: [PATCH 04/10] Merge latest modification --- mmengine/config/config.py | 60 ++++++++++++++++++++++++++++------- mmengine/config/old_config.py | 7 ++-- mmengine/config/utils.py | 2 +- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/mmengine/config/config.py b/mmengine/config/config.py index 438115ad1a..9ef37b1441 100644 --- a/mmengine/config/config.py +++ b/mmengine/config/config.py @@ -1,21 +1,25 @@ # Copyright (c) OpenMMLab. All rights reserved. import ast import copy +import difflib import os +import platform import tempfile import warnings from abc import ABCMeta, abstractmethod from argparse import Action, ArgumentParser, Namespace from collections import OrderedDict, abc -from contextlib import contextmanager from pathlib import Path from typing import Any, Optional, Sequence, Tuple, Union from addict import Dict +from rich.console import Console +from rich.text import Text from yapf.yapflib.yapf_api import FormatCode from mmengine.fileio import dump from mmengine.logging import print_log +from mmengine.utils import digit_version from .lazy import LazyObject from .utils import ConfigParsingError, _is_builtin_module @@ -244,15 +248,18 @@ def _merge_a_into_b(a, b): for key, value in merged.items(): self[key] = value - def __getstate__(self): - state = {} - for key, value in super().items(): - state[key] = value - return state - - def __setstate__(self, state): - for key, value in state.items(): - self[key] = value + def __reduce_ex__(self, proto): + # Override __reduce_ex__ to avoid `self.items` will be + # called by CPython interpreter during pickling. See more details in + # https://github.com/python/cpython/blob/8d61a71f9c81619e34d4a30b625922ebc83c561b/Objects/typeobject.c#L6196 # noqa: E501 + if digit_version(platform.python_version()) < digit_version('3.8'): + return (self.__class__, ({k: v + for k, v in super().items()}, ), None, + None, None) + else: + return (self.__class__, ({k: v + for k, v in super().items()}, ), None, + None, None, None) def __eq__(self, other): if isinstance(other, ConfigDict): @@ -619,7 +626,8 @@ def _format_dict(input_dict, outest_level=False): use_mapping = _contain_invalid_identifier(input_dict) if use_mapping: r += '{' - for idx, (k, v) in enumerate(input_dict.items()): + for idx, (k, v) in enumerate( + sorted(input_dict.items(), key=lambda x: str(x[0]))): is_last = idx >= len(input_dict) - 1 end = '' if outest_level or is_last else ',' if isinstance(v, dict): @@ -783,6 +791,36 @@ def merge_from_dict(self, Config._merge_a_into_b( option_cfg_dict, cfg_dict, allow_list_keys=allow_list_keys)) + @staticmethod + def diff(cfg1: Union[str, 'Config'], cfg2: Union[str, 'Config']) -> str: + if isinstance(cfg1, str): + cfg1 = Config.fromfile(cfg1) + + if isinstance(cfg2, str): + cfg2 = Config.fromfile(cfg2) + + res = difflib.unified_diff( + cfg1.pretty_text.split('\n'), cfg2.pretty_text.split('\n')) + + # Convert into rich format for better visualization + console = Console() + text = Text() + for line in res: + if line.startswith('+'): + color = 'bright_green' + elif line.startswith('-'): + color = 'bright_red' + else: + color = 'bright_white' + _text = Text(line + '\n') + _text.stylize(color) + text.append(_text) + + with console.capture() as capture: + console.print(text) + + return capture.get() + @staticmethod def _is_lazy_import(filename: str) -> bool: if not filename.endswith('.py'): diff --git a/mmengine/config/old_config.py b/mmengine/config/old_config.py index 8bdb7d9baa..b49db96886 100644 --- a/mmengine/config/old_config.py +++ b/mmengine/config/old_config.py @@ -448,7 +448,7 @@ def _file2dict( parsed_codes = ast.parse(f.read()) parsed_codes = RemoveAssignFromAST(BASE_KEY).visit( parsed_codes) - codeobj = compile(parsed_codes, '', mode='exec') + codeobj = compile(parsed_codes, filename, mode='exec') # Support load global variable in nested function of the # config. global_locals_var = {BASE_KEY: base_cfg_dict} @@ -603,8 +603,9 @@ def is_base_line(c): cfg_dict = mmengine.load(filename) base_files = cfg_dict.get(BASE_KEY, []) else: - raise TypeError('The config type should be py, json, yaml or ' - f'yml, but got {file_format}') + raise ConfigParsingError( + 'The config type should be py, json, yaml or ' + f'yml, but got {file_format}') base_files = base_files if isinstance(base_files, list) else [base_files] return base_files diff --git a/mmengine/config/utils.py b/mmengine/config/utils.py index 7f56fd5727..36f07466b2 100644 --- a/mmengine/config/utils.py +++ b/mmengine/config/utils.py @@ -175,7 +175,7 @@ def _is_builtin_module(module_name: str) -> bool: if origin_path is None: return False origin_path = osp.abspath(origin_path) - if ('site-package' in origin_path + if ('site-package' in origin_path or 'dist-package' in origin_path or not origin_path.startswith(PYTHON_ROOT_DIR)): return False else: From 3cdc47b342b0b500a530f7eafc6120712660580a Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Thu, 24 Aug 2023 11:58:13 +0800 Subject: [PATCH 05/10] Recover lazy object during initialization. --- mmengine/config/lazy.py | 20 +++++++++++-- mmengine/config/new_config.py | 44 +++++++++++------------------ mmengine/config/old_config.py | 25 ++++++++++------ mmengine/runner/_flexible_runner.py | 4 +-- mmengine/runner/runner.py | 4 +-- 5 files changed, 55 insertions(+), 42 deletions(-) diff --git a/mmengine/config/lazy.py b/mmengine/config/lazy.py index 0331c4e5e9..fc10dcc49b 100644 --- a/mmengine/config/lazy.py +++ b/mmengine/config/lazy.py @@ -73,8 +73,6 @@ def __str__(self) -> str: return str(self.source) + '.' + self.name return self.name - __repr__ = __str__ - def __repr__(self) -> str: return f"" @@ -136,3 +134,21 @@ def __exit__(self, exc_type, exc_val, exc_tb): def __repr__(self): return f'' + + +def recover_lazy_field(cfg): + + if isinstance(cfg, dict): + for k, v in cfg.items(): + cfg[k] = recover_lazy_field(v) + return cfg + elif isinstance(cfg, (tuple, list)): + container_type = type(cfg) + cfg = list(cfg) + for i, v in enumerate(cfg): + cfg[i] = recover_lazy_field(v) + return container_type(cfg) + elif isinstance(cfg, str): + recover = LazyObject.from_str(cfg) + return recover if recover is not None else cfg + return cfg diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py index 2579734dd8..6c52efc397 100644 --- a/mmengine/config/new_config.py +++ b/mmengine/config/new_config.py @@ -13,7 +13,7 @@ from yapf.yapflib.yapf_api import FormatCode from .config import Config, ConfigDict -from .lazy import LazyImportContext, LazyObject +from .lazy import LazyImportContext, LazyObject, recover_lazy_field RESERVED_KEYS = ['filename', 'text', 'pretty_text'] _CFG_UID = 0 @@ -31,24 +31,6 @@ def format_inpsect(obj): return msg -def recover_lazy_field(cfg): - - if isinstance(cfg, dict): - for k, v in cfg.items(): - cfg[k] = recover_lazy_field(v) - return cfg - elif isinstance(cfg, (tuple, list)): - container_type = type(cfg) - cfg = list(cfg) - for i, v in enumerate(cfg): - cfg[i] = recover_lazy_field(v) - return container_type(cfg) - elif isinstance(cfg, str): - recover = LazyObject.from_str(cfg) - return recover if recover is not None else cfg - return cfg - - def dump_extra_type(value): if isinstance(value, LazyObject): return value.dump_str @@ -136,6 +118,9 @@ def __init__(self, if not isinstance(cfg_dict, ConfigDict): cfg_dict = ConfigDict(cfg_dict) + # Recover dumped lazy object like '' from string + cfg_dict = recover_lazy_field(cfg_dict) + super(Config, self).__setattr__('_cfg_dict', cfg_dict) super(Config, self).__setattr__('_filename', filename) super(Config, self).__setattr__('_format_python_code', @@ -169,9 +154,10 @@ def _sanity_check(cfg): raise ValueError(msg) @staticmethod - def fromfile(filename: Union[str, Path], - keep_imported: bool = False, - format_python_code: bool = True) -> 'ConfigV2': + def fromfile( # type: ignore + filename: Union[str, Path], + keep_imported: bool = False, + format_python_code: bool = True) -> 'ConfigV2': """Build a Config instance from config file. Args: @@ -202,8 +188,6 @@ def fromfile(filename: Union[str, Path], module_dict = dict(filter(filter_imports, module_dict.items())) cfg_dict = ConfigDict(module_dict) - # Recover dumped lazy object like '' from string - cfg_dict = recover_lazy_field(cfg_dict) cfg = ConfigV2( cfg_dict, @@ -232,7 +216,7 @@ def _get_config_module(filename: Union[str, Path]): with LazyImportContext(): spec = importlib.util.spec_from_file_location(fullname, file) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # type: ignore sys.modules[fullname] = module return module @@ -345,12 +329,16 @@ def _format_basic_types(input_): return text - def __getstate__(self) -> Tuple[dict, Optional[str], Optional[str], bool]: + def __getstate__( + self + ) -> Tuple[dict, Optional[str], Optional[str], bool]: # type: ignore return (self._cfg_dict, self._filename, self._text, self._format_python_code) - def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], - bool]): + def __setstate__( # type: ignore + self, + state: Tuple[dict, Optional[str], Optional[str], bool], + ): super(Config, self).__setattr__('_cfg_dict', state[0]) super(Config, self).__setattr__('_filename', state[1]) super(Config, self).__setattr__('_text', state[2]) diff --git a/mmengine/config/old_config.py b/mmengine/config/old_config.py index b49db96886..9c26619228 100644 --- a/mmengine/config/old_config.py +++ b/mmengine/config/old_config.py @@ -18,6 +18,7 @@ from mmengine.utils import (check_file_exist, get_installed_path, import_modules_from_strings, is_installed) from .config import BASE_KEY, Config, ConfigDict +from .lazy import recover_lazy_field from .utils import (ConfigParsingError, RemoveAssignFromAST, _get_external_cfg_base_path, _get_external_cfg_path, _get_package_and_cfg_path) @@ -91,6 +92,9 @@ def __init__(self, if not isinstance(cfg_dict, ConfigDict): cfg_dict = ConfigDict(cfg_dict) + # Recover dumped lazy object like '' from string + cfg_dict = recover_lazy_field(cfg_dict) + super(Config, self).__setattr__('_cfg_dict', cfg_dict) super(Config, self).__setattr__('_filename', filename) super(Config, self).__setattr__('_format_python_code', @@ -108,11 +112,13 @@ def __init__(self, super(Config, self).__setattr__('_env_variables', env_variables) @staticmethod - def fromfile(filename: Union[str, Path], - use_predefined_variables: bool = True, - import_custom_modules: bool = True, - use_environment_variables: bool = True, - format_python_code: bool = True) -> 'ConfigV1': + def fromfile( # type: ignore + filename: Union[str, Path], + use_predefined_variables: bool = True, + import_custom_modules: bool = True, + use_environment_variables: bool = True, + format_python_code: bool = True, + ) -> 'ConfigV1': """Build a Config instance from config file. Args: @@ -668,13 +674,16 @@ def env_variables(self) -> dict: return self._env_variables def __getstate__( - self) -> Tuple[dict, Optional[str], Optional[str], dict, bool]: + self + ) -> Tuple[dict, Optional[str], Optional[str], dict, bool]: # type: ignore state = (self._cfg_dict, self._filename, self._text, self._env_variables, self._format_python_code) return state - def __setstate__(self, state: Tuple[dict, Optional[str], Optional[str], - dict, bool]): + def __setstate__( # type: ignore + self, + state: Tuple[dict, Optional[str], Optional[str], dict, bool], + ): super(Config, self).__setattr__('_cfg_dict', state[0]) super(Config, self).__setattr__('_filename', state[1]) super(Config, self).__setattr__('_text', state[2]) diff --git a/mmengine/runner/_flexible_runner.py b/mmengine/runner/_flexible_runner.py index 714ac611b2..5f71ecb45f 100644 --- a/mmengine/runner/_flexible_runner.py +++ b/mmengine/runner/_flexible_runner.py @@ -284,9 +284,9 @@ def __init__( if isinstance(cfg, Config): self.cfg = copy.deepcopy(cfg) elif isinstance(cfg, dict): - self.cfg = Config(cfg) + self.cfg = Config(cfg) # type: ignore else: - self.cfg = Config(dict()) + self.cfg = Config(dict()) # type: ignore # lazy initialization training_related = [train_dataloader, train_cfg, optim_wrapper] diff --git a/mmengine/runner/runner.py b/mmengine/runner/runner.py index 12830cf4ad..dda4f299cc 100644 --- a/mmengine/runner/runner.py +++ b/mmengine/runner/runner.py @@ -281,9 +281,9 @@ def __init__( if isinstance(cfg, Config): self.cfg = copy.deepcopy(cfg) elif isinstance(cfg, dict): - self.cfg = Config(cfg) + self.cfg = Config(cfg) # type: ignore else: - self.cfg = Config(dict()) + self.cfg = Config(dict()) # type: ignore # lazy initialization training_related = [train_dataloader, train_cfg, optim_wrapper] From b23516ba61d0cf6c6a3b8213d6da49bd9a4c9bab Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Thu, 24 Aug 2023 14:53:40 +0800 Subject: [PATCH 06/10] Fix UT. --- mmengine/config/utils.py | 288 +-------------------------- mmengine/testing/runner_test_case.py | 2 +- tests/test_config/test_lazy.py | 153 +------------- tests/test_registry/test_registry.py | 6 +- 4 files changed, 12 insertions(+), 437 deletions(-) diff --git a/mmengine/config/utils.py b/mmengine/config/utils.py index 36f07466b2..05cc83d194 100644 --- a/mmengine/config/utils.py +++ b/mmengine/config/utils.py @@ -4,9 +4,8 @@ import re import sys import warnings -from collections import defaultdict from importlib.util import find_spec -from typing import List, Optional, Tuple, Union +from typing import Tuple from mmengine.fileio import load from mmengine.utils import check_file_exist @@ -180,288 +179,3 @@ def _is_builtin_module(module_name: str) -> bool: return False else: return True - - -class ImportTransformer(ast.NodeTransformer): - """Convert the import syntax to the assignment of - :class:`mmengine.config.LazyObject` and preload the base variable before - parsing the configuration file. - - Since you are already looking at this part of the code, I believe you must - be interested in the mechanism of the ``lazy_import`` feature of - :class:`Config`. In this docstring, we will dive deeper into its - principles. - - Most of OpenMMLab users maybe bothered with that: - - * In most of popular IDEs, they cannot navigate to the source code in - configuration file - * In most of popular IDEs, they cannot jump to the base file in current - configuration file, which is much painful when the inheritance - relationship is complex. - - In order to solve this problem, we introduce the ``lazy_import`` mode. - - A very intuitive idea for solving this problem is to import the module - corresponding to the "type" field using the ``import`` syntax. Similarly, - we can also ``import`` base file. - - However, this approach has a significant drawback. It requires triggering - the import logic to parse the configuration file, which can be - time-consuming. Additionally, it implies downloading numerous dependencies - solely for the purpose of parsing the configuration file. - However, it's possible that only a portion of the config will actually be - used. For instance, the package used in the ``train_pipeline`` may not - be necessary for an evaluation task. Forcing users to download these - unused packages is not a desirable solution. - - To avoid this problem, we introduce :class:`mmengine.config.LazyObject` and - :class:`mmengine.config.LazyAttr`. Before we proceed with further - explanations, you may refer to the documentation of these two modules to - gain an understanding of their functionalities. - - Actually, one of the functions of ``ImportTransformer`` is to hack the - ``import`` syntax. It will replace the import syntax - (exclude import the base files) with the assignment of ``LazyObject``. - - As for the import syntax of the base file, we cannot lazy import it since - we're eager to merge the fields of current file and base files. Therefore, - another function of the ``ImportTransformer`` is to collaborate with - ``Config._parse_lazy_import`` to parse the base files. - - Args: - global_dict (dict): The global dict of the current configuration file. - If we divide ordinary Python syntax into two parts, namely the - import section and the non-import section (assuming a simple case - with imports at the beginning and the rest of the code following), - the variables generated by the import statements are stored in - global variables for subsequent code use. In this context, - the ``global_dict`` represents the global variables required when - executing the non-import code. ``global_dict`` will be filled - during visiting the parsed code. - base_dict (dict): All variables defined in base files. - - Examples: - >>> from mmengine.config import read_base - >>> - >>> - >>> with read_base(): - >>> from .._base_.default_runtime import * - >>> from .._base_.datasets.coco_detection import dataset - - In this case, the base_dict will be: - - Examples: - >>> base_dict = { - >>> '.._base_.default_runtime': ... - >>> '.._base_.datasets.coco_detection': dataset} - - and `global_dict` will be updated like this: - - Examples: - >>> global_dict.update(base_dict['.._base_.default_runtime']) # `import *` means update all data - >>> global_dict.update(dataset=base_dict['.._base_.datasets.coco_detection']['dataset']) # only update `dataset` - """ # noqa: E501 - - def __init__(self, - global_dict: dict, - base_dict: Optional[dict] = None, - filename: Optional[str] = None): - self.base_dict = base_dict if base_dict is not None else {} - self.global_dict = global_dict - # In Windows, the filename could be like this: - # "C:\\Users\\runneradmin\\AppData\\Local\\" - # Although it has been an raw string, ast.parse will firstly escape - # it as the executed code: - # "C:\Users\runneradmin\AppData\Local\\\" - # As you see, the `\U` will be treated as a part of - # the escape sequence during code parsing, leading to an - # parsing error - # Here we use `encode('unicode_escape').decode()` for double escaping - if isinstance(filename, str): - filename = filename.encode('unicode_escape').decode() - self.filename = filename - self.imported_obj: set = set() - super().__init__() - - def visit_ImportFrom( - self, node: ast.ImportFrom - ) -> Optional[Union[List[ast.Assign], ast.ImportFrom]]: - """Hack the ``from ... import ...`` syntax and update the global_dict. - - Examples: - >>> from mmdet.models import RetinaNet - - Will be parsed as: - - Examples: - >>> RetinaNet = lazyObject('mmdet.models', 'RetinaNet') - - ``global_dict`` will also be updated by ``base_dict`` as the - class docstring says. - - Args: - node (ast.AST): The node of the current import statement. - - Returns: - Optional[List[ast.Assign]]: There three cases: - - * If the node is a statement of importing base files. - None will be returned. - * If the node is a statement of importing a builtin module, - node will be directly returned - * Otherwise, it will return the assignment statements of - ``LazyObject``. - """ - # Built-in modules will not be parsed as LazyObject - module = f'{node.level*"."}{node.module}' - if _is_builtin_module(module): - # Make sure builtin module will be added into `self.imported_obj` - for alias in node.names: - if alias.asname is not None: - self.imported_obj.add(alias.asname) - elif alias.name == '*': - raise ConfigParsingError( - 'Cannot import * from non-base config') - else: - self.imported_obj.add(alias.name) - return node - - if module in self.base_dict: - for alias_node in node.names: - if alias_node.name == '*': - self.global_dict.update(self.base_dict[module]) - return None - if alias_node.asname is not None: - base_key = alias_node.asname - else: - base_key = alias_node.name - self.global_dict[base_key] = self.base_dict[module][ - alias_node.name] - return None - - nodes: List[ast.Assign] = [] - for alias_node in node.names: - # `ast.alias` has lineno attr after Python 3.10, - if hasattr(alias_node, 'lineno'): - lineno = alias_node.lineno - else: - lineno = node.lineno - if alias_node.name == '*': - # TODO: If users import * from a non-config module, it should - # fallback to import the real module and raise a warning to - # remind users the real module will be imported which will slow - # down the parsing speed. - raise ConfigParsingError( - 'Illegal syntax in config! `from xxx import *` is not ' - 'allowed to appear outside the `if base:` statement') - elif alias_node.asname is not None: - # case1: - # from mmengine.dataset import BaseDataset as Dataset -> - # Dataset = LazyObject('mmengine.dataset', 'BaseDataset') - code = f'{alias_node.asname} = LazyObject("{module}", "{alias_node.name}", "{self.filename}, line {lineno}")' # noqa: E501 - self.imported_obj.add(alias_node.asname) - else: - # case2: - # from mmengine.model import BaseModel - # BaseModel = LazyObject('mmengine.model', 'BaseModel') - code = f'{alias_node.name} = LazyObject("{module}", "{alias_node.name}", "{self.filename}, line {lineno}")' # noqa: E501 - self.imported_obj.add(alias_node.name) - try: - nodes.append(ast.parse(code).body[0]) # type: ignore - except Exception as e: - raise ConfigParsingError( - f'Cannot import {alias_node} from {module}' - '1. Cannot import * from 3rd party lib in the config ' - 'file\n' - '2. Please check if the module is a base config which ' - 'should be added to `_base_`\n') from e - return nodes - - def visit_Import(self, node) -> Union[ast.Assign, ast.Import]: - """Work with ``_gather_abs_import_lazyobj`` to hack the ``import ...`` - syntax. - - Examples: - >>> import mmcls.models - >>> import mmcls.datasets - >>> import mmcls - - Will be parsed as: - - Examples: - >>> # import mmcls.models; import mmcls.datasets; import mmcls - >>> mmcls = lazyObject(['mmcls', 'mmcls.datasets', 'mmcls.models']) - - Args: - node (ast.AST): The node of the current import statement. - - Returns: - ast.Assign: If the import statement is ``import ... as ...``, - ast.Assign will be returned, otherwise node will be directly - returned. - """ - # For absolute import like: `import mmdet.configs as configs`. - # It will be parsed as: - # configs = LazyObject('mmdet.configs') - # For absolute import like: - # `import mmdet.configs` - # `import mmdet.configs.default_runtime` - # This will be parsed as - # mmdet = LazyObject(['mmdet.configs.default_runtime', 'mmdet.configs]) - # However, visit_Import cannot gather other import information, so - # `_gather_abs_import_LazyObject` will gather all import information - # from the same module and construct the LazyObject. - alias_list = node.names - assert len(alias_list) == 1, ( - 'Illegal syntax in config! import multiple modules in one line is ' - 'not supported') - # TODO Support multiline import - alias = alias_list[0] - if alias.asname is not None: - self.imported_obj.add(alias.asname) - if _is_builtin_module(alias.name.split('.')[0]): - return node - return ast.parse( # type: ignore - f'{alias.asname} = LazyObject(' - f'"{alias.name}",' - f'location="{self.filename}, line {node.lineno}")').body[0] - return node - - -def _gather_abs_import_lazyobj(tree: ast.Module, - filename: Optional[str] = None): - """Experimental implementation of gathering absolute import information.""" - if isinstance(filename, str): - filename = filename.encode('unicode_escape').decode() - imported = defaultdict(list) - abs_imported = set() - new_body: List[ast.stmt] = [] - # module2node is used to get lineno when Python < 3.10 - module2node: dict = dict() - for node in tree.body: - if isinstance(node, ast.Import): - for alias in node.names: - # Skip converting built-in module to LazyObject - if _is_builtin_module(alias.name): - new_body.append(node) - continue - module = alias.name.split('.')[0] - module2node.setdefault(module, node) - imported[module].append(alias) - continue - new_body.append(node) - - for key, value in imported.items(): - names = [_value.name for _value in value] - if hasattr(value[0], 'lineno'): - lineno = value[0].lineno - else: - lineno = module2node[key].lineno - lazy_module_assign = ast.parse( - f'{key} = LazyObject({names}, location="{filename}, line {lineno}")' # noqa: E501 - ) # noqa: E501 - abs_imported.add(key) - new_body.insert(0, lazy_module_assign.body[0]) - tree.body = new_body - return tree, abs_imported diff --git a/mmengine/testing/runner_test_case.py b/mmengine/testing/runner_test_case.py index f64594acef..1b77b4b948 100644 --- a/mmengine/testing/runner_test_case.py +++ b/mmengine/testing/runner_test_case.py @@ -133,7 +133,7 @@ def setUp(self) -> None: custom_hooks=[], env_cfg=dict(dist_cfg=dict(backend='nccl')), experiment_name='test1') - self.epoch_based_cfg = Config(epoch_based_cfg) + self.epoch_based_cfg = Config(epoch_based_cfg) # type: ignore # prepare iter based cfg. self.iter_based_cfg: Config = copy.deepcopy(self.epoch_based_cfg) diff --git a/tests/test_config/test_lazy.py b/tests/test_config/test_lazy.py index d69822814b..8e30cb9de9 100644 --- a/tests/test_config/test_lazy.py +++ b/tests/test_config/test_lazy.py @@ -1,122 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. -import ast import copy -import os -import os.path as osp from importlib import import_module -from importlib.util import find_spec from unittest import TestCase -import numpy -import numpy.compat -import numpy.linalg as linalg -from rich.progress import Progress - import mmengine -from mmengine.config import Config -from mmengine.config.lazy import LazyAttr, LazyObject -from mmengine.config.utils import ImportTransformer, _gather_abs_import_lazyobj -from mmengine.fileio import LocalBackend, PetrelBackend - - -class TestImportTransformer(TestCase): - - @classmethod - def setUpClass(cls) -> None: - cls.data_dir = osp.join( # type: ignore - osp.dirname(__file__), '..', 'data', 'config', - 'lazy_module_config') - super().setUpClass() - - def test_lazy_module(self): - cfg_path = osp.join(self.data_dir, 'test_ast_transform.py') - with open(cfg_path) as f: - codestr = f.read() - codeobj = ast.parse(codestr) - global_dict = { - 'LazyObject': LazyObject, - } - base_dict = { - '._base_.default_runtime': { - 'default_scope': 'test_config' - }, - '._base_.scheduler': { - 'val_cfg': {} - }, - } - codeobj = ImportTransformer(global_dict, base_dict).visit(codeobj) - codeobj, _ = _gather_abs_import_lazyobj(codeobj) - codeobj = ast.fix_missing_locations(codeobj) - - exec(compile(codeobj, cfg_path, mode='exec'), global_dict, global_dict) - # 1. absolute import - # 1.1 import module as LazyObject - lazy_numpy = global_dict['numpy'] - self.assertIsInstance(lazy_numpy, LazyObject) - - # 1.2 getattr as LazyAttr - self.assertIsInstance(lazy_numpy.linalg, LazyAttr) - self.assertIsInstance(lazy_numpy.compat, LazyAttr) - - # 1.3 Build module from LazyObject. amp and functional can be accessed - imported_numpy = lazy_numpy.build() - self.assertIs(imported_numpy.linalg, linalg) - self.assertIs(imported_numpy.compat, numpy.compat) - - # 1.4.1 Build module from LazyAttr - imported_linalg = lazy_numpy.linalg.build() - imported_compat = lazy_numpy.compat.build() - self.assertIs(imported_compat, numpy.compat) - self.assertIs(imported_linalg, linalg) - - # 1.4.2 build class method from LazyAttr - start = global_dict['start'] - self.assertEqual(start.module, 'rich.progress.Progress') - self.assertEqual(str(start), 'start') - self.assertIs(start.build(), Progress.start) - - # 1.5 import ... as, and build module from LazyObject - lazy_linalg = global_dict['linalg'] - self.assertIsInstance(lazy_linalg, LazyObject) - self.assertIs(lazy_linalg.build(), linalg) - self.assertIsInstance(lazy_linalg.norm, LazyAttr) - self.assertIs(lazy_linalg.norm.build(), linalg.norm) - - # 1.6 import built in module - imported_os = global_dict['os'] - self.assertIs(imported_os, os) - - # 2. Relative import - # 2.1 from ... import ... - lazy_local_backend = global_dict['local'] - self.assertIsInstance(lazy_local_backend, LazyObject) - self.assertIs(lazy_local_backend.build(), LocalBackend) - - # 2.2 from ... import ... as ... - lazy_petrel_backend = global_dict['PetrelBackend'] - self.assertIsInstance(lazy_petrel_backend, LazyObject) - self.assertIs(lazy_petrel_backend.build(), PetrelBackend) - - # 2.3 from ... import builtin module or obj from `mmengine.Config` - self.assertIs(global_dict['find_module'], find_spec) - self.assertIs(global_dict['Config'], Config) - - # 3 test import base config - # 3.1 simple from ... import and from ... import ... as - self.assertEqual(global_dict['scope'], 'test_config') - self.assertDictEqual(global_dict['val_cfg'], {}) - - # 4. Error catching - cfg_path = osp.join(self.data_dir, - 'test_ast_transform_error_catching1.py') - with open(cfg_path) as f: - codestr = f.read() - codeobj = ast.parse(codestr) - global_dict = {'LazyObject': LazyObject} - with self.assertRaisesRegex( - RuntimeError, - r'Illegal syntax in config! `from xxx import \*`'): - codeobj = ImportTransformer(global_dict).visit(codeobj) +from mmengine.config.lazy import LazyObject +from mmengine.fileio import LocalBackend class TestLazyObject(TestCase): @@ -126,14 +15,6 @@ def test_init(self): LazyObject('mmengine.fileio') LazyObject('mmengine.fileio', 'LocalBackend') - # module must be str - with self.assertRaises(TypeError): - LazyObject(1) - - # imported must be a sequence of string or None - with self.assertRaises(TypeError): - LazyObject('mmengine', ['error_type']) - def test_build(self): lazy_mmengine = LazyObject('mmengine') self.assertIs(lazy_mmengine.build(), mmengine) @@ -142,37 +23,19 @@ def test_build(self): self.assertIs(lazy_mmengine_fileio.build(), import_module('mmengine.fileio')) - lazy_local_backend = LazyObject('mmengine.fileio', 'LocalBackend') + lazy_local_backend = LazyObject('LocalBackend', + LazyObject('mmengine.fileio')) self.assertIs(lazy_local_backend.build(), LocalBackend) - # TODO: The commented test is required, we need to test the built - # LazyObject can access the `mmengine.dataset`. We need to clean the - # environment to make sure the `dataset` is not imported before, and - # it is triggered by lazy_mmengine.build(). However, if we simply - # pop the `mmengine.dataset` will lead to other tests failed, of which - # reason is still unknown. We need to figure out the reason and fix it - # in the latter - - # sys.modules.pop('mmengine.config') - # sys.modules.pop('mmengine.fileio') - # sys.modules.pop('mmengine') - # lazy_mmengine = LazyObject(['mmengine', 'mmengine.dataset']) - # self.assertIs(lazy_mmengine.build().dataset, - # import_module('mmengine.config')) copied = copy.deepcopy(lazy_local_backend) self.assertDictEqual(copied.__dict__, lazy_local_backend.__dict__) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): lazy_mmengine() with self.assertRaises(ImportError): LazyObject('unknown').build() - -class TestLazyAttr(TestCase): - # Since LazyAttr should only be built from LazyObect, we only test - # the build method here. - def test_build(self): lazy_mmengine = LazyObject('mmengine') local_backend = lazy_mmengine.fileio.LocalBackend self.assertIs(local_backend.build(), LocalBackend) @@ -180,10 +43,8 @@ def test_build(self): copied = copy.deepcopy(local_backend) self.assertDictEqual(copied.__dict__, local_backend.__dict__) - with self.assertRaises(RuntimeError): + with self.assertRaises(TypeError): local_backend() - with self.assertRaisesRegex( - ImportError, - 'Failed to import mmengine.fileio.LocalBackend.unknown'): + with self.assertRaisesRegex(ImportError, 'Failed to import'): local_backend.unknown.build() diff --git a/tests/test_registry/test_registry.py b/tests/test_registry/test_registry.py index 6a902c5b71..30ac3e50e3 100644 --- a/tests/test_registry/test_registry.py +++ b/tests/test_registry/test_registry.py @@ -323,12 +323,12 @@ class LittlePedigreeSamoyed: assert LITTLE_HOUNDS.get('mid_hound.PedigreeSamoyedddddd') is None # Get mmengine.utils by string - utils = LITTLE_HOUNDS.get('mmengine.utils') + utils = LITTLE_HOUNDS.get('') import mmengine.utils assert utils is mmengine.utils - unknown = LITTLE_HOUNDS.get('mmengine.unknown') - assert unknown is None + with pytest.raises(RuntimeError, match='Failed to get'): + LITTLE_HOUNDS.get('') def test__search_child(self): # Hierarchical Registry From 5d79f39cdc5ee8839469eb819eac950d81bcce82 Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Tue, 29 Aug 2023 13:10:57 +0800 Subject: [PATCH 07/10] Imporve according to comments --- mmengine/config/lazy.py | 51 ++++++++++++++++++++++------------- mmengine/config/new_config.py | 13 ++++++--- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/mmengine/config/lazy.py b/mmengine/config/lazy.py index fc10dcc49b..78a88f3662 100644 --- a/mmengine/config/lazy.py +++ b/mmengine/config/lazy.py @@ -2,8 +2,8 @@ import importlib import re import sys -from importlib.util import spec_from_loader -from typing import Any +from importlib.util import find_spec, spec_from_loader +from typing import Any, Optional class LazyObject: @@ -16,30 +16,27 @@ class LazyObject: >>> import torch.nn as nn >>> from mmdet.models import RetinaNet >>> import mmcls.models - >>> import mmcls.datasets - >>> import mmcls Will be parsed as: Examples: >>> # import torch.nn as nn - >>> nn = lazyObject('torch.nn') + >>> nn = LazyObject('torch.nn') >>> # from mmdet.models import RetinaNet - >>> RetinaNet = lazyObject('mmdet.models', 'RetinaNet') - >>> # import mmcls.models; import mmcls.datasets; import mmcls - >>> mmcls = lazyObject(['mmcls', 'mmcls.datasets', 'mmcls.models']) + >>> RetinaNet = LazyObject('RetinaNet', LazyObject('mmdet.models')) + >>> # import mmcls.models + >>> mmcls = LazyObject('mmcls.models') ``LazyObject`` records all module information and will be further referenced by the configuration file. Args: - module (str or list or tuple): The module name to be imported. - imported (str, optional): The imported module name. Defaults to None. - location (str, optional): The filename and line number of the imported - module statement happened. + name (str): The name of a module or attribution. + source (LazyObject, optional): The source of the lazy object. + Defaults to None. """ - def __init__(self, name: str, source: 'LazyObject' = None): + def __init__(self, name: str, source: Optional['LazyObject'] = None): self.name = name self.source = source @@ -58,9 +55,20 @@ def build(self) -> Any: f'Failed to import {self.name} from {self.source}') else: try: - return importlib.import_module(self.name) - except Exception as e: - raise type(e)(f'Failed to import {self.name} for {e}') + for idx in range(self.name.count('.') + 1): + module, *attrs = self.name.rsplit('.', idx) + try: + spec = find_spec(module) + except ImportError: + spec = None + if spec is not None: + res = importlib.import_module(module) + for attr in attrs: + res = getattr(res, attr) + return res + raise ImportError(f'No module named `{module}`.') + except (ImportError, AttributeError) as e: + raise ImportError(f'Failed to import {self.name} for {e}') def __deepcopy__(self, memo): return LazyObject(self.name, self.source) @@ -74,10 +82,13 @@ def __str__(self) -> str: return self.name def __repr__(self) -> str: - return f"" + arg = f'name={repr(self.name)}' + if self.source is not None: + arg += f', source={repr(self.source)}' + return f'LazyObject({arg})' @property - def dump_str(self): + def dump_str(self) -> str: return f'<{str(self)}>' @classmethod @@ -130,6 +141,10 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): sys.meta_path.remove(self) for name in self.lazy_modules: + if '.' in name: + parent_module, _, child_name = name.rpartition('.') + if parent_module in sys.modules: + delattr(sys.modules[parent_module], child_name) sys.modules.pop(name, None) def __repr__(self): diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py index 6c52efc397..5edc64f1c9 100644 --- a/mmengine/config/new_config.py +++ b/mmengine/config/new_config.py @@ -35,13 +35,13 @@ def dump_extra_type(value): if isinstance(value, LazyObject): return value.dump_str if isinstance(value, (type, FunctionType, BuiltinFunctionType)): - return '<' + value.__module__ + '.' + value.__name__ + '>' + return LazyObject(value.__name__, value.__module__).dump_str if isinstance(value, ModuleType): - return f'<{value.__name__}>' + return LazyObject(value.__name__).dump_str typename = type(value).__module__ + type(value).__name__ if typename == 'torch.dtype': - return '<' + str(value) + '>' + return LazyObject(str(value)).dump_str return None @@ -393,6 +393,13 @@ def __enter__(self): old_import = builtins.__import__ def new_import(name, globals=None, locals=None, fromlist=(), level=0): + # For relative import, the new import allows import from files + # which are not in a package. + # For absolute import, the new import will try to find the python + # file according to the module name literally, it's used to handle + # importing from installed packages, like + # `mmpretrain.configs.resnet.resnet18_8xb32_in1k`. + cur_file = None # Try to import the base config source file From b9af2852fdd2e03e76f7a7ca23a159877a87d056 Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Wed, 13 Sep 2023 03:47:36 +0000 Subject: [PATCH 08/10] Fix dump extra type --- mmengine/config/new_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py index 5edc64f1c9..c8d6f93d4a 100644 --- a/mmengine/config/new_config.py +++ b/mmengine/config/new_config.py @@ -39,7 +39,7 @@ def dump_extra_type(value): if isinstance(value, ModuleType): return LazyObject(value.__name__).dump_str - typename = type(value).__module__ + type(value).__name__ + typename = type(value).__module__ + '.' + type(value).__name__ if typename == 'torch.dtype': return LazyObject(str(value)).dump_str From 5c7a83bb81c35f46f885ef802ba36243860ab347 Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Wed, 11 Oct 2023 13:52:10 +0800 Subject: [PATCH 09/10] Update dump string of lazy object to avoid confusion --- mmengine/config/lazy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmengine/config/lazy.py b/mmengine/config/lazy.py index 78a88f3662..f96e8cdf1b 100644 --- a/mmengine/config/lazy.py +++ b/mmengine/config/lazy.py @@ -89,11 +89,11 @@ def __repr__(self) -> str: @property def dump_str(self) -> str: - return f'<{str(self)}>' + return f'' @classmethod def from_str(cls, string): - match_ = re.match(r'<([\w\.]+)>', string) + match_ = re.match(r'^$', string) if match_ and '.' in match_.group(1): source, _, name = match_.group(1).rpartition('.') return cls(name, cls(source)) From 52a5c7aefd056d462086ee5cee46d2c8aadf3fbe Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Fri, 25 Oct 2024 15:29:12 +0800 Subject: [PATCH 10/10] Update FormatCode in new config --- mmengine/config/new_config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mmengine/config/new_config.py b/mmengine/config/new_config.py index c8d6f93d4a..61c086f828 100644 --- a/mmengine/config/new_config.py +++ b/mmengine/config/new_config.py @@ -10,8 +10,10 @@ from types import BuiltinFunctionType, FunctionType, ModuleType from typing import Optional, Tuple, Union +import yapf from yapf.yapflib.yapf_api import FormatCode +from mmengine.utils import digit_version from .config import Config, ConfigDict from .lazy import LazyImportContext, LazyObject, recover_lazy_field @@ -321,8 +323,11 @@ def _format_basic_types(input_): blank_line_before_nested_class_or_def=True, split_before_expression_after_opening_paren=True) try: - text, _ = FormatCode( - text, style_config=yapf_style, verify=True) + if digit_version(yapf.__version__) >= digit_version('0.40.2'): + text, _ = FormatCode(text, style_config=yapf_style) + else: + text, _ = FormatCode( + text, style_config=yapf_style, verify=True) except: # noqa: E722 raise SyntaxError('Failed to format the config file, please ' f'check the syntax of: \n{text}')