Update config to support predefined variables (#348)

* update config with predefined variables * rm redun Signed-off-by: lixuanyi <lixuanyi@sensetime.com> * add test for config Signed-off-by: lixuanyi <lixuanyi@sensetime.com> * support all types Signed-off-by: lixuanyi <lixuanyi@sensetime.com> * newline at the end Signed-off-by: lixuanyi <lixuanyi@sensetime.com> * update Signed-off-by: lixuanyi <lixuanyi@sensetime.com> * extract code into a function and add docs Signed-off-by: lixuanyi <lixuanyi@sensetime.com> * fix and add tests Signed-off-by: lixuanyi <lixuanyi@sensetime.com> * add unit tests and fix * fix * fix minor * fix test
2020-07-08 20:53:54 +08:00 · 2020-07-08 20:53:54 +08:00 · 27cc439d01
parent 926ac07bb8
commit 27cc439d01
6 changed files with 152 additions and 17 deletions
--- a/docs/utils.md
+++ b/docs/utils.md
@ -26,6 +26,39 @@ To load and use configs
 ...      d='string')
 ```

+For all format configs, some predefined variables are supported. It will convert the variable in `{{ var }}` with its real value.
+
+Currently, it supports four predefined variables:
+
+`{{ fileDirname }}` - the current opened file's dirname, e.g. /home/your-username/your-project/folder
+
+`{{ fileBasename }}` - the current opened file's basename, e.g. file.ext
+
+`{{ fileBasenameNoExtension }}` - the current opened file's basename with no file extension, e.g. file
+
+`{{ fileExtname }}` - the current opened file's extension, e.g. .ext
+
+These variable names are referred from https://code.visualstudio.com/docs/editor/variables-reference.
+
+Here is one examples of config with predefined variables.
+
+`config_a.py`
+
+```python
+a = 1
+b = './work_dir/{{ fileBasenameNoExtension }}'
+c = '{{ fileExtname }}'
+```
+
+```python
+>>> cfg = Config.fromfile('./config_a.py')
+>>> print(cfg)
+>>> dict(a=1,
+...      b='./work_dir/config_a',
+...      c='.py')
+```
+
+
 For all format configs, inheritance is supported. To reuse fields in other config files,
 specify `_base_='./config_a.py'` or a list of configs `_base_=['./config_a.py', './config_b.py']`.
 Here are 4 examples of config inheritance.
--- a/mmcv/utils/config.py
+++ b/mmcv/utils/config.py
@ -1,6 +1,7 @@
 # Copyright (c) Open-MMLab. All rights reserved.
 import ast
 import os.path as osp
+import re
 import shutil
 import sys
 import tempfile
@ -91,16 +92,43 @@ class Config:
                              f'file {filename}')

    @staticmethod
-    def _file2dict(filename):
+    def _substitute_predefined_vars(filename, temp_config_name):
+        file_dirname = osp.dirname(filename)
+        file_basename = osp.basename(filename)
+        file_basename_no_extension = osp.splitext(file_basename)[0]
+        file_extname = osp.splitext(filename)[1]
+        support_templates = dict(
+            fileDirname=file_dirname,
+            fileBasename=file_basename,
+            fileBasenameNoExtension=file_basename_no_extension,
+            fileExtname=file_extname)
+        config_file = open(filename).read()
+        for key, value in support_templates.items():
+            regexp = r'\{\{\s*' + str(key) + r'\s*\}\}'
+            config_file = re.sub(regexp, value, config_file)
+        with open(temp_config_name, 'w') as tmp_config_file:
+            tmp_config_file.write(config_file)
+
+    @staticmethod
+    def _file2dict(filename, use_predefined_variables=True):
        filename = osp.abspath(osp.expanduser(filename))
        check_file_exist(filename)
-        if filename.endswith('.py'):
-            with tempfile.TemporaryDirectory() as temp_config_dir:
-                temp_config_file = tempfile.NamedTemporaryFile(
-                    dir=temp_config_dir, suffix='.py')
-                temp_config_name = osp.basename(temp_config_file.name)
-                shutil.copyfile(filename,
-                                osp.join(temp_config_dir, temp_config_name))
+        fileExtname = osp.splitext(filename)[1]
+        if fileExtname not in ['.py', '.json', '.yaml', 'yml']:
+            raise IOError('Only py/yml/yaml/json type are supported now!')
+
+        with tempfile.TemporaryDirectory() as temp_config_dir:
+            temp_config_file = tempfile.NamedTemporaryFile(
+                dir=temp_config_dir, suffix=fileExtname)
+            temp_config_name = osp.basename(temp_config_file.name)
+            # Substitute predefined variables
+            if use_predefined_variables:
+                Config._substitute_predefined_vars(filename,
+                                                   temp_config_file.name)
+            else:
+                shutil.copyfile(filename, temp_config_file.name)
+
+            if filename.endswith('.py'):
                temp_module_name = osp.splitext(temp_config_name)[0]
                sys.path.insert(0, temp_config_dir)
                Config._validate_py_syntax(filename)
@ -113,13 +141,11 @@ class Config:
                }
                # delete imported module
                del sys.modules[temp_module_name]
-                # close temp file
-                temp_config_file.close()
-        elif filename.endswith(('.yml', '.yaml', '.json')):
-            import mmcv
-            cfg_dict = mmcv.load(filename)
-        else:
-            raise IOError('Only py/yml/yaml/json type are supported now!')
+            elif filename.endswith(('.yml', '.yaml', '.json')):
+                import mmcv
+                cfg_dict = mmcv.load(temp_config_file.name)
+            # close temp file
+            temp_config_file.close()

        cfg_text = filename + '\n'
        with open(filename, 'r') as f:
@ -173,8 +199,9 @@ class Config:
        return b

    @staticmethod
-    def fromfile(filename):
-        cfg_dict, cfg_text = Config._file2dict(filename)
+    def fromfile(filename, use_predefined_variables=True):
+        cfg_dict, cfg_text = Config._file2dict(filename,
+                                               use_predefined_variables)
        return Config(cfg_dict, cfg_text=cfg_text, filename=filename)

    @staticmethod
--- a/tests/data/config/h.py
+++ b/tests/data/config/h.py
@ -0,0 +1,3 @@
+item1 = '{{fileBasename}}'
+item2 = '{{ fileDirname}}'
+item3 = 'abc_{{ fileBasenameNoExtension }}'
--- a/tests/data/config/o.json
+++ b/tests/data/config/o.json
@ -0,0 +1,3 @@
+{
+    "item1": "{{ fileDirname }}"
+}
--- a/tests/data/config/p.yaml
+++ b/tests/data/config/p.yaml
@ -0,0 +1 @@
+item1: '{{ fileDirname }}'
--- a/tests/test_config.py
+++ b/tests/test_config.py
@ -60,6 +60,74 @@ def test_construct():
        assert cfg.dump() == open(dump_file, 'r').read()
        assert Config.fromfile(dump_file)

+    # test h.py
+    cfg_file = osp.join(osp.dirname(__file__), 'data/config/h.py')
+    cfg_dict = dict(
+        item1='h.py',
+        item2=f'{osp.dirname(__file__)}/data/config',
+        item3='abc_h')
+    cfg = Config(cfg_dict, filename=cfg_file)
+    assert isinstance(cfg, Config)
+    assert cfg.filename == cfg_file
+    assert cfg.text == open(cfg_file, 'r').read()
+    assert cfg.dump() == cfg.pretty_text
+    with tempfile.TemporaryDirectory() as temp_config_dir:
+        dump_file = osp.join(temp_config_dir, 'h.py')
+        cfg.dump(dump_file)
+        assert cfg.dump() == open(dump_file, 'r').read()
+        assert Config.fromfile(dump_file)
+        assert Config.fromfile(dump_file)['item1'] == cfg_dict['item1']
+        assert Config.fromfile(dump_file)['item2'] == cfg_dict['item2']
+        assert Config.fromfile(dump_file)['item3'] == cfg_dict['item3']
+
+    # test no use_predefined_variable
+    cfg_dict = dict(
+        item1='{{fileBasename}}',
+        item2='{{ fileDirname}}',
+        item3='abc_{{ fileBasenameNoExtension }}')
+    assert Config.fromfile(cfg_file, False)
+    assert Config.fromfile(cfg_file, False)['item1'] == cfg_dict['item1']
+    assert Config.fromfile(cfg_file, False)['item2'] == cfg_dict['item2']
+    assert Config.fromfile(cfg_file, False)['item3'] == cfg_dict['item3']
+
+    # test p.yaml
+    cfg_file = osp.join(osp.dirname(__file__), 'data/config/p.yaml')
+    cfg_dict = dict(item1=f'{osp.dirname(__file__)}/data/config')
+    cfg = Config(cfg_dict, filename=cfg_file)
+    assert isinstance(cfg, Config)
+    assert cfg.filename == cfg_file
+    assert cfg.text == open(cfg_file, 'r').read()
+    assert cfg.dump() == yaml.dump(cfg_dict)
+    with tempfile.TemporaryDirectory() as temp_config_dir:
+        dump_file = osp.join(temp_config_dir, 'p.yaml')
+        cfg.dump(dump_file)
+        assert cfg.dump() == open(dump_file, 'r').read()
+        assert Config.fromfile(dump_file)
+        assert Config.fromfile(dump_file)['item1'] == cfg_dict['item1']
+
+    # test no use_predefined_variable
+    assert Config.fromfile(cfg_file, False)
+    assert Config.fromfile(cfg_file, False)['item1'] == '{{ fileDirname }}'
+
+    # test o.json
+    cfg_file = osp.join(osp.dirname(__file__), 'data/config/o.json')
+    cfg_dict = dict(item1=f'{osp.dirname(__file__)}/data/config')
+    cfg = Config(cfg_dict, filename=cfg_file)
+    assert isinstance(cfg, Config)
+    assert cfg.filename == cfg_file
+    assert cfg.text == open(cfg_file, 'r').read()
+    assert cfg.dump() == json.dumps(cfg_dict)
+    with tempfile.TemporaryDirectory() as temp_config_dir:
+        dump_file = osp.join(temp_config_dir, 'o.json')
+        cfg.dump(dump_file)
+        assert cfg.dump() == open(dump_file, 'r').read()
+        assert Config.fromfile(dump_file)
+        assert Config.fromfile(dump_file)['item1'] == cfg_dict['item1']
+
+    # test no use_predefined_variable
+    assert Config.fromfile(cfg_file, False)
+    assert Config.fromfile(cfg_file, False)['item1'] == '{{ fileDirname }}'
+

 def test_fromfile():
    for filename in ['a.py', 'a.b.py', 'b.json', 'c.yaml']: