From d70f9cf14b860111a0f6ae32d44cd118623e8dfd Mon Sep 17 00:00:00 2001 From: Ma Zerun Date: Tue, 27 Jul 2021 04:02:24 -0400 Subject: [PATCH] [Fix] Fix searching path of config when invoking train, test and gridsearch command (#75) * Add `followlinks` argument in `recursively_find`. * When searching config, `pkg/.mim/configs` is the default path to be searched if it exists. --- mim/commands/gridsearch.py | 11 ++++++++++- mim/commands/test.py | 11 ++++++++++- mim/commands/train.py | 11 ++++++++++- mim/utils/utils.py | 5 +++-- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/mim/commands/gridsearch.py b/mim/commands/gridsearch.py index 1f6a374..47a497c 100644 --- a/mim/commands/gridsearch.py +++ b/mim/commands/gridsearch.py @@ -222,7 +222,16 @@ def gridsearch( pkg_root = get_installed_path(package) if not osp.exists(config): - files = recursively_find(pkg_root, osp.basename(config)) + # configs is put in pkg/.mim in PR #68 + config_root = osp.join(pkg_root, '.mim', 'configs') + if not osp.exists(config_root): + # If not pkg/.mim/config, try to search the whole pkg root. + config_root = pkg_root + + # pkg/.mim/configs is a symbolic link to the real config folder, + # so we need to follow links. + files = recursively_find( + pkg_root, osp.basename(config), followlinks=True) if len(files) == 0: msg = (f"The path {config} doesn't exist and we can not " diff --git a/mim/commands/test.py b/mim/commands/test.py index c5978e3..4fe1049 100644 --- a/mim/commands/test.py +++ b/mim/commands/test.py @@ -211,7 +211,16 @@ def test( pkg_root = get_installed_path(package) if not osp.exists(config): - files = recursively_find(pkg_root, osp.basename(config)) + # configs is put in pkg/.mim in PR #68 + config_root = osp.join(pkg_root, '.mim', 'configs') + if not osp.exists(config_root): + # If not pkg/.mim/config, try to search the whole pkg root. + config_root = pkg_root + + # pkg/.mim/configs is a symbolic link to the real config folder, + # so we need to follow links. + files = recursively_find( + pkg_root, osp.basename(config), followlinks=True) if len(files) == 0: msg = (f"The path {config} doesn't exist and we can not find " diff --git a/mim/commands/train.py b/mim/commands/train.py index ae179ec..dc18fac 100644 --- a/mim/commands/train.py +++ b/mim/commands/train.py @@ -188,7 +188,16 @@ def train( pkg_root = get_installed_path(package) if not osp.exists(config): - files = recursively_find(pkg_root, osp.basename(config)) + # configs is put in pkg/.mim in PR #68 + config_root = osp.join(pkg_root, '.mim', 'configs') + if not osp.exists(config_root): + # If not pkg/.mim/config, try to search the whole pkg root. + config_root = pkg_root + + # pkg/.mim/configs is a symbolic link to the real config folder, + # so we need to follow links. + files = recursively_find( + pkg_root, osp.basename(config), followlinks=True) if len(files) == 0: msg = (f"The path {config} doesn't exist and we can not find " diff --git a/mim/utils/utils.py b/mim/utils/utils.py index c5cfa13..338d297 100644 --- a/mim/utils/utils.py +++ b/mim/utils/utils.py @@ -369,18 +369,19 @@ def cast2lowercase(input: Union[list, tuple, str]) -> Any: return outputs -def recursively_find(root: str, base_name: str) -> list: +def recursively_find(root: str, base_name: str, followlinks=False) -> list: """Recursive list a directory, return all files with a given base_name. Args: root (str): The root directory to list. base_name (str): The base_name. + followlinks (bool): Follow symbolic links. Defaults to False. Return: Files with given base_name. """ files = [] - for _root, _, _files in os.walk(root): + for _root, _, _files in os.walk(root, followlinks=followlinks): if base_name in _files: files.append(osp.join(_root, base_name))