diff --git a/mim/commands/download.py b/mim/commands/download.py index e93d9b8..5e0b2c9 100644 --- a/mim/commands/download.py +++ b/mim/commands/download.py @@ -1,5 +1,4 @@ import os.path as osp -from pkg_resources import resource_filename from typing import List, Optional import click @@ -8,9 +7,9 @@ from mim.click import OptionEatAll, get_downstream_package, param2lowercase from mim.commands.search import get_model_info from mim.utils import ( DEFAULT_CACHE_DIR, - PKG2MODULE, download_from_file, echo_success, + get_installed_path, highlighted_error, is_installed, split_package_version, @@ -101,17 +100,23 @@ def download(package: str, config_paths = model_info[config]['config'] for config_path in config_paths.split(','): - module_name = PKG2MODULE.get(package, package) - config_path = resource_filename(module_name, config_path) - if not osp.exists(config_path): + installed_path = get_installed_path(package) + # configs will be put in package/.mim in PR #68 + possible_config_paths = [ + osp.join(installed_path, '.mim', config_path), + osp.join(installed_path, config_path) + ] + for config_path in possible_config_paths: + if osp.exists(config_path): + config_obj = Config.fromfile(config_path) + saved_config_path = osp.join(dest_root, f'{config}.py') + config_obj.dump(saved_config_path) + echo_success( + f'Successfully dumped {config}.py to {dest_root}') + checkpoints.append(filename) + break + else: raise ValueError( highlighted_error(f'{config_path} is not found.')) - config_obj = Config.fromfile(config_path) - saved_config_path = osp.join(dest_root, f'{config}.py') - config_obj.dump(saved_config_path) - echo_success(f'Successfully dumped {config}.py to {dest_root}') - - checkpoints.append(filename) - return checkpoints diff --git a/mim/commands/gridsearch.py b/mim/commands/gridsearch.py index 362b2f4..3838850 100644 --- a/mim/commands/gridsearch.py +++ b/mim/commands/gridsearch.py @@ -238,7 +238,10 @@ def gridsearch( f'in codebase {package}, will use {files[0]} instead.') config = files[0] - train_script = osp.join(pkg_root, 'tools/train.py') + # tools will be put in package/.mim in PR #68 + train_script = osp.join(pkg_root, '.mim', 'tools', 'train.py') + if not osp.exists(train_script): + train_script = osp.join(pkg_root, 'tools', 'train.py') # parsing search_args # the search_args looks like: diff --git a/mim/commands/install.py b/mim/commands/install.py index ca89ca0..ba16ab4 100644 --- a/mim/commands/install.py +++ b/mim/commands/install.py @@ -3,7 +3,7 @@ import os.path as osp import shutil import tempfile from distutils.version import LooseVersion -from pkg_resources import parse_requirements +from pkg_resources import parse_requirements, resource_filename from typing import List import click @@ -405,17 +405,18 @@ def install_from_repo(repo_root: str, def copy_file_to_package(): # rename the model_zoo.yml to model-index.yml but support both of them # for backward compatibility - items = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml'] + filenames = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml'] module_name = PKG2MODULE.get(package, package) - pkg_root = osp.join(repo_root, module_name) + # configs, tools and model-index.yml will be copied to package/.mim + mim_root = resource_filename(module_name, '.mim') + os.makedirs(mim_root, exist_ok=True) - for item in items: - src_path = osp.join(repo_root, item) - dst_path = osp.join(pkg_root, item) + for filename in filenames: + src_path = osp.join(repo_root, filename) + dst_path = osp.join(mim_root, filename) if osp.exists(src_path): if osp.islink(dst_path): os.unlink(dst_path) - if osp.isfile(src_path): shutil.copyfile(src_path, dst_path) elif osp.isdir(src_path): @@ -428,13 +429,18 @@ def install_from_repo(repo_root: str, # symlinks to package, which will synchronize the modified files. # Besides, rename the model_zoo.yml to model-index.yml but support both # of them for backward compatibility - items = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml'] + filenames = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml'] module_name = PKG2MODULE.get(package, package) pkg_root = osp.join(repo_root, module_name) + # configs, tools and model-index.yml will be linked to package/.mim + mim_root = osp.join(pkg_root, '.mim') + os.makedirs(mim_root, exist_ok=True) - for item in items: - src_path = osp.join(repo_root, item) - dst_path = osp.join(pkg_root, item) + for filename in filenames: + src_path = osp.join(repo_root, filename) + dst_path = osp.join(mim_root, filename) + if osp.exists(dst_path): + continue if osp.exists(src_path): if osp.isfile(dst_path) or osp.islink(dst_path): os.remove(dst_path) @@ -443,11 +449,6 @@ def install_from_repo(repo_root: str, os.symlink(src_path, dst_path) - if is_editable: - link_file_to_package() - else: - copy_file_to_package() - # install dependencies. For example, # install mmcls should install mmcv-full first if it is not installed or # its(mmcv) verison does not match. @@ -457,7 +458,7 @@ def install_from_repo(repo_root: str, if dependencies: install_dependencies(dependencies, timeout, is_yes, is_user_dir) - third_dependencies = osp.join(repo_root, 'requirements', '/build.txt') + third_dependencies = osp.join(repo_root, 'requirements', 'build.txt') if osp.exists(third_dependencies): dep_cmd = [ 'python', '-m', 'pip', 'install', '-r', third_dependencies, @@ -489,6 +490,11 @@ def install_from_repo(repo_root: str, call_command(install_cmd) + if is_editable: + link_file_to_package() + else: + copy_file_to_package() + def install_from_github(package: str, version: str = '', diff --git a/mim/commands/list.py b/mim/commands/list.py index 8478490..6fe8386 100644 --- a/mim/commands/list.py +++ b/mim/commands/list.py @@ -6,7 +6,7 @@ from typing import List, Tuple import click from tabulate import tabulate -from mim.utils import parse_home_page +from mim.utils import get_installed_path, parse_home_page @click.command('list') @@ -48,22 +48,21 @@ def list_package(all: bool = False) -> List[Tuple[str, ...]]: if all: pkgs_info.append((pkg_name, pkg.version)) else: - if pkg.has_metadata('top_level.txt'): - module_name = pkg.get_metadata('top_level.txt').split('\n')[0] - if not module_name: - continue + home_page = parse_home_page(pkg_name) + if not home_page: + home_page = pkg.location - home_page = parse_home_page(pkg_name) - if not home_page: - home_page = pkg.location - - # rename the model_zoo.yml to model-index.yml but support both - # of them for backward compatibility - possible_metadata_paths = [ - osp.join(pkg.location, module_name, 'model-index.yml'), - osp.join(pkg.location, module_name, 'model_zoo.yml') - ] - if pkg_name.startswith('mmcv') or any( - map(osp.exists, possible_metadata_paths)): - pkgs_info.append((pkg_name, pkg.version, home_page)) + installed_path = get_installed_path(pkg_name) + # rename the model_zoo.yml to model-index.yml but support both + # of them for backward compatibility. In addition, model-index.yml + # will be put in package/.mim in PR #68 + possible_metadata_paths = [ + osp.join(installed_path, '.mim', 'model-index.yml'), + osp.join(installed_path, 'model-index.yml'), + osp.join(installed_path, '.mim', 'model_zoo.yml'), + osp.join(installed_path, 'model_zoo.yml') + ] + if pkg_name.startswith('mmcv') or any( + map(osp.exists, possible_metadata_paths)): + pkgs_info.append((pkg_name, pkg.version, home_page)) return pkgs_info diff --git a/mim/commands/run.py b/mim/commands/run.py index a8593b2..d9d63ee 100644 --- a/mim/commands/run.py +++ b/mim/commands/run.py @@ -2,6 +2,7 @@ import os import os.path as osp import subprocess +from genericpath import exists from typing import Tuple, Union import click @@ -108,8 +109,14 @@ def run( return False, msg pkg_root = get_installed_path(package) - - prefix = osp.join(pkg_root, 'tools/') + possible_prefixes = [ + osp.join(pkg_root, '.mim', 'tools/'), + osp.join(pkg_root, 'tools/') + ] + for possible_prefix in possible_prefixes: + if osp.exists(possible_prefix): + prefix = possible_prefix + break command_domain = '' if ':' in command: diff --git a/mim/commands/search.py b/mim/commands/search.py index 790df1a..f2fb7be 100644 --- a/mim/commands/search.py +++ b/mim/commands/search.py @@ -211,21 +211,23 @@ def load_metadata_from_local(package: str): """ if is_installed(package): # rename the model_zoo.yml to model-index.yml but support both of them - # for backward compatibility + # for backward compatibility. In addition, model-index.yml will be put + # in package/.mim in PR #68 installed_path = get_installed_path(package) - metadata_path = osp.join(installed_path, 'model-index.yml') - if not osp.exists(metadata_path): - metadata_path = osp.join(installed_path, 'model_zoo.yml') - if not osp.exists(metadata_path): - raise FileNotFoundError( - highlighted_error( - f'{installed_path}/model-index.yml or {installed_path}' - '/model_zoo.yml is not found, please upgrade your ' - f'{package} to support search command')) - - metadata = load(metadata_path) - - return metadata + possible_metadata_paths = [ + osp.join(installed_path, '.mim', 'model-index.yml'), + osp.join(installed_path, 'model-index.yml'), + osp.join(installed_path, '.mim', 'model_zoo.yml'), + osp.join(installed_path, 'model_zoo.yml'), + ] + for metadata_path in possible_metadata_paths: + if osp.exists(metadata_path): + return load(metadata_path) + raise FileNotFoundError( + highlighted_error( + f'{installed_path}/model-index.yml or {installed_path}' + '/model_zoo.yml is not found, please upgrade your ' + f'{package} to support search command')) else: raise ImportError( highlighted_error( @@ -254,7 +256,7 @@ def load_metadata_from_remote(package: str) -> Optional[ModelIndex]: pkl_path = osp.join(DEFAULT_CACHE_DIR, f'{package}-{version}.pkl') if osp.exists(pkl_path): with open(pkl_path, 'rb') as fr: - metadata = pickle.load(fr) + return pickle.load(fr) else: clone_cmd = ['git', 'clone', github_url] if version: @@ -267,23 +269,21 @@ def load_metadata_from_remote(package: str) -> Optional[ModelIndex]: # rename the model_zoo.yml to model-index.yml but support both of # them for backward compatibility - metadata_path = osp.join(repo_root, 'model-index.yml') - if not osp.exists(metadata_path): - metadata_path = osp.join(repo_root, 'model_zoo.yml') - if not osp.exists(metadata_path): - raise FileNotFoundError( - highlighted_error( - 'model-index.yml or model_zoo.yml is not found, ' - f'please upgrade your {package} to support search ' - 'command')) - - metadata = load(metadata_path) - - if version: - with open(pkl_path, 'wb') as fw: - pickle.dump(metadata, fw) - - return metadata + possible_metadata_paths = [ + osp.join(repo_root, 'model-index.yml'), + osp.join(repo_root, 'model_zoo.yml'), + ] + for metadata_path in possible_metadata_paths: + if osp.exists(metadata_path): + metadata = load(metadata_path) + if version: + with open(pkl_path, 'wb') as fw: + pickle.dump(metadata, fw) + return metadata + raise FileNotFoundError( + highlighted_error( + 'model-index.yml or model_zoo.yml is not found, please ' + f'upgrade your {package} to support search command')) def convert2df(metadata: ModelIndex) -> DataFrame: diff --git a/mim/commands/test.py b/mim/commands/test.py index 559ed47..7c6f951 100644 --- a/mim/commands/test.py +++ b/mim/commands/test.py @@ -228,7 +228,10 @@ def test( config = files[0] # We know that 'config' exists and is legal. - test_script = osp.join(pkg_root, 'tools/test.py') + test_script = osp.join(pkg_root, 'tools', 'test.py') + # tools will be put in package/.mim in PR #68 + if not osp.exists(test_script): + test_script = osp.join(pkg_root, '.mim', 'tools', 'test.py') common_args = ['--launcher', launcher] + list(other_args) diff --git a/mim/commands/train.py b/mim/commands/train.py index c363e67..3f7d824 100644 --- a/mim/commands/train.py +++ b/mim/commands/train.py @@ -205,7 +205,10 @@ def train( f'in codebase {package}, will use {files[0]} instead.') config = files[0] - train_script = osp.join(pkg_root, 'tools/train.py') + # tools will be put in package/.mim in PR #68 + train_script = osp.join(pkg_root, '.mim', 'tools', 'train.py') + if not osp.exists(train_script): + train_script = osp.join(pkg_root, 'tools', 'train.py') common_args = ['--launcher', launcher] + list(other_args) diff --git a/tests/test_list.py b/tests/test_list.py index f6d3c3d..360fd11 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -6,12 +6,11 @@ from mim.commands.list import list_package def test_list(): runner = CliRunner() - # mim install mmcls==0.11.0 --yes + # mim install mmcls==0.12.0 --yes result = runner.invoke(install, ['mmcls==0.12.0', '--yes']) assert result.exit_code == 0 # mim list target = ('mmcls', '0.12.0', 'https://github.com/open-mmlab/mmclassification') result = list_package() - print(result) assert target in result