mirrors/mim
1
0
Fork 0
mirror of https://github.com/open-mmlab/mim.git synced 2025-06-03 14:59:11 +08:00
Code Issues Projects Releases Wiki Activity

[Refactor] Change resources directory to .mim (#68)

Browse Source 
* [Enhancement] Change the directory of configs to be saved

* configs tools model-index.yml will be copied to .mim

* copy files to .mim if setup.py does not handle this

* simply list command

* remove redundant code

* remove redundant code

* modify according to comment
This commit is contained in:
Zaida Zhou 2021-07-24 17:27:21 +08:00 committed by GitHub
parent 0bd1f4d2dd
commit acab45a6fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 111 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
mim/commands/download.py
View File

@ -1,5 +1,4 @@
import os.path as osp
from pkg_resources import resource_filename
from typing import List, Optional
import click
@ -8,9 +7,9 @@ from mim.click import OptionEatAll, get_downstream_package, param2lowercase
from mim.commands.search import get_model_info
from mim.utils import (
DEFAULT_CACHE_DIR,
PKG2MODULE,
download_from_file,
echo_success,
get_installed_path,
highlighted_error,
is_installed,
split_package_version,
@ -101,17 +100,23 @@ def download(package: str,
config_paths = model_info[config]['config']
for config_path in config_paths.split(','):
module_name = PKG2MODULE.get(package, package)
config_path = resource_filename(module_name, config_path)
if not osp.exists(config_path):
installed_path = get_installed_path(package)
# configs will be put in package/.mim in PR #68
possible_config_paths = [
osp.join(installed_path, '.mim', config_path),
osp.join(installed_path, config_path)
]
for config_path in possible_config_paths:
if osp.exists(config_path):
config_obj = Config.fromfile(config_path)
saved_config_path = osp.join(dest_root, f'{config}.py')
config_obj.dump(saved_config_path)
echo_success(
f'Successfully dumped {config}.py to {dest_root}')
checkpoints.append(filename)
break
else:
raise ValueError(
highlighted_error(f'{config_path} is not found.'))
config_obj = Config.fromfile(config_path)
saved_config_path = osp.join(dest_root, f'{config}.py')
config_obj.dump(saved_config_path)
echo_success(f'Successfully dumped {config}.py to {dest_root}')
checkpoints.append(filename)
return checkpoints

5
mim/commands/gridsearch.py
View File

@ -238,7 +238,10 @@ def gridsearch(
f'in codebase {package}, will use {files[0]} instead.')
config = files[0]
train_script = osp.join(pkg_root, 'tools/train.py')
# tools will be put in package/.mim in PR #68
train_script = osp.join(pkg_root, '.mim', 'tools', 'train.py')
if not osp.exists(train_script):
train_script = osp.join(pkg_root, 'tools', 'train.py')
# parsing search_args
# the search_args looks like:

40
mim/commands/install.py
View File

@ -3,7 +3,7 @@ import os.path as osp
import shutil
import tempfile
from distutils.version import LooseVersion
from pkg_resources import parse_requirements
from pkg_resources import parse_requirements, resource_filename
from typing import List
import click
@ -405,17 +405,18 @@ def install_from_repo(repo_root: str,
def copy_file_to_package():
# rename the model_zoo.yml to model-index.yml but support both of them
# for backward compatibility
items = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml']
filenames = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml']
module_name = PKG2MODULE.get(package, package)
pkg_root = osp.join(repo_root, module_name)
# configs, tools and model-index.yml will be copied to package/.mim
mim_root = resource_filename(module_name, '.mim')
os.makedirs(mim_root, exist_ok=True)
for item in items:
src_path = osp.join(repo_root, item)
dst_path = osp.join(pkg_root, item)
for filename in filenames:
src_path = osp.join(repo_root, filename)
dst_path = osp.join(mim_root, filename)
if osp.exists(src_path):
if osp.islink(dst_path):
os.unlink(dst_path)
if osp.isfile(src_path):
shutil.copyfile(src_path, dst_path)
elif osp.isdir(src_path):
@ -428,13 +429,18 @@ def install_from_repo(repo_root: str,
# symlinks to package, which will synchronize the modified files.
# Besides, rename the model_zoo.yml to model-index.yml but support both
# of them for backward compatibility
items = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml']
filenames = ['tools', 'configs', 'model_zoo.yml', 'model-index.yml']
module_name = PKG2MODULE.get(package, package)
pkg_root = osp.join(repo_root, module_name)
# configs, tools and model-index.yml will be linked to package/.mim
mim_root = osp.join(pkg_root, '.mim')
os.makedirs(mim_root, exist_ok=True)
for item in items:
src_path = osp.join(repo_root, item)
dst_path = osp.join(pkg_root, item)
for filename in filenames:
src_path = osp.join(repo_root, filename)
dst_path = osp.join(mim_root, filename)
if osp.exists(dst_path):
continue
if osp.exists(src_path):
if osp.isfile(dst_path) or osp.islink(dst_path):
os.remove(dst_path)
@ -443,11 +449,6 @@ def install_from_repo(repo_root: str,
os.symlink(src_path, dst_path)
if is_editable:
link_file_to_package()
else:
copy_file_to_package()
# install dependencies. For example,
# install mmcls should install mmcv-full first if it is not installed or
# its(mmcv) verison does not match.
@ -457,7 +458,7 @@ def install_from_repo(repo_root: str,
if dependencies:
install_dependencies(dependencies, timeout, is_yes, is_user_dir)
third_dependencies = osp.join(repo_root, 'requirements', '/build.txt')
third_dependencies = osp.join(repo_root, 'requirements', 'build.txt')
if osp.exists(third_dependencies):
dep_cmd = [
'python', '-m', 'pip', 'install', '-r', third_dependencies,
@ -489,6 +490,11 @@ def install_from_repo(repo_root: str,
call_command(install_cmd)
if is_editable:
link_file_to_package()
else:
copy_file_to_package()
def install_from_github(package: str,
version: str = '',

35
mim/commands/list.py
View File

@ -6,7 +6,7 @@ from typing import List, Tuple
import click
from tabulate import tabulate
from mim.utils import parse_home_page
from mim.utils import get_installed_path, parse_home_page
@click.command('list')
@ -48,22 +48,21 @@ def list_package(all: bool = False) -> List[Tuple[str, ...]]:
if all:
pkgs_info.append((pkg_name, pkg.version))
else:
if pkg.has_metadata('top_level.txt'):
module_name = pkg.get_metadata('top_level.txt').split('\n')[0]
if not module_name:
continue
home_page = parse_home_page(pkg_name)
if not home_page:
home_page = pkg.location
home_page = parse_home_page(pkg_name)
if not home_page:
home_page = pkg.location
# rename the model_zoo.yml to model-index.yml but support both
# of them for backward compatibility
possible_metadata_paths = [
osp.join(pkg.location, module_name, 'model-index.yml'),
osp.join(pkg.location, module_name, 'model_zoo.yml')
]
if pkg_name.startswith('mmcv') or any(
map(osp.exists, possible_metadata_paths)):
pkgs_info.append((pkg_name, pkg.version, home_page))
installed_path = get_installed_path(pkg_name)
# rename the model_zoo.yml to model-index.yml but support both
# of them for backward compatibility. In addition, model-index.yml
# will be put in package/.mim in PR #68
possible_metadata_paths = [
osp.join(installed_path, '.mim', 'model-index.yml'),
osp.join(installed_path, 'model-index.yml'),
osp.join(installed_path, '.mim', 'model_zoo.yml'),
osp.join(installed_path, 'model_zoo.yml')
]
if pkg_name.startswith('mmcv') or any(
map(osp.exists, possible_metadata_paths)):
pkgs_info.append((pkg_name, pkg.version, home_page))
return pkgs_info

11
mim/commands/run.py
View File

@ -2,6 +2,7 @@
import os
import os.path as osp
import subprocess
from genericpath import exists
from typing import Tuple, Union
import click
@ -108,8 +109,14 @@ def run(
return False, msg
pkg_root = get_installed_path(package)
prefix = osp.join(pkg_root, 'tools/')
possible_prefixes = [
osp.join(pkg_root, '.mim', 'tools/'),
osp.join(pkg_root, 'tools/')
]
for possible_prefix in possible_prefixes:
if osp.exists(possible_prefix):
prefix = possible_prefix
break
command_domain = ''
if ':' in command:

64
mim/commands/search.py
View File

@ -211,21 +211,23 @@ def load_metadata_from_local(package: str):
"""
if is_installed(package):
# rename the model_zoo.yml to model-index.yml but support both of them
# for backward compatibility
# for backward compatibility. In addition, model-index.yml will be put
# in package/.mim in PR #68
installed_path = get_installed_path(package)
metadata_path = osp.join(installed_path, 'model-index.yml')
if not osp.exists(metadata_path):
metadata_path = osp.join(installed_path, 'model_zoo.yml')
if not osp.exists(metadata_path):
raise FileNotFoundError(
highlighted_error(
f'{installed_path}/model-index.yml or {installed_path}'
'/model_zoo.yml is not found, please upgrade your '
f'{package} to support search command'))
metadata = load(metadata_path)
return metadata
possible_metadata_paths = [
osp.join(installed_path, '.mim', 'model-index.yml'),
osp.join(installed_path, 'model-index.yml'),
osp.join(installed_path, '.mim', 'model_zoo.yml'),
osp.join(installed_path, 'model_zoo.yml'),
]
for metadata_path in possible_metadata_paths:
if osp.exists(metadata_path):
return load(metadata_path)
raise FileNotFoundError(
highlighted_error(
f'{installed_path}/model-index.yml or {installed_path}'
'/model_zoo.yml is not found, please upgrade your '
f'{package} to support search command'))
else:
raise ImportError(
highlighted_error(
@ -254,7 +256,7 @@ def load_metadata_from_remote(package: str) -> Optional[ModelIndex]:
pkl_path = osp.join(DEFAULT_CACHE_DIR, f'{package}-{version}.pkl')
if osp.exists(pkl_path):
with open(pkl_path, 'rb') as fr:
metadata = pickle.load(fr)
return pickle.load(fr)
else:
clone_cmd = ['git', 'clone', github_url]
if version:
@ -267,23 +269,21 @@ def load_metadata_from_remote(package: str) -> Optional[ModelIndex]:
# rename the model_zoo.yml to model-index.yml but support both of
# them for backward compatibility
metadata_path = osp.join(repo_root, 'model-index.yml')
if not osp.exists(metadata_path):
metadata_path = osp.join(repo_root, 'model_zoo.yml')
if not osp.exists(metadata_path):
raise FileNotFoundError(
highlighted_error(
'model-index.yml or model_zoo.yml is not found, '
f'please upgrade your {package} to support search '
'command'))
metadata = load(metadata_path)
if version:
with open(pkl_path, 'wb') as fw:
pickle.dump(metadata, fw)
return metadata
possible_metadata_paths = [
osp.join(repo_root, 'model-index.yml'),
osp.join(repo_root, 'model_zoo.yml'),
]
for metadata_path in possible_metadata_paths:
if osp.exists(metadata_path):
metadata = load(metadata_path)
if version:
with open(pkl_path, 'wb') as fw:
pickle.dump(metadata, fw)
return metadata
raise FileNotFoundError(
highlighted_error(
'model-index.yml or model_zoo.yml is not found, please '
f'upgrade your {package} to support search command'))
def convert2df(metadata: ModelIndex) -> DataFrame:

5
mim/commands/test.py
View File

@ -228,7 +228,10 @@ def test(
config = files[0]
# We know that 'config' exists and is legal.
test_script = osp.join(pkg_root, 'tools/test.py')
test_script = osp.join(pkg_root, 'tools', 'test.py')
# tools will be put in package/.mim in PR #68
if not osp.exists(test_script):
test_script = osp.join(pkg_root, '.mim', 'tools', 'test.py')
common_args = ['--launcher', launcher] + list(other_args)

5
mim/commands/train.py
View File

@ -205,7 +205,10 @@ def train(
f'in codebase {package}, will use {files[0]} instead.')
config = files[0]
train_script = osp.join(pkg_root, 'tools/train.py')
# tools will be put in package/.mim in PR #68
train_script = osp.join(pkg_root, '.mim', 'tools', 'train.py')
if not osp.exists(train_script):
train_script = osp.join(pkg_root, 'tools', 'train.py')
common_args = ['--launcher', launcher] + list(other_args)

3
tests/test_list.py
View File

@ -6,12 +6,11 @@ from mim.commands.list import list_package
def test_list():
runner = CliRunner()
# mim install mmcls==0.11.0 --yes
# mim install mmcls==0.12.0 --yes
result = runner.invoke(install, ['mmcls==0.12.0', '--yes'])
assert result.exit_code == 0
# mim list
target = ('mmcls', '0.12.0',
'https://github.com/open-mmlab/mmclassification')
result = list_package()
print(result)
assert target in result