[Feature] add log collector (#1175)
* [Feature] add log collector * Update .dev/log_collector/readme.md Co-authored-by: Miao Zheng <76149310+MeowZheng@users.noreply.github.com> * Update .dev/log_collector/example_config.py Co-authored-by: Miao Zheng <76149310+MeowZheng@users.noreply.github.com> * fix typo and so on * modify readme * fix some bugs and revise the readme.md * more elegant * Update .dev/log_collector/readme.md Co-authored-by: Junjun2016 <hejunjun@sjtu.edu.cn> Co-authored-by: Miao Zheng <76149310+MeowZheng@users.noreply.github.com> Co-authored-by: Junjun2016 <hejunjun@sjtu.edu.cn>pull/1801/head
Rockey
GitHub
parent
2f4d52f4d1
commit
ee5fbcff74
|
|
@ -0,0 +1,18 @@
|
|||
work_dir = '../../work_dirs'
|
||||
metric = 'mIoU'
|
||||
|
||||
# specify the log files we would like to collect in `log_items`
|
||||
log_items = [
|
||||
'segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup',
|
||||
'segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr',
|
||||
'segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr',
|
||||
'segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr'
|
||||
]
|
||||
# or specify ignore_keywords, then the folders whose name contain
|
||||
# `'segformer'` won't be collected
|
||||
# ignore_keywords = ['segformer']
|
||||
|
||||
# should not include metric
|
||||
other_info_keys = ['mAcc']
|
||||
markdown_file = 'markdowns/lr_in_trans.json.md'
|
||||
json_file = 'jsons/trans_in_cnn.json'
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
# Copyright (c) OpenMMLab. All rights reserved.
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import os.path as osp
|
||||
from collections import OrderedDict
|
||||
|
||||
from utils import load_config
|
||||
|
||||
# automatically collect all the results
|
||||
|
||||
# The structure of the directory:
|
||||
# ├── work-dir
|
||||
# │ ├── config_1
|
||||
# │ │ ├── time1.log.json
|
||||
# │ │ ├── time2.log.json
|
||||
# │ │ ├── time3.log.json
|
||||
# │ │ ├── time4.log.json
|
||||
# │ ├── config_2
|
||||
# │ │ ├── time5.log.json
|
||||
# │ │ ├── time6.log.json
|
||||
# │ │ ├── time7.log.json
|
||||
# │ │ ├── time8.log.json
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='extract info from log.json')
|
||||
parser.add_argument('config_dir')
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def has_keyword(name: str, keywords: list):
|
||||
for a_keyword in keywords:
|
||||
if a_keyword in name:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
cfg = load_config(args.config_dir)
|
||||
work_dir = cfg['work_dir']
|
||||
metric = cfg['metric']
|
||||
log_items = cfg.get('log_items', [])
|
||||
ignore_keywords = cfg.get('ignore_keywords', [])
|
||||
other_info_keys = cfg.get('other_info_keys', [])
|
||||
markdown_file = cfg.get('markdown_file', None)
|
||||
json_file = cfg.get('json_file', None)
|
||||
|
||||
if json_file and osp.split(json_file)[0] != '':
|
||||
os.makedirs(osp.split(json_file)[0], exist_ok=True)
|
||||
if markdown_file and osp.split(markdown_file)[0] != '':
|
||||
os.makedirs(osp.split(markdown_file)[0], exist_ok=True)
|
||||
|
||||
assert not (log_items and ignore_keywords), \
|
||||
'log_items and ignore_keywords cannot be specified at the same time'
|
||||
assert metric not in other_info_keys, \
|
||||
'other_info_keys should not contain metric'
|
||||
|
||||
if ignore_keywords and isinstance(ignore_keywords, str):
|
||||
ignore_keywords = [ignore_keywords]
|
||||
if other_info_keys and isinstance(other_info_keys, str):
|
||||
other_info_keys = [other_info_keys]
|
||||
if log_items and isinstance(log_items, str):
|
||||
log_items = [log_items]
|
||||
|
||||
if not log_items:
|
||||
log_items = [
|
||||
item for item in sorted(os.listdir(work_dir))
|
||||
if not has_keyword(item, ignore_keywords)
|
||||
]
|
||||
|
||||
experiment_info_list = []
|
||||
for config_dir in log_items:
|
||||
preceding_path = os.path.join(work_dir, config_dir)
|
||||
log_list = [
|
||||
item for item in os.listdir(preceding_path)
|
||||
if item.endswith('.log.json')
|
||||
]
|
||||
log_list = sorted(
|
||||
log_list,
|
||||
key=lambda time_str: datetime.datetime.strptime(
|
||||
time_str, '%Y%m%d_%H%M%S.log.json'))
|
||||
val_list = []
|
||||
last_iter = 0
|
||||
for log_name in log_list:
|
||||
with open(os.path.join(preceding_path, log_name), 'r') as f:
|
||||
# ignore the info line
|
||||
f.readline()
|
||||
all_lines = f.readlines()
|
||||
val_list.extend([
|
||||
json.loads(line) for line in all_lines
|
||||
if json.loads(line)['mode'] == 'val'
|
||||
])
|
||||
for index in range(len(all_lines) - 1, -1, -1):
|
||||
line_dict = json.loads(all_lines[index])
|
||||
if line_dict['mode'] == 'train':
|
||||
last_iter = max(last_iter, line_dict['iter'])
|
||||
break
|
||||
|
||||
new_log_dict = dict(
|
||||
method=config_dir, metric_used=metric, last_iter=last_iter)
|
||||
for index, log in enumerate(val_list, 1):
|
||||
new_ordered_dict = OrderedDict()
|
||||
new_ordered_dict['eval_index'] = index
|
||||
new_ordered_dict[metric] = log[metric]
|
||||
for key in other_info_keys:
|
||||
if key in log:
|
||||
new_ordered_dict[key] = log[key]
|
||||
val_list[index - 1] = new_ordered_dict
|
||||
|
||||
assert len(val_list) >= 1, \
|
||||
f"work dir {config_dir} doesn't contain any evaluation."
|
||||
new_log_dict['last eval'] = val_list[-1]
|
||||
new_log_dict['best eval'] = max(val_list, key=lambda x: x[metric])
|
||||
experiment_info_list.append(new_log_dict)
|
||||
print(f'{config_dir} is processed')
|
||||
|
||||
if json_file:
|
||||
with open(json_file, 'w') as f:
|
||||
json.dump(experiment_info_list, f, indent=4)
|
||||
|
||||
if markdown_file:
|
||||
lines_to_write = []
|
||||
for index, log in enumerate(experiment_info_list, 1):
|
||||
lines_to_write.append(
|
||||
f"|{index}|{log['method']}|{log['best eval'][metric]}"
|
||||
f"|{log['best eval']['eval_index']}|"
|
||||
f"{log['last eval'][metric]}|"
|
||||
f"{log['last eval']['eval_index']}|{log['last_iter']}|\n")
|
||||
with open(markdown_file, 'w') as f:
|
||||
f.write(f'|exp_num|method|{metric} best|best index|'
|
||||
f'{metric} last|last index|last iter num|\n')
|
||||
f.write('|:---:|:---:|:---:|:---:|:---:|:---:|:---:|\n')
|
||||
f.writelines(lines_to_write)
|
||||
|
||||
print('processed successfully')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
# Log Collector
|
||||
|
||||
## Function
|
||||
|
||||
Automatically collect logs and write the result in a json file or markdown file.
|
||||
|
||||
If there are several `.log.json` files in one folder, Log Collector assumes that the `.log.json` files other than the first one are resume from the preceding `.log.json` file. Log Collector returns the result considering all `.log.json` files.
|
||||
|
||||
## Usage:
|
||||
|
||||
To use log collector, you need to write a config file to configure the log collector first.
|
||||
|
||||
For example:
|
||||
|
||||
example_config.py:
|
||||
|
||||
```python
|
||||
# The work directory that contains folders that contains .log.json files.
|
||||
work_dir = '../../work_dirs'
|
||||
# The metric used to find the best evaluation.
|
||||
metric = 'mIoU'
|
||||
|
||||
# **Don't specify the log_items and ignore_keywords at the same time.**
|
||||
# Specify the log files we would like to collect in `log_items`.
|
||||
# The folders specified should be the subdirectories of `work_dir`.
|
||||
log_items = [
|
||||
'segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup',
|
||||
'segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr',
|
||||
'segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr',
|
||||
'segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr'
|
||||
]
|
||||
# Or specify `ignore_keywords`. The folders whose name contain one
|
||||
# of the keywords in the `ignore_keywords` list(e.g., `'segformer'`)
|
||||
# won't be collected.
|
||||
# ignore_keywords = ['segformer']
|
||||
|
||||
# Other log items in .log.json that you want to collect.
|
||||
# should not include metric.
|
||||
other_info_keys = ["mAcc"]
|
||||
# The output markdown file's name.
|
||||
markdown_file ='markdowns/lr_in_trans.json.md'
|
||||
# The output json file's name. (optional)
|
||||
json_file = 'jsons/trans_in_cnn.json'
|
||||
```
|
||||
|
||||
The structure of the work-dir directory should be like：
|
||||
|
||||
```text
|
||||
├── work-dir
|
||||
│ ├── folder1
|
||||
│ │ ├── time1.log.json
|
||||
│ │ ├── time2.log.json
|
||||
│ │ ├── time3.log.json
|
||||
│ │ ├── time4.log.json
|
||||
│ ├── folder2
|
||||
│ │ ├── time5.log.json
|
||||
│ │ ├── time6.log.json
|
||||
│ │ ├── time7.log.json
|
||||
│ │ ├── time8.log.json
|
||||
```
|
||||
|
||||
Then , cd to the log collector folder.
|
||||
|
||||
Now you can run log_collector.py by using command:
|
||||
|
||||
```bash
|
||||
python log_collector.py ./example_config.py
|
||||
```
|
||||
|
||||
The output markdown file is like:
|
||||
|
||||
|exp_num|method|mIoU best|best index|mIoU last|last index|last iter num|
|
||||
|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
|
||||
|1|segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup|0.2776|10|0.2776|10|160000|
|
||||
|2|segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr|0.2802|10|0.2802|10|160000|
|
||||
|3|segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr|0.4943|11|0.4943|11|160000|
|
||||
|4|segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr|0.4883|11|0.4883|11|160000|
|
||||
|
||||
The output json file is like:
|
||||
```json
|
||||
[
|
||||
{
|
||||
"method": "segformer_mit-b5_512x512_160k_ade20k_cnn_lr_with_warmup",
|
||||
"metric_used": "mIoU",
|
||||
"last_iter": 160000,
|
||||
"last eval": {
|
||||
"eval_index": 10,
|
||||
"mIoU": 0.2776,
|
||||
"mAcc": 0.3779
|
||||
},
|
||||
"best eval": {
|
||||
"eval_index": 10,
|
||||
"mIoU": 0.2776,
|
||||
"mAcc": 0.3779
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "segformer_mit-b5_512x512_160k_ade20k_cnn_no_warmup_lr",
|
||||
"metric_used": "mIoU",
|
||||
"last_iter": 160000,
|
||||
"last eval": {
|
||||
"eval_index": 10,
|
||||
"mIoU": 0.2802,
|
||||
"mAcc": 0.3764
|
||||
},
|
||||
"best eval": {
|
||||
"eval_index": 10,
|
||||
"mIoU": 0.2802,
|
||||
"mAcc": 0.3764
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "segformer_mit-b5_512x512_160k_ade20k_mit_trans_lr",
|
||||
"metric_used": "mIoU",
|
||||
"last_iter": 160000,
|
||||
"last eval": {
|
||||
"eval_index": 11,
|
||||
"mIoU": 0.4943,
|
||||
"mAcc": 0.6097
|
||||
},
|
||||
"best eval": {
|
||||
"eval_index": 11,
|
||||
"mIoU": 0.4943,
|
||||
"mAcc": 0.6097
|
||||
}
|
||||
},
|
||||
{
|
||||
"method": "segformer_mit-b5_512x512_160k_ade20k_swin_trans_lr",
|
||||
"metric_used": "mIoU",
|
||||
"last_iter": 160000,
|
||||
"last eval": {
|
||||
"eval_index": 11,
|
||||
"mIoU": 0.4883,
|
||||
"mAcc": 0.6061
|
||||
},
|
||||
"best eval": {
|
||||
"eval_index": 11,
|
||||
"mIoU": 0.4883,
|
||||
"mAcc": 0.6061
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
# Copyright (c) OpenMMLab. All rights reserved.
|
||||
# modified from https://github.dev/open-mmlab/mmcv
|
||||
import os.path as osp
|
||||
import sys
|
||||
from importlib import import_module
|
||||
|
||||
|
||||
def load_config(cfg_dir: str) -> dict:
|
||||
assert cfg_dir.endswith('.py')
|
||||
root_path, file_name = osp.split(cfg_dir)
|
||||
temp_module = osp.splitext(file_name)[0]
|
||||
sys.path.insert(0, root_path)
|
||||
mod = import_module(temp_module)
|
||||
sys.path.pop(0)
|
||||
cfg_dict = {
|
||||
k: v
|
||||
for k, v in mod.__dict__.items() if not k.startswith('__')
|
||||
}
|
||||
del sys.modules[temp_module]
|
||||
return cfg_dict
|
||||
Loading…
Reference in New Issue