From 9ddef4f7d910831bd0b3cfe42379e2bca23ce9f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Wangbo=20Zhao=28=E9=BB=91=E8=89=B2=E6=9E=B7=E9=94=81=29?=
 <56866854+wangbo-zhao@users.noreply.github.com>
Date: Mon, 29 Aug 2022 14:31:06 +0800
Subject: [PATCH] [Refactor] Modify documents of classification, detection and
 segmentation (#426)

* modify config files

* modify benchmark docs

* Update classification.md

* modify benchmark docs

* modify benchmark docs

* modify benchmark docs

* modify benchmark docs

* modify benchmark docs

* modify doc

* modify benchmark docs

* modify benchmark docs

* modify benchmark docs

* modify benchmark docs

* modify benchmark docs

* modify benchmark doc
---
 docs/en/user_guides/classification.md         | 66 ++++++++++++++-----
 docs/en/user_guides/detection.md              | 58 ++++++++++++++--
 docs/en/user_guides/segmentation.md           | 46 ++++++++++++-
 .../classification/mim_dist_test.sh           | 17 +++++
 .../classification/mim_slurm_test.sh          | 23 +++++++
 tools/benchmarks/mmdetection/mim_dist_test.sh | 20 ++++++
 .../benchmarks/mmdetection/mim_slurm_test.sh  | 27 ++++++++
 .../mmsegmentation/mim_dist_test.sh           | 20 ++++++
 .../mmsegmentation/mim_slurm_test.sh          | 27 ++++++++
 9 files changed, 278 insertions(+), 26 deletions(-)
 create mode 100644 tools/benchmarks/classification/mim_dist_test.sh
 create mode 100644 tools/benchmarks/classification/mim_slurm_test.sh
 create mode 100644 tools/benchmarks/mmdetection/mim_dist_test.sh
 create mode 100644 tools/benchmarks/mmdetection/mim_slurm_test.sh
 create mode 100644 tools/benchmarks/mmsegmentation/mim_dist_test.sh
 create mode 100644 tools/benchmarks/mmsegmentation/mim_slurm_test.sh

diff --git a/docs/en/user_guides/classification.md b/docs/en/user_guides/classification.md
index 457994e2..65f55238 100644
--- a/docs/en/user_guides/classification.md
+++ b/docs/en/user_guides/classification.md
@@ -1,20 +1,19 @@
 # Classification
 
-In MMSelfSup, we provide many benchmarks, thus the models can be evaluated on different downstream tasks. Here are comprehensive tutorials and examples to explain how to run all benchmarks with MMSelfSup.
-
 - [Classification](#classification)
   - [VOC SVM / Low-shot SVM](#voc-svm--low-shot-svm)
-  - [Linear Evaluation](#linear-evaluation)
+  - [Linear Evaluation and Fine-tuning](#linear-evaluation-and-fine-tuning)
   - [ImageNet Semi-Supervised Classification](#imagenet-semi-supervised-classification)
   - [ImageNet Nearest-Neighbor Classification](#imagenet-nearest-neighbor-classification)
 
-As for classification, we provide scripts in folder `tools/benchmarks/classification/`, which has 4 `.sh` files, 1 folder for VOC SVM related classification task and 1 folder for ImageNet nearest-neighbor classification task.
+In MMSelfSup, we provide many benchmarks for classification, thus the models can be evaluated on different classification tasks. Here are comprehensive tutorials and examples to explain how to run all classification benchmarks with MMSelfSup.
+We provide scripts in folder `tools/benchmarks/classification/`, which has 2 `.sh` files, 1 folder for VOC SVM related classification task and 1 folder for ImageNet nearest-neighbor classification task.
 
 ## VOC SVM / Low-shot SVM
 
-To run these benchmarks, you should first prepare your VOC datasets. Please refer to [prepare_data.md](https://github.com/open-mmlab/mmselfsup/blob/master/docs/en/prepare_data.md) for the details of data preparation.
+To run these benchmarks, you should first prepare your VOC datasets. Please refer to [prepare_data.md](./2_dataset_prepare.md) for the details of data preparation.
 
-To evaluate the pre-trained models, you can run command below.
+To evaluate the pre-trained models, you can run the command below.
 
 ```shell
 # distributed version
@@ -24,7 +23,7 @@ bash tools/benchmarks/classification/svm_voc07/dist_test_svm_pretrain.sh ${SELFS
 bash tools/benchmarks/classification/svm_voc07/slurm_test_svm_pretrain.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${PRETRAIN} ${FEATURE_LIST}
 ```
 
-Besides, if you want to evaluate the ckpt files saved by runner, you can run command below.
+Besides, if you want to evaluate the ckpt files saved by runner, you can run the command below.
 
 ```shell
 # distributed version
@@ -44,16 +43,27 @@ Remarks:
 - if you want to change GPU numbers, you could add `GPUS_PER_NODE=4 GPUS=4` at the beginning of the command.
 - `EPOCH` is the epoch number of the ckpt that you want to test
 
-## Linear Evaluation
+## Linear Evaluation and Fine-tuning
 
-The linear evaluation is one of the most general benchmarks, we integrate several papers' config settings, also including multi-head linear evaluation. We write classification model in our own codebase for the multi-head function, thus, to run linear evaluation, we still use `.sh` script to launch training. The supported datasets are **ImageNet**, **Places205** and **iNaturalist18**.
+Linear evaluation and fine-tuning are two of the most general benchmarks. We provide config files and scripts to launch the training and testing
+for Linear Evaluation and Fine-tuning. The supported datasets are **ImageNet**, **Places205** and **iNaturalist18**.
+
+First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab.
+
+```shell
+pip install openmim
+```
+
+Besides, please refer to MMClassification for [installation](https://github.com/open-mmlab/mmclassification/blob/dev-1.x/docs/en/install.md) and [data preparation](https://github.com/open-mmlab/mmclassification/blob/dev-1.x/docs/en/getting_started.md).
+
+Then, run the command below.
 
 ```shell
 # distributed version
-bash tools/benchmarks/classification/dist_train_linear.sh ${CONFIG} ${PRETRAIN}
+bash tools/benchmarks/classification/mim_dist_train.sh ${CONFIG} ${PRETRAIN}
 
 # slurm version
-bash tools/benchmarks/classification/slurm_train_linear.sh ${PARTITION} ${JOB_NAME} ${CONFIG} ${PRETRAIN}
+bash tools/benchmarks/classification/mim_slurm_train.sh ${PARTITION} ${JOB_NAME} ${CONFIG} ${PRETRAIN}
 ```
 
 Remarks:
@@ -62,27 +72,49 @@ Remarks:
 - `CONFIG`: Use config files under `configs/benchmarks/classification/`. Specifically, `imagenet` (excluding `imagenet_*percent` folders), `places205` and `inaturalist2018`.
 - `PRETRAIN`: the pre-trained model file.
 
-## ImageNet Semi-Supervised Classification
+Example:
 
-To run ImageNet semi-supervised classification, we still use `.sh` script to launch training.
+```shell
+bash ./tools/benchmarks/classification/mim_dist_train.sh \
+configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py \
+work_dir/pretrained_model.pth
+```
+
+If you want to test the well-trained model, please run the command below.
 
 ```shell
 # distributed version
-bash tools/benchmarks/classification/dist_train_semi.sh ${CONFIG} ${PRETRAIN}
+bash tools/benchmarks/classification/mim_dist_test.sh ${CONFIG} ${CHECKPOINT}
 
 # slurm version
-bash tools/benchmarks/classification/slurm_train_semi.sh ${PARTITION} ${JOB_NAME} ${CONFIG} ${PRETRAIN}
+bash tools/benchmarks/classification//mim_slurm_test.sh ${PARTITION} ${CONFIG} ${CHECKPOINT}
 ```
 
 Remarks:
 
+- `CHECKPOINT`: The well-trained classification model that you want to test.
+
+Example:
+
+```shell
+bash ./tools/benchmarks/mmsegmentation/mim_dist_test.sh \
+configs/benchmarks/classification/imagenet/resnet50_linear-8xb32-coslr-100e_in1k.py \
+work_dir/model.pth
+```
+
+## ImageNet Semi-Supervised Classification
+
+To run ImageNet semi-supervised classification, we still use the same `.sh` script as Linear Evaluation and Fine-tuning to launch training.
+
+Remarks:
+
 - The default GPU number is 4.
 - `CONFIG`: Use config files under `configs/benchmarks/classification/imagenet/`, named `imagenet_*percent` folders.
 - `PRETRAIN`: the pre-trained model file.
 
 ## ImageNet Nearest-Neighbor Classification
 
-To evaluate the pre-trained models using the nearest-neighbor benchmark, you can run command below.
+To evaluate the pre-trained models using the nearest-neighbor benchmark, you can run the command below.
 
 ```shell
 # distributed version
@@ -92,7 +124,7 @@ bash tools/benchmarks/classification/knn_imagenet/dist_test_knn_pretrain.sh ${SE
 bash tools/benchmarks/classification/knn_imagenet/slurm_test_knn_pretrain.sh ${PARTITION} ${JOB_NAME} ${SELFSUP_CONFIG} ${PRETRAIN}
 ```
 
-Besides, if you want to evaluate the ckpt files saved by runner, you can run command below.
+Besides, if you want to evaluate the ckpt files saved by runner, you can run the command below.
 
 ```shell
 # distributed version
diff --git a/docs/en/user_guides/detection.md b/docs/en/user_guides/detection.md
index 10d869e6..684ee8f9 100644
--- a/docs/en/user_guides/detection.md
+++ b/docs/en/user_guides/detection.md
@@ -2,6 +2,7 @@
 
 - [Detection](#detection)
   - [Train](#train)
+  - [Test](#test)
 
 Here, we prefer to use MMDetection to do the detection task. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab.
 
@@ -15,27 +16,70 @@ Besides, please refer to MMDet for [installation](https://github.com/open-mmlab/
 
 ## Train
 
-After installation, you can run MMDet with simple command.
+After installation, you can run MMDetection with simple command.
 
 ```shell
 # distributed version
-bash tools/benchmarks/mmdetection/mim_dist_train.sh ${CONFIG} ${PRETRAIN} ${GPUS}
+bash tools/benchmarks/mmdetection/mim_dist_train_c4.sh ${CONFIG} ${PRETRAIN} ${GPUS}
+bash tools/benchmarks/mmdetection/mim_dist_train_fpn.sh ${CONFIG} ${PRETRAIN} ${GPUS}
 
 # slurm version
-bash tools/benchmarks/mmdetection/mim_slurm_train.sh ${PARTITION} ${CONFIG} ${PRETRAIN}
+bash tools/benchmarks/mmdetection/mim_slurm_train_c4.sh ${PARTITION} ${CONFIG} ${PRETRAIN}
+bash tools/benchmarks/mmdetection/mim_slurm_train_fpn.sh ${PARTITION} ${CONFIG} ${PRETRAIN}
 ```
 
 Remarks:
 
-- `CONFIG`: Use config files under `configs/benchmarks/mmdetection/` or write your own config files
-- `PRETRAIN`: the pre-trained model file.
+- `CONFIG`: Use config files under `configs/benchmarks/mmdetection/`. Since repositories of OpenMMLab have support referring config files across different
+  repositories, we can easily leverage the configs from MMDetection like:
 
-Or if you want to do detection task with [detectron2](https://github.com/facebookresearch/detectron2), we also provides some config files.
+```shell
+_base_ = 'mmdet::mask_rcnn/mask_rcnn_r50_caffe_c4_1x_coco.py'
+```
+
+Writing your config files from scratch is also supported.
+
+- `PRETRAIN`: the pre-trained model file.
+- `GPUS`: The number of GPUs that you want to use to train. We adopt 8 GPUs for detection tasks by default.
+
+Example:
+
+```shell
+bash ./tools/benchmarks/mmdetection/mim_dist_train_c4.sh \
+configs/benchmarks/mmdetection/coco/mask_rcnn_r50_c4_mstrain_1x_coco.py \
+work_dir/byol_resnet50_8xb32-accum16-coslr-200e_in1k_20220225-5c8b2c2e.pth 8
+```
+
+Or if you want to do detection task with [detectron2](https://github.com/facebookresearch/detectron2), we also provide some config files.
 Please refer to [INSTALL.md](https://github.com/facebookresearch/detectron2/blob/main/INSTALL.md) for installation and follow the [directory structure](https://github.com/facebookresearch/detectron2/tree/main/datasets) to prepare your datasets required by detectron2.
 
 ```shell
 conda activate detectron2 # use detectron2 environment here, otherwise use open-mmlab environment
-cd benchmarks/detection
+cd tools/benchmarks/detectron2
 python convert-pretrain-to-detectron2.py ${WEIGHT_FILE} ${OUTPUT_FILE} # must use .pkl as the output extension.
 bash run.sh ${DET_CFG} ${OUTPUT_FILE}
 ```
+
+## Test
+
+After training, you can also run the command below to test your model.
+
+```shell
+# distributed version
+bash tools/benchmarks/mmdetection/mim_dist_test.sh ${CONFIG} ${CHECKPOINT} ${GPUS}
+
+# slurm version
+bash tools/benchmarks/mmdetection/mim_slurm_test.sh ${PARTITION} ${CONFIG} ${CHECKPOINT}
+```
+
+Remarks:
+
+- `CHECKPOINT`: The well-trained detection model that you want to test.
+
+Example:
+
+```shell
+bash ./tools/benchmarks/mmdetection/mim_dist_test.sh \
+configs/benchmarks/mmdetection/coco/mask_rcnn_r50_fpn_mstrain_1x_coco.py \
+work_dir/epoch_12.pth 8
+```
diff --git a/docs/en/user_guides/segmentation.md b/docs/en/user_guides/segmentation.md
index 1e96acb1..9fae7c85 100644
--- a/docs/en/user_guides/segmentation.md
+++ b/docs/en/user_guides/segmentation.md
@@ -2,6 +2,7 @@
 
 - [Segmentation](#segmentation)
   - [Train](#train)
+  - [Test](#test)
 
 For semantic segmentation task, we use MMSegmentation. First, make sure you have installed [MIM](https://github.com/open-mmlab/mim), which is also a project of OpenMMLab.
 
@@ -11,7 +12,7 @@ pip install openmim
 
 It is very easy to install the package.
 
-Besides, please refer to MMSeg for [installation](https://github.com/open-mmlab/mmsegmentation/blob/master/docs/get_started.md) and [data preparation](https://github.com/open-mmlab/mmsegmentation/blob/master/docs/dataset_prepare.md#prepare-datasets).
+Besides, please refer to MMSegmentation for [installation](https://github.com/open-mmlab/mmsegmentation/blob/dev-1.x/docs/en/get_started.md) and [data preparation](https://github.com/open-mmlab/mmsegmentation/blob/dev-1.x/docs/en/user_guides/2_dataset_prepare.md).
 
 ## Train
 
@@ -27,5 +28,46 @@ bash tools/benchmarks/mmsegmentation/mim_slurm_train.sh ${PARTITION} ${CONFIG} $
 
 Remarks:
 
-- `CONFIG`: Use config files under `configs/benchmarks/mmsegmentation/` or write your own config files
+- `CONFIG`: Use config files under `configs/benchmarks/mmsegmentation/`. Since repositories of OpenMMLab have support referring config files across different
+  repositories, we can easily leverage the configs from MMSegmentation like:
+
+```shell
+_base_ = 'mmseg::fcn/fcn_r50-d8_769x769_40k_cityscapes.py'
+```
+
+Writing your config files from scratch is also supported.
+
 - `PRETRAIN`: the pre-trained model file.
+- `GPUS`: The number of GPUs that you want to use to train. We adopt 4 GPUs for segmentation tasks by default.
+
+Example:
+
+```shell
+bash ./tools/benchmarks/mmsegmentation/mim_dist_train.sh \
+configs/benchmarks/mmsegmentation/voc12aug/fcn_r50-d8_512x512_20k_voc12aug.py \
+work_dir/byol_resnet50_8xb32-accum16-coslr-200e_in1k_20220225-5c8b2c2e.pth 4
+```
+
+## Test
+
+After training, you can also run the command below to test your model.
+
+```shell
+# distributed version
+bash tools/benchmarks/mmsegmentation/mim_dist_test.sh ${CONFIG} ${CHECKPOINT} ${GPUS}
+
+# slurm version
+bash tools/benchmarks/mmsegmentation/mim_slurm_test.sh ${PARTITION} ${CONFIG} ${CHECKPOINT}
+```
+
+Remarks:
+
+- `CHECKPOINT`: The well-trained segmentation model that you want to test.
+
+Example:
+
+```shell
+bash ./tools/benchmarks/mmsegmentation/mim_dist_test.sh \
+configs/benchmarks/mmsegmentation/voc12aug/fcn_r50-d8_512x512_20k_voc12aug.py \
+work_dir/iter_20000.pth 4
+```
diff --git a/tools/benchmarks/classification/mim_dist_test.sh b/tools/benchmarks/classification/mim_dist_test.sh
new file mode 100644
index 00000000..aed4602c
--- /dev/null
+++ b/tools/benchmarks/classification/mim_dist_test.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -x
+
+CFG=$1
+CHECKPOINT=$2
+GPUS=${GPUS:-8}
+PY_ARGS=${@:3}
+
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+mim test mmcls \
+    $CFG \
+    --checkpoint $CHECKPOINT \
+    --launcher pytorch \
+    -G $GPUS \
+    $PY_ARGS
diff --git a/tools/benchmarks/classification/mim_slurm_test.sh b/tools/benchmarks/classification/mim_slurm_test.sh
new file mode 100644
index 00000000..18ca6c7e
--- /dev/null
+++ b/tools/benchmarks/classification/mim_slurm_test.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -x
+
+PARTITION=$1
+CFG=$2  # use cfgs under "configs/benchmarks/classification/imagenet/*.py"
+CHECKPOINT=$3
+GPUS=${GPUS:-8}
+GPUS_PER_NODE=${GPUS_PER_NODE:-8}
+CPUS_PER_TASK=${CPUS_PER_TASK:-5}
+SRUN_ARGS=${SRUN_ARGS:-""}
+PY_ARGS=${@:4}
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+mim test mmcls \
+    $CFG \
+    --checkpoint $CHECKPOINT \
+    --launcher slurm -G $GPUS \
+    --gpus-per-node $GPUS_PER_NODE \
+    --cpus-per-task $CPUS_PER_TASK \
+    --partition $PARTITION \
+    --srun-args "$SRUN_ARGS" \
+    $PY_ARGS
diff --git a/tools/benchmarks/mmdetection/mim_dist_test.sh b/tools/benchmarks/mmdetection/mim_dist_test.sh
new file mode 100644
index 00000000..108ebe64
--- /dev/null
+++ b/tools/benchmarks/mmdetection/mim_dist_test.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -x
+
+CFG=$1
+CHECKPOINT=$2
+GPUS=$3
+PY_ARGS=${@:4}
+
+# set work_dir according to config path and pretrained model to distinguish different models
+WORK_DIR="$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/$(echo $PRETRAIN | rev | cut -d/ -f 1 | rev)"
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+mim test mmdet \
+    $CFG \
+    --checkpoint $CHECKPOINT \
+    --launcher pytorch \
+    -G $GPUS \
+    --work-dir $WORK_DIR \
+    $PY_ARGS
diff --git a/tools/benchmarks/mmdetection/mim_slurm_test.sh b/tools/benchmarks/mmdetection/mim_slurm_test.sh
new file mode 100644
index 00000000..10cec8be
--- /dev/null
+++ b/tools/benchmarks/mmdetection/mim_slurm_test.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -x
+
+PARTITION=$1
+CFG=$2
+CHECKPOINT=$3
+GPUS=${GPUS:-8}
+GPUS_PER_NODE=${GPUS_PER_NODE:-8}
+CPUS_PER_TASK=${CPUS_PER_TASK:-5}
+SRUN_ARGS=${SRUN_ARGS:-""}
+PY_ARGS=${@:4}
+
+# set work_dir according to config path and pretrained model to distinguish different models
+WORK_DIR="$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/$(echo $PRETRAIN | rev | cut -d/ -f 1 | rev)"
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+mim test mmdet \
+    $CFG \
+    --checkpoint $CHECKPOINT \
+    --launcher slurm -G $GPUS \
+    --gpus-per-node $GPUS_PER_NODE \
+    --cpus-per-task $CPUS_PER_TASK \
+    --partition $PARTITION \
+    --work-dir $WORK_DIR \
+    --srun-args "$SRUN_ARGS" \
+    $PY_ARGS
diff --git a/tools/benchmarks/mmsegmentation/mim_dist_test.sh b/tools/benchmarks/mmsegmentation/mim_dist_test.sh
new file mode 100644
index 00000000..22464a43
--- /dev/null
+++ b/tools/benchmarks/mmsegmentation/mim_dist_test.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -x
+
+CFG=$1
+CHECKPOINT=$2
+GPUS=$3
+PY_ARGS=${@:4}
+
+# set work_dir according to config path and pretrained model to distinguish different models
+WORK_DIR="$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/$(echo $PRETRAIN | rev | cut -d/ -f 1 | rev)"
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+mim test mmseg \
+    $CFG \
+    --checkpoint $CHECKPOINT \
+    --launcher pytorch \
+    -G $GPUS \
+    --work-dir $WORK_DIR \
+    $PY_ARGS
diff --git a/tools/benchmarks/mmsegmentation/mim_slurm_test.sh b/tools/benchmarks/mmsegmentation/mim_slurm_test.sh
new file mode 100644
index 00000000..deb25c68
--- /dev/null
+++ b/tools/benchmarks/mmsegmentation/mim_slurm_test.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -x
+
+PARTITION=$1
+CFG=$2
+CHECKPOINT=$3
+GPUS=${GPUS:-4}
+GPUS_PER_NODE=${GPUS_PER_NODE:-4}
+CPUS_PER_TASK=${CPUS_PER_TASK:-5}
+SRUN_ARGS=${SRUN_ARGS:-""}
+PY_ARGS=${@:4}
+
+# set work_dir according to config path and pretrained model to distinguish different models
+WORK_DIR="$(echo ${CFG%.*} | sed -e "s/configs/work_dirs/g")/$(echo $PRETRAIN | rev | cut -d/ -f 1 | rev)"
+
+PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \
+mim test mmseg \
+    $CFG \
+    --checkpoint $CHECKPOINT \
+    --launcher slurm -G $GPUS \
+    --gpus-per-node $GPUS_PER_NODE \
+    --cpus-per-task $CPUS_PER_TASK \
+    --partition $PARTITION \
+    --work-dir $WORK_DIR \
+    --srun-args "$SRUN_ARGS" \
+    $PY_ARGS