From 5ad764cc18455b635da8c3b05841e78f7a4401af Mon Sep 17 00:00:00 2001 From: wjf5203 Date: Fri, 26 Jul 2024 18:04:17 +0800 Subject: [PATCH] update track loss for video finetune --- assets/TEST.md | 6 ++-- assets/TRAIN.md | 28 +++++++++++++++++++ .../{video => videos}/Lite/BURST_Lite.yaml | 0 .../configs/{video => videos}/Lite/LVVIS.yaml | 0 .../{video => videos}/Lite/TAO_Lite.yaml | 0 .../{video => videos}/Lite/ovis_base.yaml | 0 .../{video => videos}/Lite/ytvis19_base.yaml | 0 .../{video => videos}/Plus/BURST_Plus.yaml | 0 .../{video => videos}/Plus/TAO_Plus.yaml | 0 .../{video => videos}/Plus/ovis_Plus.yaml | 0 .../{video => videos}/Plus/ytvis19_Plus.yaml | 0 11 files changed, 31 insertions(+), 3 deletions(-) rename projects/GLEE/configs/{video => videos}/Lite/BURST_Lite.yaml (100%) rename projects/GLEE/configs/{video => videos}/Lite/LVVIS.yaml (100%) rename projects/GLEE/configs/{video => videos}/Lite/TAO_Lite.yaml (100%) rename projects/GLEE/configs/{video => videos}/Lite/ovis_base.yaml (100%) rename projects/GLEE/configs/{video => videos}/Lite/ytvis19_base.yaml (100%) rename projects/GLEE/configs/{video => videos}/Plus/BURST_Plus.yaml (100%) rename projects/GLEE/configs/{video => videos}/Plus/TAO_Plus.yaml (100%) rename projects/GLEE/configs/{video => videos}/Plus/ovis_Plus.yaml (100%) rename projects/GLEE/configs/{video => videos}/Plus/ytvis19_Plus.yaml (100%) diff --git a/assets/TEST.md b/assets/TEST.md index 2b585bc..cbd189f 100644 --- a/assets/TEST.md +++ b/assets/TEST.md @@ -1,4 +1,4 @@ -# Tutorial for Testing (Continuously Updated) +# Tutorial for Testing (Updating) GLEE can be directly tested on classic detection and segmentation datasets locally. For some video datasets, the results need to be submitted to Codalab for evaluation. Additionally, certain datasets such as TAO, BURST, and OmniLabel require evaluation using additional tools. We will continue to update the evaluation tutorials for all datasets reported in the paper here. @@ -49,7 +49,7 @@ python3 projects/GLEE/train_net.py --config-file projects/GLEE/configs/images/Li -# Video Tasks (Continuously Updated) +# Video Tasks ### Youtube-VIS, OVIS @@ -173,5 +173,5 @@ ${GLEE_ROOT} -# Omnilabel and ODinW (Continuously Updated) +# Omnilabel and ODinW (Updating) diff --git a/assets/TRAIN.md b/assets/TRAIN.md index 02df32e..700c31f 100644 --- a/assets/TRAIN.md +++ b/assets/TRAIN.md @@ -68,6 +68,15 @@ ${GLEE_ROOT} -- Stage1_pretrain_openimage_obj365_CLIPfrozen_EVA02L_LSJ1536.yaml -- Stage2_joint_training_CLIPteacher_EVA02L.yaml -- Stage3_scaleup_CLIPteacher_EVA02L.yaml + -- videos + -- Lite + -- BURST_Lite.yaml + -- LVVIS.yaml + -- TAO_Lite.yaml + -- ytvis19_base.yaml + -- ovis_base.yaml + -- Plus + ... ``` @@ -82,8 +91,27 @@ Here, `` should be replaced with the number of machines you intend +# Video Tasks +The finetune and inference codes and scripts for the video tasksd (VIS、TAO、LVVIS、BURST) have been updated. +For finetune GLEE on VIS tasks (especially OVIS) to get better performance, you can use + +```bash +# YouTube-VIS finetune from GLEE stage2 weights +# Lite +python3 projects/GLEE/train_net.py --config-file projects/GLEE/configs/videos/Lite/ytvis19_base.yaml --num-gpus 8 MODEL.WEIGHTS /path/to/model_zoo/GLEE_Lite_joint.pth +# Plus +python3 projects/GLEE/train_net.py --config-file projects/GLEE/configs/videos/Plus/ytvis19_Plus.yaml --num-gpus 8 MODEL.WEIGHTS /path/to/model_zoo/GLEE_Plus_joint.pth + +# OVIS finetune +# Lite +python3 projects/GLEE/train_net.py --config-file projects/GLEE/configs/videos/Lite/ovis_base.yaml --num-gpus 8 MODEL.WEIGHTS /path/to/model_zoo/GLEE_Lite_joint.pth +# Plus +python3 projects/GLEE/train_net.py --config-file projects/GLEE/configs/videos/Plus/ovis_Plus.yaml --num-gpus 8 MODEL.WEIGHTS /path/to/model_zoo/GLEE_Plus_joint.pth +``` + +for evaluation of these video tasks, please refer to [TEST.md](TEST.md). # Finetune (Continuously Updated) diff --git a/projects/GLEE/configs/video/Lite/BURST_Lite.yaml b/projects/GLEE/configs/videos/Lite/BURST_Lite.yaml similarity index 100% rename from projects/GLEE/configs/video/Lite/BURST_Lite.yaml rename to projects/GLEE/configs/videos/Lite/BURST_Lite.yaml diff --git a/projects/GLEE/configs/video/Lite/LVVIS.yaml b/projects/GLEE/configs/videos/Lite/LVVIS.yaml similarity index 100% rename from projects/GLEE/configs/video/Lite/LVVIS.yaml rename to projects/GLEE/configs/videos/Lite/LVVIS.yaml diff --git a/projects/GLEE/configs/video/Lite/TAO_Lite.yaml b/projects/GLEE/configs/videos/Lite/TAO_Lite.yaml similarity index 100% rename from projects/GLEE/configs/video/Lite/TAO_Lite.yaml rename to projects/GLEE/configs/videos/Lite/TAO_Lite.yaml diff --git a/projects/GLEE/configs/video/Lite/ovis_base.yaml b/projects/GLEE/configs/videos/Lite/ovis_base.yaml similarity index 100% rename from projects/GLEE/configs/video/Lite/ovis_base.yaml rename to projects/GLEE/configs/videos/Lite/ovis_base.yaml diff --git a/projects/GLEE/configs/video/Lite/ytvis19_base.yaml b/projects/GLEE/configs/videos/Lite/ytvis19_base.yaml similarity index 100% rename from projects/GLEE/configs/video/Lite/ytvis19_base.yaml rename to projects/GLEE/configs/videos/Lite/ytvis19_base.yaml diff --git a/projects/GLEE/configs/video/Plus/BURST_Plus.yaml b/projects/GLEE/configs/videos/Plus/BURST_Plus.yaml similarity index 100% rename from projects/GLEE/configs/video/Plus/BURST_Plus.yaml rename to projects/GLEE/configs/videos/Plus/BURST_Plus.yaml diff --git a/projects/GLEE/configs/video/Plus/TAO_Plus.yaml b/projects/GLEE/configs/videos/Plus/TAO_Plus.yaml similarity index 100% rename from projects/GLEE/configs/video/Plus/TAO_Plus.yaml rename to projects/GLEE/configs/videos/Plus/TAO_Plus.yaml diff --git a/projects/GLEE/configs/video/Plus/ovis_Plus.yaml b/projects/GLEE/configs/videos/Plus/ovis_Plus.yaml similarity index 100% rename from projects/GLEE/configs/video/Plus/ovis_Plus.yaml rename to projects/GLEE/configs/videos/Plus/ovis_Plus.yaml diff --git a/projects/GLEE/configs/video/Plus/ytvis19_Plus.yaml b/projects/GLEE/configs/videos/Plus/ytvis19_Plus.yaml similarity index 100% rename from projects/GLEE/configs/video/Plus/ytvis19_Plus.yaml rename to projects/GLEE/configs/videos/Plus/ytvis19_Plus.yaml