diff --git a/deploy/configs/PULC/traffic_sign/inference_traffic_sign.yaml b/deploy/configs/PULC/traffic_sign/inference_traffic_sign.yaml new file mode 100644 index 000000000..09c4521f2 --- /dev/null +++ b/deploy/configs/PULC/traffic_sign/inference_traffic_sign.yaml @@ -0,0 +1,35 @@ +Global: + infer_imgs: "./images/PULC/traffic_sign/99603_17806.jpg" + inference_model_dir: "./models/traffic_sign_infer" + batch_size: 1 + use_gpu: True + enable_mkldnn: True + cpu_num_threads: 10 + benchmark: False + use_fp16: False + ir_optim: True + use_tensorrt: False + gpu_mem: 8000 + enable_profile: False + +PreProcess: + transform_ops: + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 0.00392157 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + channel_num: 3 + - ToCHWImage: + +PostProcess: + main_indicator: Topk + Topk: + topk: 5 + class_id_map_file: "../dataset/traffic_sign/label_name_id.txt" + SavePreLabel: + save_dir: ./pre_label/ diff --git a/deploy/configs/PULC/vehicle_attr/inference_vehicle_attr.yaml b/deploy/configs/PULC/vehicle_attr/inference_vehicle_attr.yaml new file mode 100644 index 000000000..f47a73ab3 --- /dev/null +++ b/deploy/configs/PULC/vehicle_attr/inference_vehicle_attr.yaml @@ -0,0 +1,32 @@ +Global: + infer_imgs: "./images/PULC/vehicle_attr/0002_c002_00030670_0.jpg" + inference_model_dir: "./models/vehicle_attr_infer" + batch_size: 1 + use_gpu: True + enable_mkldnn: True + cpu_num_threads: 10 + benchmark: False + use_fp16: False + ir_optim: True + use_tensorrt: False + gpu_mem: 8000 + enable_profile: False + +PreProcess: + transform_ops: + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + channel_num: 3 + - ToCHWImage: + +PostProcess: + main_indicator: VehicleAttribute + VehicleAttribute: + color_threshold: 0.5 + type_threshold: 0.5 + diff --git a/deploy/images/PULC/traffic_sign/100999_83928.jpg b/deploy/images/PULC/traffic_sign/100999_83928.jpg new file mode 100644 index 000000000..6f32ed5ae Binary files /dev/null and b/deploy/images/PULC/traffic_sign/100999_83928.jpg differ diff --git a/deploy/images/PULC/traffic_sign/99603_17806.jpg b/deploy/images/PULC/traffic_sign/99603_17806.jpg new file mode 100644 index 000000000..c792fdf6e Binary files /dev/null and b/deploy/images/PULC/traffic_sign/99603_17806.jpg differ diff --git a/deploy/images/PULC/vehicle_attr/0002_c002_00030670_0.jpg b/deploy/images/PULC/vehicle_attr/0002_c002_00030670_0.jpg new file mode 100644 index 000000000..bb5de9fc6 Binary files /dev/null and b/deploy/images/PULC/vehicle_attr/0002_c002_00030670_0.jpg differ diff --git a/deploy/images/PULC/vehicle_attr/0014_c012_00040750_0.jpg b/deploy/images/PULC/vehicle_attr/0014_c012_00040750_0.jpg new file mode 100644 index 000000000..76207d43c Binary files /dev/null and b/deploy/images/PULC/vehicle_attr/0014_c012_00040750_0.jpg differ diff --git a/deploy/python/postprocess.py b/deploy/python/postprocess.py index 1107b8050..9fe15bea8 100644 --- a/deploy/python/postprocess.py +++ b/deploy/python/postprocess.py @@ -280,3 +280,45 @@ class Attribute(object): batch_res.append([label_res, pred_res]) return batch_res + + +class VehicleAttribute(object): + def __init__(self, color_threshold=0.5, type_threshold=0.5): + self.color_threshold = color_threshold + self.type_threshold = type_threshold + self.color_list = [ + "yellow", "orange", "green", "gray", "red", "blue", "white", + "golden", "brown", "black" + ] + self.type_list = [ + "sedan", "suv", "van", "hatchback", "mpv", "pickup", "bus", + "truck", "estate" + ] + + def __call__(self, batch_preds, file_names=None): + # postprocess output of predictor + batch_res = [] + for res in batch_preds: + res = res.tolist() + label_res = [] + color_idx = np.argmax(res[:10]) + type_idx = np.argmax(res[10:]) + if res[color_idx] >= self.color_threshold: + color_info = f"Color: ({self.color_list[color_idx]}, prob: {res[color_idx]})" + else: + color_info = "Color unknown" + + if res[type_idx + 10] >= self.type_threshold: + type_info = f"Type: ({self.type_list[type_idx]}, prob: {res[type_idx + 10]})" + else: + type_info = "Type unknown" + + label_res = f"{color_info}, {type_info}" + + threshold_list = [self.color_threshold + ] * 10 + [self.type_threshold] * 9 + pred_res = (np.array(res) > np.array(threshold_list) + ).astype(np.int8).tolist() + + batch_res.append([label_res, pred_res]) + return batch_res diff --git a/deploy/python/predict_cls.py b/deploy/python/predict_cls.py index 41b46090a..440624e0c 100644 --- a/deploy/python/predict_cls.py +++ b/deploy/python/predict_cls.py @@ -138,7 +138,9 @@ def main(config): continue batch_results = cls_predictor.predict(batch_imgs) for number, result_dict in enumerate(batch_results): - if "Attribute" in config["PostProcess"]: + if "Attribute" in config[ + "PostProcess"] or "VehicleAttribute" in config[ + "PostProcess"]: filename = batch_names[number] attr_message = result_dict[0] pred_res = result_dict[1] diff --git a/docs/images/PULC/docs/traffic_sign_data_demo.png b/docs/images/PULC/docs/traffic_sign_data_demo.png new file mode 100644 index 000000000..6fac97a29 Binary files /dev/null and b/docs/images/PULC/docs/traffic_sign_data_demo.png differ diff --git a/docs/images/PULC/docs/vehicle_attr_data_demo.png b/docs/images/PULC/docs/vehicle_attr_data_demo.png new file mode 100644 index 000000000..68c67acb3 Binary files /dev/null and b/docs/images/PULC/docs/vehicle_attr_data_demo.png differ diff --git a/docs/zh_CN/PULC/PULC_traffic_sign.md b/docs/zh_CN/PULC/PULC_traffic_sign.md new file mode 100644 index 000000000..342bd67f1 --- /dev/null +++ b/docs/zh_CN/PULC/PULC_traffic_sign.md @@ -0,0 +1,420 @@ +# PULC 交通标志分类模型 + +------ + + +## 目录 + +- [1. 模型和应用场景介绍](#1) +- [2. 模型快速体验](#2) +- [3. 模型训练、评估和预测](#3) + - [3.1 环境配置](#3.1) + - [3.2 数据准备](#3.2) + - [3.2.1 数据集来源](#3.2.1) + - [3.2.2 数据集获取](#3.2.2) + - [3.3 模型训练](#3.3) + - [3.4 模型评估](#3.4) + - [3.5 模型预测](#3.5) +- [4. 模型压缩](#4) + - [4.1 SKL-UGI 知识蒸馏](#4.1) + - [4.1.1 教师模型训练](#4.1.1) + - [4.1.2 蒸馏训练](#4.1.2) +- [5. 超参搜索](#5) +- [6. 模型推理部署](#6) + - [6.1 推理模型准备](#6.1) + - [6.1.1 基于训练得到的权重导出 inference 模型](#6.1.1) + - [6.1.2 直接下载 inference 模型](#6.1.2) + - [6.2 基于 Python 预测引擎推理](#6.2) + - [6.2.1 预测单张图像](#6.2.1) + - [6.2.2 基于文件夹的批量预测](#6.2.2) + - [6.3 基于 C++ 预测引擎推理](#6.3) + - [6.4 服务化部署](#6.4) + - [6.5 端侧部署](#6.5) + - [6.6 Paddle2ONNX 模型转换与预测](#6.6) + + + + +## 1. 模型和应用场景介绍 + +该案例提供了用户使用 PaddleClas 的超轻量图像分类方案（PULC，Practical Ultra Lightweight Classification）快速构建轻量级、高精度、可落地的交通标志分类模型。该模型可以广泛应用于自动驾驶、道路监控等场景。 + +下表列出了不同交通标志分类模型的相关指标，前两行展现了使用 SwinTranformer_tiny 和 MobileNetV3_large_x1_0 作为 backbone 训练得到的模型的相关指标，第三行至第六行依次展现了替换 backbone 为 PPLCNet_x1_0、使用 SSLD 预训练模型、使用 SSLD 预训练模型 + EDA 策略、使用 SSLD 预训练模型 + EDA 策略 + SKL-UGI 知识蒸馏策略训练得到的模型的相关指标。 + + +| 模型 | Top-1 Acc（%） | 延时（ms） | 存储（M） | 策略 | +|-------|-----------|----------|---------------|---------------| +| SwinTranformer_tiny | 98.11 | 87.19 | 111 | 使用ImageNet预训练模型 | +| MobileNetV3_large_x1_0 | 97.79 | 5.59 | 23 | 使用ImageNet预训练模型 | +| PPLCNet_x1_0 | 97.78 | 2.67 | 8.2 | 使用ImageNet预训练模型 | +| PPLCNet_x1_0 | 97.84 | 2.67 | 8.2 | 使用SSLD预训练模型 | +| PPLCNet_x1_0 | 98.14 | 2.67 | 8.2 | 使用SSLD预训练模型+EDA策略| +| PPLCNet_x1_0 | 98.35 | 2.67 | 8.2 | 使用SSLD预训练模型+EDA策略+SKL-UGI知识蒸馏策略| + +从表中可以看出，backbone 为 SwinTranformer_tiny 时精度较高，但是推理速度较慢。将 backbone 替换为轻量级模型 MobileNetV3_large_x1_0 后，速度可以大幅提升，但是精度下降明显。将 backbone 替换为 PPLCNet_x1_0 时，精度低0.01%，但是速度提升 2 倍左右。在此基础上，使用 SSLD 预训练模型后，在不改变推理速度的前提下，精度可以提升约 0.06%，进一步地，当融合EDA策略后，精度可以再提升 0.3%，最后，在使用 SKL-UGI 知识蒸馏后，精度可以继续提升 0.21%。此时，PPLCNet_x1_0 的精度超越了SwinTranformer_tiny，速度快32倍。关于 PULC 的训练方法和推理部署方法将在下面详细介绍。 + +**备注：** + +* 关于PPLCNet的介绍可以参考[PPLCNet介绍](../models/PP-LCNet.md)，相关论文可以查阅[PPLCNet paper](https://arxiv.org/abs/2109.15099)。 + + + + +## 2. 模型快速体验 + + （pip方式，待补充） + + + + +## 3. 模型训练、评估和预测 + + + +### 3.1 环境配置 + +* 安装：请先参考 [Paddle 安装教程](../installation/install_paddle.md) 以及 [PaddleClas 安装教程](../installation/install_paddleclas.md) 配置 PaddleClas 运行环境。 + + + +### 3.2 数据准备 + + + +#### 3.2.1 数据集来源 + +本案例中所使用的数据为[Tsinghua-Tencent 100K dataset (CC-BY-NC license)](https://cg.cs.tsinghua.edu.cn/traffic-sign/)，在使用的过程中，对交通标志检测框进行随机扩充与裁剪，从而得到用于训练与测试的图像，下面简称该数据集为`TT100K`数据集。 + + + +#### 3.2.2 数据集获取 + +在TT00K数据集上，对交通标志检测框进行随机扩充与裁剪，从而得到用于训练与测试的图像。随机扩充检测框的逻辑如下所示。 + +```python +def get_random_crop_box(xmin, ymin, xmax, ymax, img_height, img_width, ratio=1.0): + h = ymax - ymin + w = ymax - ymin + + xmin_diff = random.random() * ratio * min(w, xmin/ratio) + ymin_diff = random.random() * ratio * min(h, ymin/ratio) + xmax_diff = random.random() * ratio * min(w, (img_width-xmin-1)/ratio) + ymax_diff = random.random() * ratio * min(h, (img_height-ymin-1)/ratio) + + new_xmin = round(xmin - xmin_diff) + new_ymin = round(ymin - ymin_diff) + new_xmax = round(xmax + xmax_diff) + new_ymax = round(ymax + ymax_diff) + + return new_xmin, new_ymin, new_xmax, new_ymax +``` + +完整的预处理逻辑，可以参考下载好的数据集文件夹中的`deal.py`文件。 + + +处理后的数据集部分数据可视化如下。 + +
+ +
+ + +此处提供了经过上述方法处理好的数据，可以直接下载得到。 + +进入 PaddleClas 目录。 + +``` +cd path_to_PaddleClas +``` + +进入 `dataset/` 目录，下载并解压交通标志分类场景的数据。 + +```shell +cd dataset +wget https://paddleclas.bj.bcebos.com/data/cls_demo/traffic_sign.tar +tar -xf traffic_sign.tar +cd ../ +``` + +执行上述命令后，`dataset/`下存在`traffic_sign`目录，该目录中具有以下数据： + +``` +traffic_sign +├── train +│ ├── 0_62627.jpg +│ ├── 100000_89031.jpg +│ ├── 100001_89031.jpg +... +├── test +│ ├── 100423_2315.jpg +│ ├── 100424_2315.jpg +│ ├── 100425_2315.jpg +... +├── other +│ ├── 100603_3422.jpg +│ ├── 100604_3422.jpg +... +├── label_list_train.txt +├── label_list_test.txt +├── label_list_other.txt +├── label_list_train_for_distillation.txt +├── label_list_train.txt.debug +├── label_list_test.txt.debug +├── label_name_id.txt +├── deal.py +``` + +其中`train/`和`test/`分别为训练集和验证集。`label_list_train.txt`和`label_list_test.txt`分别为训练集和验证集的标签文件，`label_list_train.txt.debug`和`label_list_test.txt.debug`分别为训练集和验证集的`debug`标签文件，其分别是`label_list_train.txt`和`label_list_test.txt`的子集，用该文件可以快速体验本案例的流程。`train`与`other`的混合数据用于本案例的`SKL-UGI知识蒸馏策略`，对应的训练标签文件为`label_list_train_for_distillation.txt`。 + + +**备注：** + +* 关于 `label_list_train.txt`、`label_list_test.txt`的格式说明，可以参考[PaddleClas分类数据集格式说明](../data_preparation/classification_dataset.md#1-数据集格式说明) 。 + +* 关于如何得到蒸馏的标签文件可以参考[知识蒸馏标签获得方法](@ruoyu)。 + + + + +### 3.3 模型训练 + + +在 `ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml` 中提供了基于该场景的训练配置，可以通过如下脚本启动训练： + +```shell +export CUDA_VISIBLE_DEVICES=0,1,2,3 +python3 -m paddle.distributed.launch \ + --gpus="0,1,2,3" \ + tools/train.py \ + -c ./ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml +``` + +验证集的最佳指标在 `98.14%` 左右（数据集较小，一般有0.1%左右的波动）。 + + + + +### 3.4 模型评估 + +训练好模型之后，可以通过以下命令实现对模型指标的评估。 + +```bash +python3 tools/eval.py \ + -c ./ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml \ + -o Global.pretrained_model="output/PPLCNet_x1_0/best_model" +``` + +其中 `-o Global.pretrained_model="output/PPLCNet_x1_0/best_model"` 指定了当前最佳权重所在的路径，如果指定其他权重，只需替换对应的路径即可。 + + + +### 3.5 模型预测 + +模型训练完成之后，可以加载训练得到的预训练模型，进行模型预测。在模型库的 `tools/infer.py` 中提供了完整的示例，只需执行下述命令即可完成模型预测： + +```bash +python3 tools/infer.py \ + -c ./ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml \ + -o Global.pretrained_model=output/DistillationModel/best_model +``` + +输出结果如下： + +``` +99603_17806.jpg: class id(s): [216, 145, 49, 207, 169], score(s): [1.00, 0.00, 0.00, 0.00, 0.00], label_name(s): ['pm20', 'pm30', 'pm40', 'pl25', 'pm15'] +``` + +**备注：** + +* 这里`-o Global.pretrained_model="output/PPLCNet_x1_0/best_model"` 指定了当前最佳权重所在的路径，如果指定其他权重，只需替换对应的路径即可。 + +* 默认是对 `deploy/images/PULC/traffic_sign/99603_17806.jpg` 进行预测，此处也可以通过增加字段 `-o Infer.infer_imgs=xxx` 对其他图片预测。 + + + +## 4. 模型压缩 + + + +### 4.1 SKL-UGI 知识蒸馏 + +SKL-UGI 知识蒸馏是 PaddleClas 提出的一种简单有效的知识蒸馏方法，关于该方法的介绍，可以参考[SKL-UGI 知识蒸馏](@ruoyu)。 + + + +#### 4.1.1 教师模型训练 + +复用 `ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml` 中的超参数，训练教师模型，训练脚本如下： + +```shell +export CUDA_VISIBLE_DEVICES=0,1,2,3 +python3 -m paddle.distributed.launch \ + --gpus="0,1,2,3" \ + tools/train.py \ + -c ./ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml \ + -o Arch.name=ResNet101_vd +``` + +验证集的最佳指标为 `98.59%` 左右，当前教师模型最好的权重保存在 `output/ResNet101_vd/best_model.pdparams`。 + + + +#### 4.1.2 蒸馏训练 + +配置文件`ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_distillation.yaml`提供了`SKL-UGI知识蒸馏策略`的配置。该配置将`ResNet101_vd`当作教师模型，`PPLCNet_x1_0`当作学生模型，使用ImageNet数据集的验证集作为新增的无标签数据。训练脚本如下： + +```shell +export CUDA_VISIBLE_DEVICES=0,1,2,3 +python3 -m paddle.distributed.launch \ + --gpus="0,1,2,3" \ + tools/train.py \ + -c ./ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_distillation.yaml \ + -o Arch.models.0.Teacher.pretrained=output/ResNet101_vd/best_model +``` + +验证集的最佳指标为 `98.35%` 左右，当前模型最好的权重保存在 `output/DistillationModel/best_model_student.pdparams`。 + + + + +## 5. 超参搜索 + +在 [3.2 节](#3.2)和 [4.1 节](#4.1)所使用的超参数是根据 PaddleClas 提供的 `SHAS 超参数搜索策略` 搜索得到的，如果希望在自己的数据集上得到更好的结果，可以参考[SHAS 超参数搜索策略](#TODO)来获得更好的训练超参数。 + +**备注：** 此部分内容是可选内容，搜索过程需要较长的时间，您可以根据自己的硬件情况来选择执行。如果没有更换数据集，可以忽略此节内容。 + + + +## 6. 模型推理部署 + + + +### 6.1 推理模型准备 + +Paddle Inference 是飞桨的原生推理库， 作用于服务器端和云端，提供高性能的推理能力。相比于直接基于预训练模型进行预测，Paddle Inference可使用MKLDNN、CUDNN、TensorRT 进行预测加速，从而实现更优的推理性能。更多关于Paddle Inference推理引擎的介绍，可以参考[Paddle Inference官网教程](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/infer/inference/inference_cn.html)。 + +当使用 Paddle Inference 推理时，加载的模型类型为 inference 模型。本案例提供了两种获得 inference 模型的方法，如果希望得到和文档相同的结果，请选择[直接下载 inference 模型](#6.1.2)的方式。 + + + +### 6.1.1 基于训练得到的权重导出 inference 模型 + +此处，我们提供了将权重和模型转换的脚本，执行该脚本可以得到对应的 inference 模型： + +```bash +python3 tools/export_model.py \ + -c ./ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml \ + -o Global.pretrained_model=output/DistillationModel/best_model_student \ + -o Global.save_inference_dir=deploy/models/PPLCNet_x1_0_traffic_sign_infer +``` +执行完该脚本后会在 `deploy/models/` 下生成 `PPLCNet_x1_0_traffic_sign_infer` 文件夹，`models` 文件夹下应有如下文件结构： + +``` +├── PPLCNet_x1_0_traffic_sign_infer +│ ├── inference.pdiparams +│ ├── inference.pdiparams.info +│ └── inference.pdmodel +``` + +**备注：** 此处的最佳权重是经过知识蒸馏后的权重路径，如果没有执行知识蒸馏的步骤，最佳模型保存在`output/PPLCNet_x1_0/best_model.pdparams`中。 + + + +### 6.1.2 直接下载 inference 模型 + +[6.1.1 小节](#6.1.1)提供了导出 inference 模型的方法，此处也提供了该场景可以下载的 inference 模型，可以直接下载体验。 + +``` +cd deploy/models +# 下载 inference 模型并解压 +wget https://paddleclas.bj.bcebos.com/models/PULC/traffic_sign_infer.tar && tar -xf traffic_sign_infer.tar +``` + +解压完毕后，`models` 文件夹下应有如下文件结构： + +``` +├── traffic_sign_infer +│ ├── inference.pdiparams +│ ├── inference.pdiparams.info +│ └── inference.pdmodel +``` + + + +### 6.2 基于 Python 预测引擎推理 + + + + +#### 6.2.1 预测单张图像 + +返回 `deploy` 目录： + +``` +cd ../ +``` + +运行下面的命令，对图像 `./images/PULC/traffic_sign/99603_17806.jpg` 进行交通标志分类。 + +```shell +# 使用下面的命令使用 GPU 进行预测 +python3.7 python/predict_cls.py -c configs/PULC/traffic_sign/inference_traffic_sign.yaml +# 使用下面的命令使用 CPU 进行预测 +python3.7 python/predict_cls.py -c configs/PULC/traffic_sign/inference_traffic_sign.yaml -o Global.use_gpu=False +``` + +输出结果如下。 + +``` +99603_17806.jpg: class id(s): [216, 145, 49, 207, 169], score(s): [1.00, 0.00, 0.00, 0.00, 0.00], label_name(s): ['pm20', 'pm30', 'pm40', 'pl25', 'pm15'] +``` + + + +#### 6.2.2 基于文件夹的批量预测 + +如果希望预测文件夹内的图像，可以直接修改配置文件中的 `Global.infer_imgs` 字段，也可以通过下面的 `-o` 参数修改对应的配置。 + +```shell +# 使用下面的命令使用 GPU 进行预测，如果希望使用 CPU 预测，可以在命令后面添加 -o Global.use_gpu=False +python3.7 python/predict_cls.py -c configs/PULC/traffic_sign/inference_traffic_sign.yaml -o Global.infer_imgs="./images/PULC/traffic_sign/" +``` + +终端中会输出该文件夹内所有图像的分类结果，如下所示。 + +``` +100999_83928.jpg: class id(s): [182, 179, 162, 128, 24], score(s): [0.99, 0.01, 0.00, 0.00, 0.00], label_name(s): ['pl110', 'pl100', 'pl120', 'p26', 'pm10'] +99603_17806.jpg: class id(s): [216, 145, 49, 24, 169], score(s): [1.00, 0.00, 0.00, 0.00, 0.00], label_name(s): ['pm20', 'pm30', 'pm40', 'pm10', 'pm15'] +``` + +输出的 `label_name`可以从`dataset/traffic_sign/report.pdf`文件中查阅对应的图片。 + + + +### 6.3 基于 C++ 预测引擎推理 + +PaddleClas 提供了基于 C++ 预测引擎推理的示例，您可以参考[服务器端 C++ 预测](../inference_deployment/cpp_deploy.md)来完成相应的推理部署。如果您使用的是 Windows 平台，可以参考[基于 Visual Studio 2019 Community CMake 编译指南](../inference_deployment/cpp_deploy_on_windows.md)完成相应的预测库编译和模型预测工作。 + + + +### 6.4 服务化部署 + +Paddle Serving 提供高性能、灵活易用的工业级在线推理服务。Paddle Serving 支持 RESTful、gRPC、bRPC 等多种协议，提供多种异构硬件和多种操作系统环境下推理解决方案。更多关于Paddle Serving 的介绍，可以参考[Paddle Serving 代码仓库](https://github.com/PaddlePaddle/Serving)。 + +PaddleClas 提供了基于 Paddle Serving 来完成模型服务化部署的示例，您可以参考[模型服务化部署](../inference_deployment/paddle_serving_deploy.md)来完成相应的部署工作。 + + + +### 6.5 端侧部署 + +Paddle Lite 是一个高性能、轻量级、灵活性强且易于扩展的深度学习推理框架，定位于支持包括移动端、嵌入式以及服务器端在内的多硬件平台。更多关于 Paddle Lite 的介绍，可以参考[Paddle Lite 代码仓库](https://github.com/PaddlePaddle/Paddle-Lite)。 + +PaddleClas 提供了基于 Paddle Lite 来完成模型端侧部署的示例，您可以参考[端侧部署](../inference_deployment/paddle_lite_deploy.md)来完成相应的部署工作。 + + + +### 6.6 Paddle2ONNX 模型转换与预测 + +Paddle2ONNX 支持将 PaddlePaddle 模型格式转化到 ONNX 模型格式。通过 ONNX 可以完成将 Paddle 模型到多种推理引擎的部署，包括TensorRT/OpenVINO/MNN/TNN/NCNN，以及其它对 ONNX 开源格式进行支持的推理引擎或硬件。更多关于 Paddle2ONNX 的介绍，可以参考[Paddle2ONNX 代码仓库](https://github.com/PaddlePaddle/Paddle2ONNX)。 + +PaddleClas 提供了基于 Paddle2ONNX 来完成 inference 模型转换 ONNX 模型并作推理预测的示例，您可以参考[Paddle2ONNX 模型转换与预测](@shuilong)来完成相应的部署工作。 diff --git a/docs/zh_CN/PULC/PULC_vehicle_attr.md b/docs/zh_CN/PULC/PULC_vehicle_attr.md new file mode 100644 index 000000000..80e038ac8 --- /dev/null +++ b/docs/zh_CN/PULC/PULC_vehicle_attr.md @@ -0,0 +1,414 @@ +# PULC 车辆属性识别模型 + +------ + + +## 目录 + +- [1. 模型和应用场景介绍](#1) +- [2. 模型快速体验](#2) +- [3. 模型训练、评估和预测](#3) + - [3.1 环境配置](#3.1) + - [3.2 数据准备](#3.2) + - [3.2.1 数据集来源](#3.2.1) + - [3.2.2 数据集获取](#3.2.2) + - [3.3 模型训练](#3.3) + - [3.4 模型评估](#3.4) + - [3.5 模型预测](#3.5) +- [4. 模型压缩](#4) + - [4.1 SKL-UGI 知识蒸馏](#4.1) + - [4.1.1 教师模型训练](#4.1.1) + - [4.1.2 蒸馏训练](#4.1.2) +- [5. 超参搜索](#5) +- [6. 模型推理部署](#6) + - [6.1 推理模型准备](#6.1) + - [6.1.1 基于训练得到的权重导出 inference 模型](#6.1.1) + - [6.1.2 直接下载 inference 模型](#6.1.2) + - [6.2 基于 Python 预测引擎推理](#6.2) + - [6.2.1 预测单张图像](#6.2.1) + - [6.2.2 基于文件夹的批量预测](#6.2.2) + - [6.3 基于 C++ 预测引擎推理](#6.3) + - [6.4 服务化部署](#6.4) + - [6.5 端侧部署](#6.5) + - [6.6 Paddle2ONNX 模型转换与预测](#6.6) + + + + +## 1. 模型和应用场景介绍 + +该案例提供了用户使用 PaddleClas 的超轻量图像分类方案（PULC，Practical Ultra Lightweight Classification）快速构建轻量级、高精度、可落地的车辆属性识别模型。该模型可以广泛应用于车辆识别、道路监控等场景。 + +下表列出了不同车辆属性识别模型的相关指标，前两行展现了使用 Res2Net200_vd_26w_4s 和 MobileNetV3_large_x1_0 作为 backbone 训练得到的模型的相关指标，第三行至第六行依次展现了替换 backbone 为 PPLCNet_x1_0、使用 SSLD 预训练模型、使用 SSLD 预训练模型 + EDA 策略、使用 SSLD 预训练模型 + EDA 策略 + SKL-UGI 知识蒸馏策略训练得到的模型的相关指标。 + + +| 模型 | ma（%） | 延时（ms） | 存储（M） | 策略 | +|-------|-----------|----------|---------------|---------------| +| Res2Net200_vd_26w_4s | 91.36 | 66.58 | 293 | 使用ImageNet预训练模型 | +| ResNet50 | 89.98 | 12.74 | 92 | 使用ImageNet预训练模型 | +| MobileNetV3_large_x1_0 | 89.77 | 5.59 | 23 | 使用ImageNet预训练模型 | +| PPLCNet_x1_0 | 89.57 | 2.56 | 8.2 | 使用ImageNet预训练模型 | +| PPLCNet_x1_0 | 90.07 | 2.56 | 8.2 | 使用SSLD预训练模型 | +| PPLCNet_x1_0 | 90.59 | 2.56 | 8.2 | 使用SSLD预训练模型+EDA策略| +| PPLCNet_x1_0 | 90.81 | 2.56 | 8.2 | 使用SSLD预训练模型+EDA策略+SKL-UGI知识蒸馏策略| + +从表中可以看出，backbone 为 Res2Net200_vd_26w_4s 时精度较高，但是推理速度较慢。将 backbone 替换为轻量级模型 MobileNetV3_large_x1_0 后，速度可以大幅提升，但是精度下降明显。将 backbone 替换为 PPLCNet_x1_0 时，精度低0.2%，但是速度提升 2 倍左右。在此基础上，使用 SSLD 预训练模型后，在不改变推理速度的前提下，精度可以提升约 0.5%，进一步地，当融合EDA策略后，精度可以再提升 0.52%，最后，在使用 SKL-UGI 知识蒸馏后，精度可以继续提升 0.23%。此时，PPLCNet_x1_0 的精度与 Res2Net200_vd_26w_4s 仅相差0.55%，但是速度快26倍。关于 PULC 的训练方法和推理部署方法将在下面详细介绍。 + +**备注：** + +* 关于PPLCNet的介绍可以参考[PPLCNet介绍](../models/PP-LCNet.md)，相关论文可以查阅[PPLCNet paper](https://arxiv.org/abs/2109.15099)。 + + + + +## 2. 模型快速体验 + +``` +（pip方式，待补充） +``` + + + +## 3. 模型训练、评估和预测 + + + +### 3.1 环境配置 + +* 安装：请先参考 [Paddle 安装教程](../installation/install_paddle.md) 以及 [PaddleClas 安装教程](../installation/install_paddleclas.md) 配置 PaddleClas 运行环境。 + + + +### 3.2 数据准备 + + + +#### 3.2.1 数据集来源 + +本案例中所使用的数据为[VeRi 数据集](https://www.v7labs.com/open-datasets/veri-dataset)。 + + + +#### 3.2.2 数据集获取 + +部分数据可视化如下所示。 + +
+ +
+ +首先从[VeRi数据集官网](https://www.v7labs.com/open-datasets/veri-dataset)中申请并下载数据，放在PaddleClas的`dataset`目录下，数据集目录名为`VeRi`，使用下面的命令进入该文件夹。 + +```shell +cd PaddleClas/dataset/VeRi/ +``` + +然后使用下面的代码转换label（可以在python终端中执行下面的命令，也可以将其写入一个文件，然后使用`python3 convert.py`的方式运行该文件）。 + + +```python +import os +from xml.dom.minidom import parse + +vehicleids = [] + +def convert_annotation(input_fp, output_fp): + in_file = open(input_fp) + list_file = open(output_fp, 'w') + tree = parse(in_file) + + root = tree.documentElement + + for item in root.getElementsByTagName("Item"): + label = ['0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0'] + if item.hasAttribute("imageName"): + name = item.getAttribute("imageName") + if item.hasAttribute("vehicleID"): + vehicleid = item.getAttribute("vehicleID") + if vehicleid not in vehicleids : + vehicleids.append(vehicleid) + vid = vehicleids.index(vehicleid) + if item.hasAttribute("colorID"): + colorid = int (item.getAttribute("colorID")) + label[colorid-1] = '1' + if item.hasAttribute("typeID"): + typeid = int (item.getAttribute("typeID")) + label[typeid+9] = '1' + label = ','.join(label) + list_file.write(os.path.join('image_train', name) + "\t" + label + "\n") + + list_file.close() + +convert_annotation('train_label.xml', 'train_list.txt') #imagename vehiclenum colorid typeid +convert_annotation('test_label.xml', 'test_list.txt') +``` + +执行上述命令后，`VeRi`目录中具有以下数据： + +``` +VeRi +├── image_train +│ ├── 0001_c001_00016450_0.jpg +│ ├── 0001_c001_00016460_0.jpg +│ ├── 0001_c001_00016470_0.jpg +... +├── image_test +│ ├── 0002_c002_00030600_0.jpg +│ ├── 0002_c002_00030605_1.jpg +│ ├── 0002_c002_00030615_1.jpg +... +... +├── train_list.txt +├── test_list.txt +├── train_label.xml +├── test_label.xml +``` + +其中`train/`和`test/`分别为训练集和验证集。`train_list.txt`和`test_list.txt`分别为训练集和验证集的转换后用于训练的标签文件。 + + + + +### 3.3 模型训练 + + +在 `ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml` 中提供了基于该场景的训练配置，可以通过如下脚本启动训练： + +```shell +export CUDA_VISIBLE_DEVICES=0,1,2,3 +python3 -m paddle.distributed.launch \ + --gpus="0,1,2,3" \ + tools/train.py \ + -c ./ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml +``` + +验证集的最佳指标在 `90.07%` 左右（数据集较小，一般有0.3%左右的波动）。 + + + + +### 3.4 模型评估 + +训练好模型之后，可以通过以下命令实现对模型指标的评估。 + +```bash +python3 tools/eval.py \ + -c ./ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml \ + -o Global.pretrained_model="output/PPLCNet_x1_0/best_model" +``` + +其中 `-o Global.pretrained_model="output/PPLCNet_x1_0/best_model"` 指定了当前最佳权重所在的路径，如果指定其他权重，只需替换对应的路径即可。 + + + +### 3.5 模型预测 + +模型训练完成之后，可以加载训练得到的预训练模型，进行模型预测。在模型库的 `tools/infer.py` 中提供了完整的示例，只需执行下述命令即可完成模型预测： + +```bash +python3 tools/infer.py \ + -c ./ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml \ + -o Global.pretrained_model=output/DistillationModel/best_model +``` + +输出结果如下： + +``` +[{'attr': 'Color: (yellow, prob: 0.9893478155136108), Type: (hatchback, prob: 0.9734100103378296)', 'pred': [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0], 'file_name': './deploy/images/PULC/vehicle_attr/0002_c002_00030670_0.jpg'}] +``` + +**备注：** + +* 这里`-o Global.pretrained_model="output/PPLCNet_x1_0/best_model"` 指定了当前最佳权重所在的路径，如果指定其他权重，只需替换对应的路径即可。 + +* 默认是对 `./deploy/images/PULC/vehicle_attr/0002_c002_00030670_0.jpg` 进行预测，此处也可以通过增加字段 `-o Infer.infer_imgs=xxx` 对其他图片预测。 + + + +## 4. 模型压缩 + + + +### 4.1 SKL-UGI 知识蒸馏 + +SKL-UGI 知识蒸馏是 PaddleClas 提出的一种简单有效的知识蒸馏方法，关于该方法的介绍，可以参考[SKL-UGI 知识蒸馏](@ruoyu)。 + + + +#### 4.1.1 教师模型训练 + +复用 `ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml` 中的超参数，训练教师模型，训练脚本如下： + +```shell +export CUDA_VISIBLE_DEVICES=0,1,2,3 +python3 -m paddle.distributed.launch \ + --gpus="0,1,2,3" \ + tools/train.py \ + -c ./ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml \ + -o Arch.name=ResNet101_vd +``` + +验证集的最佳指标为 `91.60%` 左右，当前教师模型最好的权重保存在 `output/ResNet101_vd/best_model.pdparams`。 + + + +#### 4.1.2 蒸馏训练 + +配置文件`ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_distillation.yaml`提供了`SKL-UGI知识蒸馏策略`的配置。该配置将`ResNet101_vd`当作教师模型，`PPLCNet_x1_0`当作学生模型。训练脚本如下： + +```shell +export CUDA_VISIBLE_DEVICES=0,1,2,3 +python3 -m paddle.distributed.launch \ + --gpus="0,1,2,3" \ + tools/train.py \ + -c ./ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_distillation.yaml \ + -o Arch.models.0.Teacher.pretrained=output/ResNet101_vd/best_model +``` + +验证集的最佳指标为 `90.81%` 左右，当前模型最好的权重保存在 `output/DistillationModel/best_model_student.pdparams`。 + + + + +## 5. 超参搜索 + +在 [3.2 节](#3.2)和 [4.1 节](#4.1)所使用的超参数是根据 PaddleClas 提供的 `SHAS 超参数搜索策略` 搜索得到的，如果希望在自己的数据集上得到更好的结果，可以参考[SHAS 超参数搜索策略](#TODO)来获得更好的训练超参数。 + +**备注：** 此部分内容是可选内容，搜索过程需要较长的时间，您可以根据自己的硬件情况来选择执行。如果没有更换数据集，可以忽略此节内容。 + + + +## 6. 模型推理部署 + + + +### 6.1 推理模型准备 + +Paddle Inference 是飞桨的原生推理库， 作用于服务器端和云端，提供高性能的推理能力。相比于直接基于预训练模型进行预测，Paddle Inference可使用MKLDNN、CUDNN、TensorRT 进行预测加速，从而实现更优的推理性能。更多关于Paddle Inference推理引擎的介绍，可以参考[Paddle Inference官网教程](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/infer/inference/inference_cn.html)。 + +当使用 Paddle Inference 推理时，加载的模型类型为 inference 模型。本案例提供了两种获得 inference 模型的方法，如果希望得到和文档相同的结果，请选择[直接下载 inference 模型](#6.1.2)的方式。 + + + +### 6.1.1 基于训练得到的权重导出 inference 模型 + +此处，我们提供了将权重和模型转换的脚本，执行该脚本可以得到对应的 inference 模型： + +```bash +python3 tools/export_model.py \ + -c ./ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml \ + -o Global.pretrained_model=output/DistillationModel/best_model_student \ + -o Global.save_inference_dir=deploy/models/PPLCNet_x1_0_vehicle_attr_infer +``` +执行完该脚本后会在 `deploy/models/` 下生成 `PPLCNet_x1_0_vehicle_attr_infer` 文件夹，`models` 文件夹下应有如下文件结构： + +``` +├── PPLCNet_x1_0_vehicle_attr_infer +│ ├── inference.pdiparams +│ ├── inference.pdiparams.info +│ └── inference.pdmodel +``` + +**备注：** 此处的最佳权重是经过知识蒸馏后的权重路径，如果没有执行知识蒸馏的步骤，最佳模型保存在`output/PPLCNet_x1_0/best_model.pdparams`中。 + + + +### 6.1.2 直接下载 inference 模型 + +[6.1.1 小节](#6.1.1)提供了导出 inference 模型的方法，此处也提供了该场景可以下载的 inference 模型，可以直接下载体验。 + +``` +cd deploy/models +# 下载 inference 模型并解压 +wget https://paddleclas.bj.bcebos.com/models/PULC/vehicle_attr_infer.tar && tar -xf vehicle_attr_infer.tar +``` + +解压完毕后，`models` 文件夹下应有如下文件结构： + +``` +├── vehicle_attr_infer +│ ├── inference.pdiparams +│ ├── inference.pdiparams.info +│ └── inference.pdmodel +``` + + + +### 6.2 基于 Python 预测引擎推理 + + + + +#### 6.2.1 预测单张图像 + +返回 `deploy` 目录： + +``` +cd ../ +``` + +运行下面的命令，对图像 `./images/PULC/vehicle_attr/0002_c002_00030670_0.jpg` 进行车辆属性识别。 + +```shell +# 使用下面的命令使用 GPU 进行预测 +python3.7 python/predict_cls.py -c configs/PULC/vehicle_attr/inference_vehicle_attr.yaml -o Global.use_gpu=True +# 使用下面的命令使用 CPU 进行预测 +python3.7 python/predict_cls.py -c configs/PULC/vehicle_attr/inference_vehicle_attr.yaml -o Global.use_gpu=False +``` + +输出结果如下。 + +``` +0002_c002_00030670_0.jpg: attributes: Color: (yellow, prob: 0.9893478155136108), Type: (hatchback, prob: 0.97340989112854), +predict output: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] +``` + + + +#### 6.2.2 基于文件夹的批量预测 + +如果希望预测文件夹内的图像，可以直接修改配置文件中的 `Global.infer_imgs` 字段，也可以通过下面的 `-o` 参数修改对应的配置。 + +```shell +# 使用下面的命令使用 GPU 进行预测，如果希望使用 CPU 预测，可以在命令后面添加 -o Global.use_gpu=False +python3.7 python/predict_cls.py -c configs/PULC/vehicle_attr/inference_vehicle_attr.yaml -o Global.infer_imgs="./images/PULC/vehicle_attr/" +``` + +终端中会输出该文件夹内所有图像的属性识别结果，如下所示。 + +``` +0002_c002_00030670_0.jpg: attributes: Color: (yellow, prob: 0.9893478155136108), Type: (hatchback, prob: 0.97340989112854), +predict output: [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] +0014_c012_00040750_0.jpg: attributes: Color: (red, prob: 0.9998721480369568), Type: (sedan, prob: 0.999976634979248), +predict output: [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] +``` + + + +### 6.3 基于 C++ 预测引擎推理 + +PaddleClas 提供了基于 C++ 预测引擎推理的示例，您可以参考[服务器端 C++ 预测](../inference_deployment/cpp_deploy.md)来完成相应的推理部署。如果您使用的是 Windows 平台，可以参考[基于 Visual Studio 2019 Community CMake 编译指南](../inference_deployment/cpp_deploy_on_windows.md)完成相应的预测库编译和模型预测工作。 + + + +### 6.4 服务化部署 + +Paddle Serving 提供高性能、灵活易用的工业级在线推理服务。Paddle Serving 支持 RESTful、gRPC、bRPC 等多种协议，提供多种异构硬件和多种操作系统环境下推理解决方案。更多关于Paddle Serving 的介绍，可以参考[Paddle Serving 代码仓库](https://github.com/PaddlePaddle/Serving)。 + +PaddleClas 提供了基于 Paddle Serving 来完成模型服务化部署的示例，您可以参考[模型服务化部署](../inference_deployment/paddle_serving_deploy.md)来完成相应的部署工作。 + + + +### 6.5 端侧部署 + +Paddle Lite 是一个高性能、轻量级、灵活性强且易于扩展的深度学习推理框架，定位于支持包括移动端、嵌入式以及服务器端在内的多硬件平台。更多关于 Paddle Lite 的介绍，可以参考[Paddle Lite 代码仓库](https://github.com/PaddlePaddle/Paddle-Lite)。 + +PaddleClas 提供了基于 Paddle Lite 来完成模型端侧部署的示例，您可以参考[端侧部署](../inference_deployment/paddle_lite_deploy.md)来完成相应的部署工作。 + + + +### 6.6 Paddle2ONNX 模型转换与预测 + +Paddle2ONNX 支持将 PaddlePaddle 模型格式转化到 ONNX 模型格式。通过 ONNX 可以完成将 Paddle 模型到多种推理引擎的部署，包括TensorRT/OpenVINO/MNN/TNN/NCNN，以及其它对 ONNX 开源格式进行支持的推理引擎或硬件。更多关于 Paddle2ONNX 的介绍，可以参考[Paddle2ONNX 代码仓库](https://github.com/PaddlePaddle/Paddle2ONNX)。 + +PaddleClas 提供了基于 Paddle2ONNX 来完成 inference 模型转换 ONNX 模型并作推理预测的示例，您可以参考[Paddle2ONNX 模型转换与预测](@shuilong)来完成相应的部署工作。 diff --git a/ppcls/arch/backbone/legendary_models/mobilenet_v3.py b/ppcls/arch/backbone/legendary_models/mobilenet_v3.py index b7fc7e9f7..3fbf9776b 100644 --- a/ppcls/arch/backbone/legendary_models/mobilenet_v3.py +++ b/ppcls/arch/backbone/legendary_models/mobilenet_v3.py @@ -154,7 +154,8 @@ class MobileNetV3(TheseusLayer): class_expand=LAST_CONV, dropout_prob=0.2, return_patterns=None, - return_stages=None): + return_stages=None, + **kwargs): super().__init__() self.cfg = config diff --git a/ppcls/arch/backbone/legendary_models/pp_lcnet.py b/ppcls/arch/backbone/legendary_models/pp_lcnet.py index 64fa61e19..23173b34a 100644 --- a/ppcls/arch/backbone/legendary_models/pp_lcnet.py +++ b/ppcls/arch/backbone/legendary_models/pp_lcnet.py @@ -94,13 +94,16 @@ class ConvBNLayer(TheseusLayer): stride=stride, padding=(filter_size - 1) // 2, groups=num_groups, - weight_attr=ParamAttr(initializer=KaimingNormal(), learning_rate=lr_mult), + weight_attr=ParamAttr( + initializer=KaimingNormal(), learning_rate=lr_mult), bias_attr=False) self.bn = BatchNorm2D( num_filters, - weight_attr=ParamAttr(regularizer=L2Decay(0.0), learning_rate=lr_mult), - bias_attr=ParamAttr(regularizer=L2Decay(0.0), learning_rate=lr_mult)) + weight_attr=ParamAttr( + regularizer=L2Decay(0.0), learning_rate=lr_mult), + bias_attr=ParamAttr( + regularizer=L2Decay(0.0), learning_rate=lr_mult)) self.hardswish = nn.Hardswish() def forward(self, x): @@ -128,8 +131,7 @@ class DepthwiseSeparable(TheseusLayer): num_groups=num_channels, lr_mult=lr_mult) if use_se: - self.se = SEModule(num_channels, - lr_mult=lr_mult) + self.se = SEModule(num_channels, lr_mult=lr_mult) self.pw_conv = ConvBNLayer( num_channels=num_channels, filter_size=1, @@ -189,7 +191,8 @@ class PPLCNet(TheseusLayer): lr_mult_list=[1.0, 1.0, 1.0, 1.0, 1.0, 1.0], use_last_conv=True, return_patterns=None, - return_stages=None): + return_stages=None, + **kwargs): super().__init__() self.scale = scale self.class_expand = class_expand @@ -271,7 +274,8 @@ class PPLCNet(TheseusLayer): self.avg_pool = AdaptiveAvgPool2D(1) if self.use_last_conv: self.last_conv = Conv2D( - in_channels=make_divisible(NET_CONFIG["blocks6"][-1][2] * scale), + in_channels=make_divisible(NET_CONFIG["blocks6"][-1][2] * + scale), out_channels=self.class_expand, kernel_size=1, stride=1, @@ -282,7 +286,8 @@ class PPLCNet(TheseusLayer): else: self.last_conv = None self.flatten = nn.Flatten(start_axis=1, stop_axis=-1) - self.fc = Linear(self.class_expand if self.use_last_conv else NET_CONFIG["blocks6"][-1][2], class_num) + self.fc = Linear(self.class_expand if self.use_last_conv else + NET_CONFIG["blocks6"][-1][2], class_num) super().init_res( stages_pattern, diff --git a/ppcls/arch/backbone/model_zoo/res2net_vd.py b/ppcls/arch/backbone/model_zoo/res2net_vd.py index 511fbaa59..2139e1988 100644 --- a/ppcls/arch/backbone/model_zoo/res2net_vd.py +++ b/ppcls/arch/backbone/model_zoo/res2net_vd.py @@ -165,7 +165,8 @@ class BottleneckBlock(nn.Layer): class Res2Net_vd(nn.Layer): - def __init__(self, layers=50, scales=4, width=26, class_num=1000): + def __init__(self, layers=50, scales=4, width=26, class_num=1000, + **kwargs): super(Res2Net_vd, self).__init__() self.layers = layers diff --git a/ppcls/configs/PULC/traffic_sign/MobileNetV3_large_x1_0.yaml b/ppcls/configs/PULC/traffic_sign/MobileNetV3_large_x1_0.yaml new file mode 100644 index 000000000..e76db0479 --- /dev/null +++ b/ppcls/configs/PULC/traffic_sign/MobileNetV3_large_x1_0.yaml @@ -0,0 +1,132 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: ./output/ + device: gpu + save_interval: 1 + eval_during_train: True + eval_interval: 1 + epochs: 10 + print_batch_step: 10 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 224, 224] + save_inference_dir: ./inference + +# model architecture +Arch: + name: MobileNetV3_large_x1_0 + class_num: 232 + pretrained: True + +# loss function config for traing/eval process +Loss: + Train: + - CELoss: + weight: 1.0 + epsilon: 0.1 + Eval: + - CELoss: + weight: 1.0 + + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.00002 + + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_train.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - RandCropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + + Eval: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_test.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + +Infer: + infer_imgs: docs/images/inference_deployment/whl_demo.jpg + batch_size: 10 + transforms: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - ToCHWImage: + PostProcess: + name: Topk + topk: 5 + class_id_map_file: dataset/traffic_sign/label_name_id.txt + +Metric: + Train: + - TopkAcc: + topk: [1, 5] + Eval: + - TopkAcc: + topk: [1, 5] + diff --git a/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml b/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml new file mode 100644 index 000000000..025d7ef2e --- /dev/null +++ b/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0.yaml @@ -0,0 +1,148 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: ./output/ + device: gpu + save_interval: 1 + eval_during_train: True + eval_interval: 1 + start_eval_epoch: 0 + epochs: 10 + print_batch_step: 10 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 224, 224] + save_inference_dir: ./inference + # training model under @to_static + to_static: False + use_dali: False + + +# model architecture +Arch: + name: PPLCNet_x1_0 + class_num: 232 + pretrained: True + use_ssld: True + +# loss function config for traing/eval process +Loss: + Train: + - CELoss: + weight: 1.0 + Eval: + - CELoss: + weight: 1.0 + + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.02 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.00004 + + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/tt100k_clas_v2/label_list_train.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - RandCropImage: + size: 224 + - TimmAutoAugment: + prob: 0.5 + config_str: rand-m9-mstd0.5-inc1 + interpolation: bicubic + img_size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - RandomErasing: + EPSILON: 0.0 + sl: 0.02 + sh: 1.0/3.0 + r1: 0.3 + attempt: 10 + use_log_aspect: True + mode: pixel + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: False + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + + Eval: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_test.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + +Infer: + infer_imgs: deploy/images/PULC/traffic_sign/99603_17806.jpg + batch_size: 10 + transforms: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - ToCHWImage: + PostProcess: + name: Topk + topk: 5 + class_id_map_file: dataset/traffic_sign/label_name_id.txt + +Metric: + Train: + - TopkAcc: + topk: [1, 5] + Eval: + - TopkAcc: + topk: [1, 5] diff --git a/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_distillation.yaml b/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_distillation.yaml new file mode 100644 index 000000000..9be452d46 --- /dev/null +++ b/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_distillation.yaml @@ -0,0 +1,172 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: ./output/ + device: gpu + save_interval: 1 + eval_during_train: True + eval_interval: 1 + epochs: 10 + print_batch_step: 10 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 224, 224] + save_inference_dir: ./inference + # training model under @to_static + to_static: False + use_dali: False + +# mixed precision training +AMP: + scale_loss: 128.0 + use_dynamic_loss_scaling: True + # O1: mixed fp16 + level: O1 + +# model architecture +Arch: + name: "DistillationModel" + class_num: &class_num 232 + # if not null, its lengths should be same as models + pretrained_list: + # if not null, its lengths should be same as models + freeze_params_list: + - True + - False + models: + - Teacher: + name: ResNet101_vd + class_num: *class_num + pretrained: False + - Student: + name: PPLCNet_x1_0 + class_num: *class_num + pretrained: True + use_ssld: True + + infer_model_name: "Student" + +# loss function config for traing/eval process +Loss: + Train: + - DistillationDMLLoss: + weight: 1.0 + model_name_pairs: + - ["Student", "Teacher"] + Eval: + - CELoss: + weight: 1.0 + + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.00004 + + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_train_for_distillation.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - RandCropImage: + size: 224 + - TimmAutoAugment: + prob: 0.0 + config_str: rand-m9-mstd0.5-inc1 + interpolation: bicubic + img_size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - RandomErasing: + EPSILON: 0.0 + sl: 0.02 + sh: 1.0/3.0 + r1: 0.3 + attempt: 10 + use_log_aspect: True + mode: pixel + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: False + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + + Eval: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_test.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + +Infer: + infer_imgs: docs/images/inference_deployment/whl_demo.jpg + batch_size: 10 + transforms: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - ToCHWImage: + PostProcess: + name: Topk + topk: 5 + class_id_map_file: dataset/traffic_sign/label_name_id.txt + +Metric: + Train: + - DistillationTopkAcc: + model_key: "Student" + topk: [1, 5] + Eval: + - TopkAcc: + topk: [1, 5] diff --git a/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_search.yaml b/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_search.yaml new file mode 100644 index 000000000..6c622b0e0 --- /dev/null +++ b/ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_search.yaml @@ -0,0 +1,149 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: ./output/ + device: gpu + save_interval: 1 + eval_during_train: True + eval_interval: 1 + start_eval_epoch: 0 + epochs: 10 + print_batch_step: 10 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 224, 224] + save_inference_dir: ./inference + # training model under @to_static + to_static: False + use_dali: False + + +# model architecture +Arch: + name: PPLCNet_x1_0 + class_num: 232 + pretrained: True + # use_ssld: True + +# loss function config for traing/eval process +Loss: + Train: + - CELoss: + weight: 1.0 + Eval: + - CELoss: + weight: 1.0 + + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.00004 + + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_train.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - RandCropImage: + size: 224 + - TimmAutoAugment: + prob: 0.0 + config_str: rand-m9-mstd0.5-inc1 + interpolation: bicubic + img_size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - RandomErasing: + EPSILON: 0.0 + sl: 0.02 + sh: 1.0/3.0 + r1: 0.3 + attempt: 10 + use_log_aspect: True + mode: pixel + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: False + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + + Eval: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_test.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + +Infer: + # infer_imgs: dataset/traffic_sign_demo/ + infer_imgs: dataset/tt100k_clas_v2/test/ + batch_size: 10 + transforms: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - ToCHWImage: + PostProcess: + name: Topk + topk: 5 + class_id_map_file: dataset/traffic_sign/label_name_id.txt + +Metric: + Train: + - TopkAcc: + topk: [1, 5] + Eval: + - TopkAcc: + topk: [1, 5] diff --git a/ppcls/configs/PULC/traffic_sign/SwinTransformer_tiny_patch4_window7_224.yaml b/ppcls/configs/PULC/traffic_sign/SwinTransformer_tiny_patch4_window7_224.yaml new file mode 100644 index 000000000..5d295e28c --- /dev/null +++ b/ppcls/configs/PULC/traffic_sign/SwinTransformer_tiny_patch4_window7_224.yaml @@ -0,0 +1,170 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: ./output/ + device: gpu + save_interval: 1 + eval_during_train: True + eval_interval: 1 + start_eval_epoch: 0 + epochs: 10 + print_batch_step: 10 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 224, 224] + save_inference_dir: ./inference + # training model under @to_static + to_static: False + use_dali: False + +# mixed precision training +AMP: + scale_loss: 128.0 + use_dynamic_loss_scaling: True + # O1: mixed fp16 + level: O1 + +# model architecture +Arch: + name: SwinTransformer_tiny_patch4_window7_224 + class_num: 232 + pretrained: True + +# loss function config for traing/eval process +Loss: + Train: + - CELoss: + weight: 1.0 + epsilon: 0.1 + Eval: + - CELoss: + weight: 1.0 + +Optimizer: + name: AdamW + beta1: 0.9 + beta2: 0.999 + epsilon: 1e-8 + weight_decay: 0.05 + no_weight_decay_name: absolute_pos_embed relative_position_bias_table .bias norm + one_dim_param_no_weight_decay: True + lr: + name: Cosine + learning_rate: 2e-4 + eta_min: 2e-6 + warmup_epoch: 5 + warmup_start_lr: 2e-7 + + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/traffic_sign/label_list_train.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - RandCropImage: + size: 224 + interpolation: bicubic + backend: pil + - RandFlipImage: + flip_code: 1 + - TimmAutoAugment: + config_str: rand-m9-mstd0.5-inc1 + interpolation: bicubic + img_size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - RandomErasing: + EPSILON: 0.25 + sl: 0.02 + sh: 1.0/3.0 + r1: 0.3 + attempt: 10 + use_log_aspect: True + mode: pixel + batch_transform_ops: + - OpSampler: + MixupOperator: + alpha: 0.8 + prob: 0.5 + CutmixOperator: + alpha: 1.0 + prob: 0.5 + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + + Eval: + dataset: + name: ImageNetDataset + image_root: ./dataset/ + cls_label_path: ./dataset/tt100k_clas_v2/label_list_test.txt + delimiter: "\t" + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + +Infer: + infer_imgs: docs/images/inference_deployment/whl_demo.jpg + batch_size: 10 + transforms: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + resize_short: 256 + - CropImage: + size: 224 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - ToCHWImage: + PostProcess: + name: Topk + topk: 5 + class_id_map_file: dataset/traffic_sign/label_name_id.txt + +Metric: + Train: + - TopkAcc: + topk: [1, 5] + Eval: + - TopkAcc: + topk: [1, 5] + + diff --git a/ppcls/configs/PULC/traffic_sign/search.yaml b/ppcls/configs/PULC/traffic_sign/search.yaml new file mode 100644 index 000000000..755ed2016 --- /dev/null +++ b/ppcls/configs/PULC/traffic_sign/search.yaml @@ -0,0 +1,40 @@ +base_config_file: ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_search.yaml +distill_config_file: ppcls/configs/PULC/traffic_sign/PPLCNet_x1_0_distillation.yaml + +gpus: 0,1,2,3 +output_dir: output/search_traffic_sign +search_times: 1 +search_dict: + - search_key: lrs + replace_config: + - Optimizer.lr.learning_rate + search_values: [0.0075, 0.01, 0.0125] + - search_key: resolutions + replace_config: + - DataLoader.Train.dataset.transform_ops.1.RandCropImage.size + - DataLoader.Train.dataset.transform_ops.2.TimmAutoAugment.img_size + search_values: [176, 192, 224] + - search_key: ra_probs + replace_config: + - DataLoader.Train.dataset.transform_ops.2.TimmAutoAugment.prob + search_values: [0.0, 0.1, 0.5] + - search_key: re_probs + replace_config: + - DataLoader.Train.dataset.transform_ops.4.RandomErasing.EPSILON + search_values: [0.0, 0.1, 0.5] + - search_key: lr_mult_list + replace_config: + - Arch.lr_mult_list + search_values: + - [0.0, 0.2, 0.4, 0.6, 0.8, 1.0] + - [0.0, 0.4, 0.4, 0.8, 0.8, 1.0] + - [1.0, 1.0, 1.0, 1.0, 1.0, 1.0] +teacher: + rm_keys: + - Arch.lr_mult_list + search_values: + - ResNet101_vd + - ResNet50_vd +final_replace: + Arch.lr_mult_list: Arch.models.1.Student.lr_mult_list + diff --git a/ppcls/configs/PULC/vehicle_attr/MobileNetV3_large_x1_0.yaml b/ppcls/configs/PULC/vehicle_attr/MobileNetV3_large_x1_0.yaml new file mode 100644 index 000000000..52f5dd8e8 --- /dev/null +++ b/ppcls/configs/PULC/vehicle_attr/MobileNetV3_large_x1_0.yaml @@ -0,0 +1,115 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: "./output/" + device: "gpu" + save_interval: 5 + eval_during_train: True + eval_interval: 1 + epochs: 30 + print_batch_step: 20 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 192, 256] + save_inference_dir: "./inference" + use_multilabel: True + +# model architecture +Arch: + name: "MobileNetV3_large_x1_0" + pretrained: True + class_num: 19 + infer_add_softmax: False + +# loss function config for traing/eval process +Loss: + Train: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + Eval: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.0005 + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/train_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - Padv2: + size: [276, 212] + pad_mode: 1 + fill_value: 0 + - RandomCropImage: + size: [256, 192] + - RandFlipImage: + flip_code: 1 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: True + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + Eval: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/test_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + + +Metric: + Eval: + - ATTRMetric: + + diff --git a/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml b/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml new file mode 100644 index 000000000..508f3a932 --- /dev/null +++ b/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0.yaml @@ -0,0 +1,149 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: "./output/" + device: "gpu" + save_interval: 5 + eval_during_train: True + eval_interval: 1 + epochs: 30 + print_batch_step: 20 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 192, 256] + save_inference_dir: "./inference" + use_multilabel: True + +# model architecture +Arch: + name: "PPLCNet_x1_0" + pretrained: True + class_num: 19 + use_ssld: True + lr_mult_list: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0] + infer_add_softmax: False + +# loss function config for traing/eval process +Loss: + Train: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + Eval: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.0125 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.0005 + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/train_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - TimmAutoAugment: + prob: 0.0 + config_str: rand-m9-mstd0.5-inc1 + interpolation: bicubic + img_size: [256, 192] + - Padv2: + size: [276, 212] + pad_mode: 1 + fill_value: 0 + - RandomCropImage: + size: [256, 192] + - RandFlipImage: + flip_code: 1 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - RandomErasing: + EPSILON: 0.5 + sl: 0.02 + sh: 1.0/3.0 + r1: 0.3 + attempt: 10 + use_log_aspect: True + mode: pixel + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: True + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + Eval: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/test_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + +Infer: + infer_imgs: ./deploy/images/PULC/vehicle_attr/0002_c002_00030670_0.jpg + batch_size: 10 + transforms: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - ToCHWImage: + PostProcess: + name: VehicleAttribute + color_threshold: 0.5 + type_threshold: 0.5 + +Metric: + Eval: + - ATTRMetric: + + diff --git a/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_distillation.yaml b/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_distillation.yaml new file mode 100644 index 000000000..c5144f373 --- /dev/null +++ b/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_distillation.yaml @@ -0,0 +1,150 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: "./output/" + device: "gpu" + save_interval: 5 + eval_during_train: True + eval_interval: 1 + epochs: 30 + print_batch_step: 20 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 192, 256] + save_inference_dir: "./inference" + use_multilabel: True + +# model architecture +Arch: + name: "DistillationModel" + class_num: &class_num 19 + # if not null, its lengths should be same as models + pretrained_list: + # if not null, its lengths should be same as models + freeze_params_list: + - True + - False + use_ssld: True + models: + - Teacher: + name: ResNet101_vd + class_num: *class_num + - Student: + name: PPLCNet_x1_0 + class_num: *class_num + pretrained: True + use_ssld: True + +# loss function config for traing/eval process +Loss: + Train: + - DistillationMultiLabelLoss: + weight: 1.0 + model_names: ["Student"] + weight_ratio: True + size_sum: True + - DistillationDMLLoss: + weight: 1.0 + weight_ratio: True + sum_across_class_dim: False + model_name_pairs: + - ["Student", "Teacher"] + + Eval: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.0005 + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/train_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - TimmAutoAugment: + prob: 0.0 + config_str: rand-m9-mstd0.5-inc1 + interpolation: bicubic + img_size: [256, 192] + - Padv2: + size: [276, 212] + pad_mode: 1 + fill_value: 0 + - RandomCropImage: + size: [256, 192] + - RandFlipImage: + flip_code: 1 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - RandomErasing: + EPSILON: 0.0 + sl: 0.02 + sh: 1.0/3.0 + r1: 0.3 + attempt: 10 + use_log_aspect: True + mode: pixel + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: True + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + Eval: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/test_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + + +Metric: + Eval: + - ATTRMetric: + + diff --git a/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_search.yaml b/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_search.yaml new file mode 100644 index 000000000..5f84c2a65 --- /dev/null +++ b/ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_search.yaml @@ -0,0 +1,129 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: "./output/" + device: "gpu" + save_interval: 5 + eval_during_train: True + eval_interval: 1 + epochs: 30 + print_batch_step: 20 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 192, 256] + save_inference_dir: "./inference" + use_multilabel: True + +# model architecture +Arch: + name: "PPLCNet_x1_0" + pretrained: True + use_ssld: True + class_num: 19 + infer_add_softmax: False + +# loss function config for traing/eval process +Loss: + Train: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + Eval: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.0005 + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/train_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - TimmAutoAugment: + prob: 0.0 + config_str: rand-m9-mstd0.5-inc1 + interpolation: bicubic + img_size: [256, 192] + - Padv2: + size: [276, 212] + pad_mode: 1 + fill_value: 0 + - RandomCropImage: + size: [256, 192] + - RandFlipImage: + flip_code: 1 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + - RandomErasing: + EPSILON: 0.0 + sl: 0.02 + sh: 1.0/3.0 + r1: 0.3 + attempt: 10 + use_log_aspect: True + mode: pixel + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: True + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + Eval: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/test_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 128 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + + +Metric: + Eval: + - ATTRMetric: + + diff --git a/ppcls/configs/PULC/vehicle_attr/Res2Net200_vd_26w_4s.yaml b/ppcls/configs/PULC/vehicle_attr/Res2Net200_vd_26w_4s.yaml new file mode 100644 index 000000000..c6618f960 --- /dev/null +++ b/ppcls/configs/PULC/vehicle_attr/Res2Net200_vd_26w_4s.yaml @@ -0,0 +1,122 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: "./output/mo" + device: "gpu" + save_interval: 5 + eval_during_train: True + eval_interval: 1 + epochs: 30 + print_batch_step: 20 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 192, 256] + save_inference_dir: "./inference" + use_multilabel: True + +# mixed precision training +AMP: + scale_loss: 128.0 + use_dynamic_loss_scaling: True + # O1: mixed fp16 + level: O1 + +# model architecture +Arch: + name: "Res2Net200_vd_26w_4s" + pretrained: True + class_num: 19 + infer_add_softmax: False + +# loss function config for traing/eval process +Loss: + Train: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + Eval: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.0005 + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/train_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - Padv2: + size: [276, 212] + pad_mode: 1 + fill_value: 0 + - RandomCropImage: + size: [256, 192] + - RandFlipImage: + flip_code: 1 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: True + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + Eval: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/test_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + + +Metric: + Eval: + - ATTRMetric: + + diff --git a/ppcls/configs/PULC/vehicle_attr/ResNet50.yaml b/ppcls/configs/PULC/vehicle_attr/ResNet50.yaml new file mode 100644 index 000000000..9218769c6 --- /dev/null +++ b/ppcls/configs/PULC/vehicle_attr/ResNet50.yaml @@ -0,0 +1,116 @@ +# global configs +Global: + checkpoints: null + pretrained_model: null + output_dir: "./output/" + device: "gpu" + save_interval: 5 + eval_during_train: True + eval_interval: 1 + epochs: 30 + print_batch_step: 20 + use_visualdl: False + # used for static mode and model export + image_shape: [3, 192, 256] + save_inference_dir: "./inference" + use_multilabel: True + +# model architecture +Arch: + name: "ResNet50" + pretrained: True + class_num: 19 + infer_add_softmax: False + +# loss function config for traing/eval process +Loss: + Train: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + Eval: + - MultiLabelLoss: + weight: 1.0 + weight_ratio: True + size_sum: True + + +Optimizer: + name: Momentum + momentum: 0.9 + lr: + name: Cosine + learning_rate: 0.01 + warmup_epoch: 5 + regularizer: + name: 'L2' + coeff: 0.0005 + +# data loader for train and eval +DataLoader: + Train: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/train_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - Padv2: + size: [276, 212] + pad_mode: 1 + fill_value: 0 + - RandomCropImage: + size: [256, 192] + - RandFlipImage: + flip_code: 1 + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: True + shuffle: True + loader: + num_workers: 8 + use_shared_memory: True + Eval: + dataset: + name: MultiLabelDataset + image_root: "dataset/VeRi/" + cls_label_path: "dataset/VeRi/test_list.txt" + label_ratio: True + transform_ops: + - DecodeImage: + to_rgb: True + channel_first: False + - ResizeImage: + size: [256, 192] + - NormalizeImage: + scale: 1.0/255.0 + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + order: '' + sampler: + name: DistributedBatchSampler + batch_size: 64 + drop_last: False + shuffle: False + loader: + num_workers: 8 + use_shared_memory: True + + +Metric: + Eval: + - ATTRMetric: + + diff --git a/ppcls/configs/PULC/vehicle_attr/search.yaml b/ppcls/configs/PULC/vehicle_attr/search.yaml new file mode 100644 index 000000000..d5f41a3cd --- /dev/null +++ b/ppcls/configs/PULC/vehicle_attr/search.yaml @@ -0,0 +1,34 @@ +base_config_file: ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_search.yaml +distill_config_file: ppcls/configs/PULC/vehicle_attr/PPLCNet_x1_0_distillation.yaml + +gpus: 0,1,2,3 +output_dir: output/search_vehicle_attr +search_times: 1 +search_dict: + - search_key: lrs + replace_config: + - Optimizer.lr.learning_rate + search_values: [0.0075, 0.01, 0.0125] + - search_key: ra_probs + replace_config: + - DataLoader.Train.dataset.transform_ops.2.TimmAutoAugment.prob + search_values: [0.0, 0.1, 0.5] + - search_key: re_probs + replace_config: + - DataLoader.Train.dataset.transform_ops.7.RandomErasing.EPSILON + search_values: [0.0, 0.1, 0.5] + - search_key: lr_mult_list + replace_config: + - Arch.lr_mult_list + search_values: + - [0.0, 0.2, 0.4, 0.6, 0.8, 1.0] + - [0.0, 0.4, 0.4, 0.8, 0.8, 1.0] + - [1.0, 1.0, 1.0, 1.0, 1.0, 1.0] +teacher: + rm_keys: + - Arch.lr_mult_list + search_values: + - ResNet101_vd + - ResNet50_vd +final_replace: + Arch.lr_mult_list: Arch.models.1.Student.lr_mult_list diff --git a/ppcls/data/postprocess/__init__.py b/ppcls/data/postprocess/__init__.py index 54678dc44..eafcf3f00 100644 --- a/ppcls/data/postprocess/__init__.py +++ b/ppcls/data/postprocess/__init__.py @@ -18,6 +18,7 @@ from . import topk, threshoutput from .topk import Topk, MultiLabelTopk from .threshoutput import ThreshOutput +from .attr_rec import VehicleAttribute def build_postprocess(config): diff --git a/ppcls/data/postprocess/attr_rec.py b/ppcls/data/postprocess/attr_rec.py new file mode 100644 index 000000000..cf0f7a59d --- /dev/null +++ b/ppcls/data/postprocess/attr_rec.py @@ -0,0 +1,71 @@ +# copyright (c) 2022 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import numpy as np +import paddle +import paddle.nn.functional as F + + +class VehicleAttribute(object): + def __init__(self, color_threshold=0.5, type_threshold=0.5): + self.color_threshold = color_threshold + self.type_threshold = type_threshold + self.color_list = [ + "yellow", "orange", "green", "gray", "red", "blue", "white", + "golden", "brown", "black" + ] + self.type_list = [ + "sedan", "suv", "van", "hatchback", "mpv", "pickup", "bus", + "truck", "estate" + ] + + def __call__(self, x, file_names=None): + if isinstance(x, dict): + x = x['logits'] + assert isinstance(x, paddle.Tensor) + if file_names is not None: + assert x.shape[0] == len(file_names) + x = F.sigmoid(x).numpy() + + # postprocess output of predictor + batch_res = [] + for idx, res in enumerate(x): + res = res.tolist() + label_res = [] + color_idx = np.argmax(res[:10]) + type_idx = np.argmax(res[10:]) + print(color_idx, type_idx) + if res[color_idx] >= self.color_threshold: + color_info = f"Color: ({self.color_list[color_idx]}, prob: {res[color_idx]})" + else: + color_info = "Color unknown" + + if res[type_idx + 10] >= self.type_threshold: + type_info = f"Type: ({self.type_list[type_idx]}, prob: {res[type_idx + 10]})" + else: + type_info = "Type unknown" + + label_res = f"{color_info}, {type_info}" + + threshold_list = [self.color_threshold + ] * 10 + [self.type_threshold] * 9 + pred_res = (np.array(res) > np.array(threshold_list) + ).astype(np.int8).tolist() + batch_res.append({ + "attr": label_res, + "pred": pred_res, + "file_name": file_names[idx] + }) + return batch_res diff --git a/ppcls/engine/engine.py b/ppcls/engine/engine.py index 8a34c1bda..8e8a20c2a 100644 --- a/ppcls/engine/engine.py +++ b/ppcls/engine/engine.py @@ -460,7 +460,7 @@ class Engine(object): assert self.mode == "export" use_multilabel = self.config["Global"].get( "use_multilabel", - False) and not "ATTRMetric" in self.config["Metric"]["Eval"][0] + False) and "ATTRMetric" in self.config["Metric"]["Eval"][0] model = ExportModel(self.config["Arch"], self.model, use_multilabel) if self.config["Global"]["pretrained_model"] is not None: load_dygraph_pretrain(model.base_model, diff --git a/ppcls/loss/__init__.py b/ppcls/loss/__init__.py index c1f2f95df..741eb3b61 100644 --- a/ppcls/loss/__init__.py +++ b/ppcls/loss/__init__.py @@ -24,6 +24,7 @@ from .distillationloss import DistillationDistanceLoss from .distillationloss import DistillationRKDLoss from .distillationloss import DistillationKLDivLoss from .distillationloss import DistillationDKDLoss +from .distillationloss import DistillationMultiLabelLoss from .multilabelloss import MultiLabelLoss from .afdloss import AFDLoss diff --git a/ppcls/loss/distillationloss.py b/ppcls/loss/distillationloss.py index c60a540db..4ca58d26d 100644 --- a/ppcls/loss/distillationloss.py +++ b/ppcls/loss/distillationloss.py @@ -22,6 +22,7 @@ from .distanceloss import DistanceLoss from .rkdloss import RKdAngle, RkdDistance from .kldivloss import KLDivLoss from .dkdloss import DKDLoss +from .multilabelloss import MultiLabelLoss class DistillationCELoss(CELoss): @@ -89,13 +90,16 @@ class DistillationDMLLoss(DMLLoss): def __init__(self, model_name_pairs=[], act="softmax", + weight_ratio=False, + sum_across_class_dim=False, key=None, name="loss_dml"): - super().__init__(act=act) + super().__init__(act=act, sum_across_class_dim=sum_across_class_dim) assert isinstance(model_name_pairs, list) self.key = key self.model_name_pairs = model_name_pairs self.name = name + self.weight_ratio = weight_ratio def forward(self, predicts, batch): loss_dict = dict() @@ -105,7 +109,10 @@ class DistillationDMLLoss(DMLLoss): if self.key is not None: out1 = out1[self.key] out2 = out2[self.key] - loss = super().forward(out1, out2) + if self.weight_ratio is True: + loss = super().forward(out1, out2, batch) + else: + loss = super().forward(out1, out2) if isinstance(loss, dict): for key in loss: loss_dict["{}_{}_{}_{}".format(key, pair[0], pair[1], @@ -122,6 +129,7 @@ class DistillationDistanceLoss(DistanceLoss): def __init__(self, mode="l2", model_name_pairs=[], + act=None, key=None, name="loss_", **kargs): @@ -130,6 +138,13 @@ class DistillationDistanceLoss(DistanceLoss): self.key = key self.model_name_pairs = model_name_pairs self.name = name + mode + assert act in [None, "sigmoid", "softmax"] + if act == "sigmoid": + self.act = nn.Sigmoid() + elif act == "softmax": + self.act = nn.Softmax(axis=-1) + else: + self.act = None def forward(self, predicts, batch): loss_dict = dict() @@ -139,6 +154,9 @@ class DistillationDistanceLoss(DistanceLoss): if self.key is not None: out1 = out1[self.key] out2 = out2[self.key] + if self.act is not None: + out1 = self.act(out1) + out2 = self.act(out2) loss = super().forward(out1, out2) for key in loss: loss_dict["{}_{}_{}".format(self.name, key, idx)] = loss[key] @@ -235,3 +253,34 @@ class DistillationDKDLoss(DKDLoss): loss = super().forward(out1, out2, batch) loss_dict[f"{self.name}_{pair[0]}_{pair[1]}"] = loss return loss_dict + + +class DistillationMultiLabelLoss(MultiLabelLoss): + """ + DistillationMultiLabelLoss + """ + + def __init__(self, + model_names=[], + epsilon=None, + size_sum=False, + weight_ratio=False, + key=None, + name="loss_mll"): + super().__init__( + epsilon=epsilon, size_sum=size_sum, weight_ratio=weight_ratio) + assert isinstance(model_names, list) + self.key = key + self.model_names = model_names + self.name = name + + def forward(self, predicts, batch): + loss_dict = dict() + for name in self.model_names: + out = predicts[name] + if self.key is not None: + out = out[self.key] + loss = super().forward(out, batch) + for key in loss: + loss_dict["{}_{}".format(key, name)] = loss[key] + return loss_dict diff --git a/ppcls/loss/dmlloss.py b/ppcls/loss/dmlloss.py index 48bf6c024..e8983ed08 100644 --- a/ppcls/loss/dmlloss.py +++ b/ppcls/loss/dmlloss.py @@ -16,13 +16,15 @@ import paddle import paddle.nn as nn import paddle.nn.functional as F +from ppcls.loss.multilabelloss import ratio2weight + class DMLLoss(nn.Layer): """ DMLLoss """ - def __init__(self, act="softmax", eps=1e-12): + def __init__(self, act="softmax", sum_across_class_dim=False, eps=1e-12): super().__init__() if act is not None: assert act in ["softmax", "sigmoid"] @@ -33,6 +35,7 @@ class DMLLoss(nn.Layer): else: self.act = None self.eps = eps + self.sum_across_class_dim = sum_across_class_dim def _kldiv(self, x, target): class_num = x.shape[-1] @@ -40,11 +43,20 @@ class DMLLoss(nn.Layer): (target + self.eps) / (x + self.eps)) * class_num return cost - def forward(self, x, target): + def forward(self, x, target, gt_label=None): if self.act is not None: x = self.act(x) target = self.act(target) loss = self._kldiv(x, target) + self._kldiv(target, x) loss = loss / 2 - loss = paddle.mean(loss) + + # for multi-label dml loss + if gt_label is not None: + gt_label, label_ratio = gt_label[:, 0, :], gt_label[:, 1, :] + targets_mask = paddle.cast(gt_label > 0.5, 'float32') + weight = ratio2weight(targets_mask, paddle.to_tensor(label_ratio)) + weight = weight * (gt_label > -1) + loss = loss * weight + + loss = loss.sum(1).mean() if self.sum_across_class_dim else loss.mean() return {"DMLLoss": loss}