name: regression test

on:
  workflow_dispatch:
    inputs:
      repository:
        required: false
        description: 'Tested repository. Default is open-mmlab'
        type: string
        default: open-mmlab
      rep_ref:
        required: false
        description: 'Tested repository ref, could be branch, tag or SHA'
        type: string
        default: ''
      test_linux:
        required: true
        description: 'Whether to start regression test on Linux x86_64'
        type: boolean
        default: false
      test_windows:
        required: true
        description: 'Whether to start regression test on Windows'
        type: boolean
        default: false
      torch_versions:
        required: true
        description: 'Tested torch versions. Default is ["1.10.0"]'
        type: string
        default: "['1.10.0']"
      models:
        required: true
        description: 'Tested model list, eg: "resnet yolov3". Default is "all".\r\n Example: resnet ssd yolov5 maskrcnn srcnn pointpillars dbnet crnn hrnet fastscnn slowfast RotatedRetinanet'
        type: string
        default: all
      codebases:
        required: true
        description: "Tested codebase list, eg: ['mmpretrain','mmdet','mmagic','mmseg','mmpose','mmocr','mmdet3d','mmaction','mmyolo','mmrotate']"
        type: string
        default: "['mmpretrain']"
      backends:
        required: true
        description: 'Tested codebase list, eg: "onnxruntime tensorrt ncnn torchscript openvino".'
        type: string
        default: "onnxruntime tensorrt"
      run_performance:
        required: false
        description: 'Whether to test performance in regression test. "-p" means run, "" means do not run.'
        type: string
        default: ''
      mapped_domain:
        required: false
        description: 'Do not change it unless you know what you are doing!'
        type: string
        default: 'https://8d17-103-108-182-56.ngrok-free.app'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:
  reg-linux:
    if: inputs.test_linux == true
    runs-on: [self-hosted, linux-3090]
    timeout-minutes: 4320 # 72hours
    environment: 'prod'
    container:
      image: openmmlab/mmdeploy:ubuntu20.04-cuda11.3
      options: "--gpus=all --ipc=host"
      volumes:
        - /data2/checkpoints:/__w/mmdeploy/mmdeploy_checkpoints
        - /data2/benchmark:/__w/mmdeploy/data
        - /data2/pip-cache:/root/.cache/pip
        - /data2/regression_log/convert_log:/__w/mmdeploy/convert_log
    defaults:
      run:
        shell: bash -el {0}
    strategy:
      matrix:
        codebase: ${{ fromJSON(github.event.inputs.codebases) }}
        torch_version:  ${{ fromJSON(github.event.inputs.torch_versions) }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          repository: '${{ github.event.inputs.repository}}/mmdeploy'
          ref: ${{ github.event.inputs.rep_ref}}
          submodules: recursive
      - name: Prepare resources
        run: |
          echo "============================== Inputs =============================="
          echo "${{github.event.inputs.test_linux}}"
          echo "${{github.event.inputs.test_windows}}"
          echo "${{github.event.inputs.models}}"
          echo "${{github.event.inputs.backends}}"
          echo "${{github.event.inputs.run_performance}}"
          echo "$GITHUB_RUN_ID"
          echo "${{matrix.codebase}}"
          echo "${{matrix.torch_version}}"
          echo ""
          export REGRESSION_DIR=/__w/mmdeploy/convert_log/${GITHUB_RUN_ID}-linux
          export MMDEPLOY_DIR=$(pwd)
          mkdir -p $REGRESSION_DIR
          echo "REGRESSION_DIR=$REGRESSION_DIR"  >> $GITHUB_ENV
          echo "MMDEPLOY_DIR=$MMDEPLOY_DIR"  >> $GITHUB_ENV
          echo "BACKENDS=${{github.event.inputs.backends}}" >> $GITHUB_ENV
          ln -sf /usr/bin/python3 /usr/bin/python
          ln -sf /__w/mmdeploy/data $MMDEPLOY_DIR/data
      - name: Install Dependencies
        run: |
          apt update && apt install unzip
          python -V
          python -m pip install --upgrade pip
          python -m pip install openmim numpy pycuda xlsxwriter packaging prettytable
          python -m pip install opencv-python==4.5.4.60 opencv-python-headless==4.5.4.60 opencv-contrib-python==4.5.4.60
          python .github/scripts/prepare_reg_test.py --torch-version ${{ matrix.torch_version }} --codebases ${{ matrix.codebase}}
          python -m pip install -r requirements.txt
          python -m pip list
      - name: Install mmdeploy with sdk
        run: |
          export Torch_DIR=$(python -c "import torch;print(torch.utils.cmake_prefix_path + '/Torch')")
          export LD_LIBRARY_PATH="$MMDEPLOY_DIR/build/lib:${LD_LIBRARY_PATH}"
          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"  >> $GITHUB_ENV
          export BACKENDS=${BACKENDS//' '/;}
          export BACKENDS=${BACKENDS//tensorrt/trt}
          export BACKENDS=${BACKENDS//onnxruntime/ort}
          echo "BACKENDS = $BACKENDS"
          mkdir -p build && cd build
          cmake .. \
              -DMMDEPLOY_BUILD_SDK=ON \
              -DMMDEPLOY_BUILD_SDK_PYTHON_API=ON \
              -DMMDEPLOY_TARGET_DEVICES="cpu;cuda" \
              -DMMDEPLOY_TARGET_BACKENDS="$BACKENDS" \
              -DMMDEPLOY_CODEBASES="all" \
              -Dpplcv_DIR=${pplcv_DIR} \
              -DTENSORRT_DIR=${TENSORRT_DIR} \
              -DONNXRUNTIME_DIR=${ONNXRUNTIME_DIR} \
              -Dncnn_DIR=${ncnn_DIR} \
              -DTorch_DIR=${Torch_DIR}

          make -j$(nproc) && make install
          cd -
          ls build/lib
          rm -rf .eggs && python -m pip install -e .
          python tools/check_env.py
      - name: Run regression test
        run: |
          export work_dir=${REGRESSION_DIR}/${{matrix.codebase}}/torch${{matrix.torch_version}}
          export log_path=${work_dir}/convert_log.txt
          mkdir -p $work_dir
          ln -sf $work_dir ./mmdeploy_regression_dir
          python tools/check_env.py 2>&1 | tee ${log_dir}/check_env_log.txt
          python tools/regression_test.py \
              --codebase ${{matrix.codebase}} \
              --work-dir mmdeploy_regression_dir \
              --device cuda:0 \
              --models ${{github.event.inputs.models}} \
              --backends ${{github.event.inputs.backends}} \
              ${{github.event.inputs.run_performance}} 2>&1 | tee ${log_path}
          echo "Saved to $REGRESSION_DIR"
      - name: Check regression result
        continue-on-error: true
        run: |
          export url_prefix=${{github.event.inputs.mapped_domain}}/convert_log/${GITHUB_RUN_ID}-linux/${{matrix.codebase}}/torch${{matrix.torch_version}}
          echo "## ${{matrix.codebase}} + [torch${{matrix.torch_version}}](${url_prefix})" >> $GITHUB_STEP_SUMMARY
          export report_prefix="${REGRESSION_DIR}/${{matrix.codebase}}/torch${{matrix.torch_version}}/${{matrix.codebase}}_report"
          python .github/scripts/gen_reg_summary.py ${report_prefix}.xlsx ${url_prefix}
          cat ${report_prefix}.md >> $GITHUB_STEP_SUMMARY

  reg-windows:
    if: inputs.test_windows == true
    timeout-minutes: 4320 # 72hours
    environment: 'prod'
    runs-on: [self-hosted, win10-3080]
    env:
      BASE_ENV: cuda11.3-cudnn8.2-py3.8-torch1.10
      DATASET_DIR: D:\reg-test\data
      REGRESSION_DIR: D:\reg-test\regression_log
      CHECKPOINT_DIR: D:\reg-test\checkpoints
    defaults:
      run:
        shell: powershell
    strategy:
      matrix:
        codebase: ${{ fromJSON(github.event.inputs.codebases) }}
        torch_version:  ${{ fromJSON(github.event.inputs.torch_versions) }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          submodules: recursive
      - name: Prepare resources
        run: |
          echo "============================== Inputs =============================="
          echo "${{github.event.inputs.test_linux}}"
          echo "${{github.event.inputs.test_windows}}"
          echo "${{github.event.inputs.models}}"
          echo "${{github.event.inputs.backends}}"
          echo "${{github.event.inputs.run_performance}}"
          echo "$env:GITHUB_RUN_ID"
          echo "${{matrix.codebase}}"
          echo "${{matrix.torch_version}}"
          echo ""
          echo "============================== Prepare dataset =============================="
          New-Item -Path $pwd/data -ItemType SymbolicLink -Target $env:DATASET_DIR -Force
          New-Item -Path $pwd/../mmdeploy_checkpoints -ItemType SymbolicLink -Target $env:CHECKPOINT_DIR -Force
          ls data
          ls ../mmdeploy_checkpoints
          echo "============================== Prepare env =============================="
          $env:TEMP_ENV = "$pwd/../temp_envs/$env:GITHUB_RUN_ID"
          New-Item -Path "$env:TEMP_ENV" -ItemType Directory -Force
          echo "TEMP_ENV=$env:TEMP_ENV" >> $env:GITHUB_ENV
      - name: Setup Python Environment
        run: |
          echo "============================== Info =============================="
          echo "env:path= $env:path"
          echo "============================== Info =============================="
          conda info
          conda info -e
          conda create -p $env:TEMP_ENV --clone $env:BASE_ENV -y
          conda activate $env:TEMP_ENV
          python -V
          python -m pip install --upgrade pip
          python -m pip install openmim numpy pycuda xlsxwriter packaging prettytable
          python -m pip install opencv-python==4.5.4.60 opencv-python-headless==4.5.4.60 opencv-contrib-python==4.5.4.60
          python .github/scripts/prepare_reg_test.py --torch-version ${{ matrix.torch_version }} --codebases ${{ matrix.codebase}}
          python -m pip install -r requirements.txt
          python -m pip list
      - name: Build mmdeploy
        run: |
          conda activate $env:TEMP_ENV
          python -V
          $env:BACKENDS = "${{github.event.inputs.backends}}"
          $env:BACKENDS=% {$env:BACKENDS -replace " ", ";"}
          $env:BACKENDS=% {$env:BACKENDS -replace "tensorrt", "trt"}
          $env:BACKENDS=% {$env:BACKENDS -replace "onnxruntime", "ort"}
          echo "env:BACKENDS = $env:BACKENDS"
          $env:Torch_DIR=% {python -c "import torch;print(torch.utils.cmake_prefix_path + '\Torch')"}
          echo "Torch_DIR=$env:Torch_DIR" >> $env:GITHUB_ENV
          echo "BACKENDS=$env:BACKENDS" >> $env:GITHUB_ENV
          New-Item -Path build -ItemType Directory -Force
          cd build
          cmake ..  -A x64 -T v142 `
            -DMMDEPLOY_BUILD_TEST=OFF `
            -DMMDEPLOY_BUILD_SDK_CSHARP_API=ON `
            -DMMDEPLOY_BUILD_SDK_PYTHON_API=ON `
            -DMMDEPLOY_BUILD_SDK=ON `
            -DMMDEPLOY_TARGET_DEVICES="cpu;cuda" `
            -DMMDEPLOY_TARGET_BACKENDS="$env:BACKENDS" `
            -DMMDEPLOY_CODEBASES="all" `
            -Dpplcv_DIR="$env:PPLCV_DIR\pplcv-build\install\lib\cmake\ppl" `
            -DOpenCV_DIR="$env:OPENCV_DIR\build\x64\vc15\lib" `
            -DTENSORRT_DIR="$env:TENSORRT_DIR" `
            -DTorch_DIR="$env:Torch_DIR" `
            -DONNXRUNTIME_DIR="$env:ONNXRUNTIME_DIR" `
            -DMMDEPLOY_BUILD_EXAMPLES=OFF `
            -DCUDNN_DIR="$env:CUDNN_DIR"

          cmake --build . --config Release -- /m
          cmake --install . --config Release
          ls $pwd\bin\Release
          ls $pwd\lib\Release
      - name: Install mmdeploy converter
        run: |
          conda activate $env:TEMP_ENV
          $env:path = "$pwd\build\bin\Release;" + $env:path
          python -m pip install -e .
          python .\tools\check_env.py
      - name: Run regression test
        run: |
          conda activate $env:TEMP_ENV
          $env:path = "$pwd\build\bin\Release;" + $env:path
          python -V
          $env:CONVERT_DIR = "$env:REGRESSION_DIR\convert_log\$env:GITHUB_RUN_ID-windows\${{matrix.codebase}}\torch${{matrix.torch_version}}"
          echo "env:CONVERT_DIR = $env:CONVERT_DIR"
          New-Item -Path "$env:CONVERT_DIR" -ItemType Directory -Force
          New-Item -Path mmdeploy_regression_dir -ItemType SymbolicLink -Target $env:CONVERT_DIR -Force
          python tools/regression_test.py `
              --codebase ${{matrix.codebase}} `
              --work-dir mmdeploy_regression_dir `
              --device cuda:0 `
              --models ${{github.event.inputs.models}} `
              --backends ${{github.event.inputs.backends}} `
              ${{github.event.inputs.run_performance}}
          echo "Saved to $env:CONVERT_DIR"
      - name: Check regression result
        continue-on-error: true
        run: |
          conda activate $env:TEMP_ENV
          python -V
          $env:URL_PREFIX = "${{github.event.inputs.mapped_domain}}/convert_log/$env:GITHUB_RUN_ID-windows/${{matrix.codebase}}/torch${{matrix.torch_version}}"
          echo "URL_PREFIX = $env:URL_PREFIX"
          $env:REPORT_PREFIX = "$env:REGRESSION_DIR\convert_log\$env:GITHUB_RUN_ID-windows\${{matrix.codebase}}\torch${{matrix.torch_version}}\${{matrix.codebase}}_report"
          echo "REPORT_PREFIX = $env:REPORT_PREFIX"
          echo "## ${{matrix.codebase}} + [torch${{matrix.torch_version}}]($env:URL_PREFIX)" >> $env:GITHUB_STEP_SUMMARY
          python .github/scripts/gen_reg_summary.py "$env:REPORT_PREFIX.xlsx" $env:URL_PREFIX
          cat "$env:REPORT_PREFIX.md" >> $env:GITHUB_STEP_SUMMARY
      - name: Clear working directory
        if: always()
        run: |
          conda env remove --prefix "$env:TEMP_ENV"
          Remove-Item -Path "$pwd\data" -Force -Recurse
          Remove-Item -Path "$pwd\build" -Force -Recurse
          Remove-Item -Path "$pwd\mmdeploy_regression_dir" -Force -Recurse
