From faf05fe358324a43f9d27778f3be0d77654d8cbb Mon Sep 17 00:00:00 2001
From: RunningLeon <mnsheng@yeah.net>
Date: Wed, 24 May 2023 19:59:47 +0800
Subject: [PATCH] fix prebuild uploading (#2112)
* fix
* change to test mode
* remove
* fix
* fix
* fix
* test pypi
* test
* add publish
* test
* revert to prod
* update
---
.github/workflows/prebuild.yml | 147 +++++++------------
.github/workflows/publish.yml | 57 +++++++
tools/package_tools/test/test_sdk.ps1 | 2 +-
tools/package_tools/test/test_sdk_python.ps1 | 2 +-
tools/package_tools/test/test_sdk_python.sh | 2 +-
5 files changed, 110 insertions(+), 100 deletions(-)
create mode 100644 .github/workflows/publish.yml
diff --git a/.github/workflows/prebuild.yml b/.github/workflows/prebuild.yml
index 86f6d8745..07a19d99c 100644
--- a/.github/workflows/prebuild.yml
+++ b/.github/workflows/prebuild.yml
@@ -36,7 +36,7 @@ jobs:
export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
echo $MMDEPLOY_VERSION
echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
- echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV
+ echo "OUTPUT_DIR=/__w/mmdeploy/prebuild/$MMDEPLOY_VERSION" >> $GITHUB_ENV
- name: Build MMDeploy
run: |
source activate mmdeploy-3.6
@@ -61,10 +61,20 @@ jobs:
--system linux --output config.yml --device cuda --build-sdk --build-sdk-monolithic \
--build-sdk-python --sdk-dynamic-net --onnxruntime-dir=$ONNXRUNTIME_GPU_DIR
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
+ - name: Zip mmdeploy sdk
+ run: |
+ cd pack/sdk
+ for folder in *
+ do
+ tar czf $folder.tar.gz $folder
+ done
- name: Move artifact
run: |
- mkdir -p /__w/mmdeploy/prebuild/$OUTPUT_DIR
- cp -r pack/* /__w/mmdeploy/prebuild/$OUTPUT_DIR
+ mkdir -p $OUTPUT_DIR/mmdeploy $OUTPUT_DIR/mmdeploy_runtime $OUTPUT_DIR/sdk
+ chmod -R 777 $OUTPUT_DIR
+ mv -vf pack/mmdeploy/* $OUTPUT_DIR/mmdeploy/
+ mv -vf pack/mmdeploy_runtime/* $OUTPUT_DIR/mmdeploy_runtime/
+ mv -vf pack/sdk/* $OUTPUT_DIR/sdk/
linux_build_cxx11abi:
if: inputs.run == true || ${{ github.event_name == 'push' }}
@@ -84,7 +94,7 @@ jobs:
export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
echo $MMDEPLOY_VERSION
echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
- echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV
+ echo "OUTPUT_DIR=/__w/mmdeploy/prebuild/$MMDEPLOY_VERSION" >> $GITHUB_ENV
- name: Build sdk cpu backend
run: |
mkdir pack; cd pack
@@ -99,10 +109,18 @@ jobs:
--system linux --output config.yml --device cuda --build-sdk --build-sdk-monolithic \
--sdk-dynamic-net --cxx11abi --onnxruntime-dir=$ONNXRUNTIME_GPU_DIR --cudnn-dir /usr
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
+ - name: Zip mmdeploy sdk
+ run: |
+ cd pack/sdk
+ for folder in *
+ do
+ tar czf $folder.tar.gz $folder
+ done
- name: Move artifact
run: |
- mkdir -p /__w/mmdeploy/prebuild/$OUTPUT_DIR
- cp -r pack/* /__w/mmdeploy/prebuild/$OUTPUT_DIR
+ mkdir -p $OUTPUT_DIR/sdk
+ chmod -R 777 $OUTPUT_DIR
+ mv -vf pack/sdk/* $OUTPUT_DIR/sdk/
linux_test:
if: inputs.run == true || ${{ github.event_name == 'push' }}
@@ -124,60 +142,21 @@ jobs:
export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
echo $MMDEPLOY_VERSION
echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
- echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV
+ echo "OUTPUT_DIR=/__w/mmdeploy/prebuild/$MMDEPLOY_VERSION" >> $GITHUB_ENV
- name: Test python
run: |
- cd /__w/mmdeploy/prebuild/$OUTPUT_DIR
+ cd $OUTPUT_DIR
bash $GITHUB_WORKSPACE/tools/package_tools/test/test_sdk_python.sh
- name: Test c/cpp
run: |
- cd /__w/mmdeploy/prebuild/$OUTPUT_DIR
+ cd $OUTPUT_DIR
bash $GITHUB_WORKSPACE/tools/package_tools/test/test_sdk.sh
- linux_upload:
- if: inputs.run == true || ${{ github.event_name == 'push' }}
- runs-on: [self-hosted, linux-3090]
- environment: 'prod'
- needs: linux_test
- env:
- PREBUILD_DIR: /data2/actions-runner/prebuild
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
- - name: Get mmdeploy version
- run: |
- export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
- echo $MMDEPLOY_VERSION
- echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
- echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV
- - name: Upload mmdeploy
- run: |
- cd $PREBUILD_DIR/$OUTPUT_DIR/mmdeploy
- pip install twine
- # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
- twine upload * -u __token__ -p ${{ secrets.pypi_password }}
- - name: Upload mmdeploy_runtime
- run: |
- cd $PREBUILD_DIR/$OUTPUT_DIR/mmdeploy_runtime
- # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
- twine upload * -u __token__ -p ${{ secrets.pypi_password }}
- - name: Zip mmdeploy sdk
- run: |
- cd $PREBUILD_DIR/$OUTPUT_DIR/sdk
- for folder in *
- do
- tar czf $folder.tar.gz $folder
- done
- - name: Upload mmdeploy sdk
- uses: softprops/action-gh-release@v1
- with:
- files: |
- $PREBUILD_DIR/$OUTPUT_DIR/sdk/*.tar.gz
-
-
windows_build:
if: inputs.run == true || ${{ github.event_name == 'push' }}
runs-on: [self-hosted, win10-3080]
+ env:
+ PREBUILD_DIR: D:\Runners\prebuild
steps:
- name: Checkout repository
uses: actions/checkout@v3
@@ -189,12 +168,13 @@ jobs:
$env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
echo $env:MMDEPLOY_VERSION
echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
- echo "OUTPUT_DIR=$env:MMDEPLOY_VERSION-$env:GITHUB_RUN_ID" >> $env:GITHUB_ENV
+ echo "OUTPUT_DIR=$env:PREBUILD_DIR\$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
- name: Build MMDeploy
run: |
. D:\DEPS\cienv\prebuild_gpu_env.ps1
conda activate mmdeploy-3.6
- mkdir pack; cd pack
+ New-Item -Path pack -ItemType Directory -Force
+ cd pack
python ../tools/package_tools/generate_build_config.py --backend 'trt;ort' `
--system windows --output config.yml --build-mmdeploy
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
@@ -216,15 +196,28 @@ jobs:
--system windows --output config.yml --device cuda --build-sdk --build-sdk-monolithic `
--build-sdk-python --sdk-dynamic-net
python ../tools/package_tools/mmdeploy_builder.py --config config.yml
+ - name: Zip mmdeploy sdk
+ run: |
+ cd pack/sdk
+ $folders = $(ls).Name
+ foreach ($folder in $folders) {
+ Compress-Archive -Path $folder -DestinationPath "$folder.zip"
+ }
- name: Move artifact
run: |
- New-Item "D:/DEPS/ciartifact/$env:OUTPUT_DIR" -ItemType Directory -Force
- Move-Item pack/* "D:/DEPS/ciartifact/$env:OUTPUT_DIR"
+ New-Item "$env:OUTPUT_DIR\mmdeploy" -ItemType Directory -Force
+ New-Item "$env:OUTPUT_DIR\mmdeploy_runtime" -ItemType Directory -Force
+ New-Item "$env:OUTPUT_DIR\sdk" -ItemType Directory -Force
+ Move-Item pack/mmdeploy/* -Force "$env:OUTPUT_DIR\mmdeploy\"
+ Move-Item pack/mmdeploy_runtime/* -Force "$env:OUTPUT_DIR\mmdeploy_runtime\"
+ Move-Item pack/sdk/* -Force "$env:OUTPUT_DIR\sdk\"
windows_test:
if: inputs.run == true || ${{ github.event_name == 'push' }}
runs-on: [self-hosted, win10-3080]
needs: windows_build
+ env:
+ PREBUILD_DIR: D:\Runners\prebuild
steps:
- name: Checkout repository
uses: actions/checkout@v3
@@ -234,55 +227,15 @@ jobs:
$env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
echo $env:MMDEPLOY_VERSION
echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
- echo "OUTPUT_DIR=$env:MMDEPLOY_VERSION-$env:GITHUB_RUN_ID" >> $env:GITHUB_ENV
+ echo "OUTPUT_DIR=$env:PREBUILD_DIR\$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
- name: Test python
run: |
- cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR"
+ cd "$env:OUTPUT_DIR"
. D:\DEPS\cienv\prebuild_cpu_env.ps1
conda activate ci-test
& "$env:GITHUB_WORKSPACE/tools/package_tools/test/test_sdk_python.ps1"
- name: Test c/cpp
run: |
- cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR"
+ cd "$env:OUTPUT_DIR"
. D:\DEPS\cienv\prebuild_cpu_env.ps1
& "$env:GITHUB_WORKSPACE/tools/package_tools/test/test_sdk.ps1"
-
- windows_upload:
- if: inputs.run == true || ${{ github.event_name == 'push' }}
- runs-on: [self-hosted, win10-3080]
- environment: 'prod'
- needs: windows_test
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
- - name: Get mmdeploy version
- run: |
- conda activate mmdeploy-3.8
- $env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
- echo $env:MMDEPLOY_VERSION
- echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV
- echo "OUTPUT_DIR=$env:MMDEPLOY_VERSION-$env:GITHUB_RUN_ID" >> $env:GITHUB_ENV
- - name: Upload mmdeploy
- run: |
- cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR/mmdeploy"
- conda activate mmdeploy-3.8
- # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
- twine upload * -u __token__ -p ${{ secrets.pypi_password }}
- - name: Upload mmdeploy_runtime
- run: |
- cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR/mmdeploy_runtime"
- conda activate mmdeploy-3.8
- # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }}
- twine upload * -u __token__ -p ${{ secrets.pypi_password }}
- - name: Zip mmdeploy sdk
- run: |
- cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR/sdk"
- $folders = $(ls).Name
- foreach ($folder in $folders) {
- Compress-Archive -Path $folder -DestinationPath "$folder.zip"
- }
- - name: Upload mmdeploy sdk
- uses: softprops/action-gh-release@v1
- with:
- files: |
- D:/DEPS/ciartifact/$env:OUTPUT_DIR/sdk/*.zip
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 000000000..a720afad8
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,57 @@
+name: publish
+
+on:
+ push:
+ tags:
+ - "v*.*.*"
+
+ workflow_dispatch:
+ inputs:
+ publish:
+ description: 'Manually publish'
+ required: false
+ type: boolean
+ default: false
+
+jobs:
+ publish:
+ if: inputs.publish == true || ${{ github.event_name == 'push' }}
+ runs-on: [self-hosted, linux-3090]
+ environment: 'prod'
+ env:
+ PREBUILD_DIR: /data2/actions-runner/prebuild
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+ - name: Get mmdeploy version
+ run: |
+ export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)")
+ echo $MMDEPLOY_VERSION
+ echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV
+ echo "OUTPUT_DIR=$PREBUILD_DIR/$MMDEPLOY_VERSION" >> $GITHUB_ENV
+ pip install twine
+ - name: Upload mmdeploy
+ continue-on-error: true
+ run: |
+ cd $OUTPUT_DIR/mmdeploy
+ ls -sha *.whl
+ # twine upload mmdeploy-${MMDEPLOY_VERSION}-py3-none-manylinux2014_x86_64.whl --repository testpypi -u testmmdeploy -p ${{ secrets.test_pypi_password }}
+ twine upload *.whl -u __token__ -p ${{ secrets.pypi_password }}
+ - name: Upload mmdeploy_runtime
+ continue-on-error: true
+ run: |
+ cd $OUTPUT_DIR/mmdeploy_runtime
+ ls -sha *.whl
+ # twine upload mmdeploy_runtime-${MMDEPLOY_VERSION}-cp38-none-manylinux2014_x86_64.whl --repository testpypi -u testmmdeploy -p ${{ secrets.test_pypi_password }}
+ twine upload *.whl -u __token__ -p ${{ secrets.pypi_password }}
+ - name: Create dummy softlinks to assets
+ run: |
+ ln -sf $OUTPUT_DIR/sdk ./prebuild
+ ls -sha ./prebuild
+ - name: Upload mmdeploy sdk
+ uses: softprops/action-gh-release@v1
+ with:
+ tag_name: v${MMDEPLOY_VERSION}
+ files: |
+ ./prebuild/*.zip
+ ./prebuild/*.tar.gz
diff --git a/tools/package_tools/test/test_sdk.ps1 b/tools/package_tools/test/test_sdk.ps1
index 1bf290868..299cd88a6 100644
--- a/tools/package_tools/test/test_sdk.ps1
+++ b/tools/package_tools/test/test_sdk.ps1
@@ -15,7 +15,7 @@ $pkgs = $(ls).Name
$test_pkg = $pkgs[0]
if ($pkgs.Count -gt 1) {
foreach ($pkg in $pkgs) {
- if ($pkg -like '*cpu*') {
+ if ($pkg -like '*-windows-amd64') {
$test_pkg = $pkg
break
}
diff --git a/tools/package_tools/test/test_sdk_python.ps1 b/tools/package_tools/test/test_sdk_python.ps1
index 894b917ea..12caacb9e 100644
--- a/tools/package_tools/test/test_sdk_python.ps1
+++ b/tools/package_tools/test/test_sdk_python.ps1
@@ -15,7 +15,7 @@ $pkgs = $(ls).Name
$test_pkg = ""
if ($pkgs.Count -gt 1) {
foreach ($pkg in $pkgs) {
- if ($pkg -like 'mmdeploy_runtime-*cp38*') {
+ if ($pkg -like 'mmdeploy_runtime-*cp38*-win_amd64.whl') {
$test_pkg = $pkg
break
}
diff --git a/tools/package_tools/test/test_sdk_python.sh b/tools/package_tools/test/test_sdk_python.sh
index e2c491605..1bc198e6d 100755
--- a/tools/package_tools/test/test_sdk_python.sh
+++ b/tools/package_tools/test/test_sdk_python.sh
@@ -12,7 +12,7 @@ cd $WORKSPACE
cd $SDK_PYTHON_DIR
PY_VERSION=$(python3 -V | awk '{print $2}' | awk '{split($0, a, "."); print a[1]a[2]}')
-test_pkg=$(ls | grep mmdeploy_runtime-*cp${PY_VERSION}*)
+test_pkg=$(ls | grep mmdeploy_runtime-*cp${PY_VERSION}*x86_64.whl)
python3 -m pip install $test_pkg --force-reinstall
python3 -m pip install opencv-python