diff --git a/.github/workflows/prebuild.yml b/.github/workflows/prebuild.yml index 86f6d8745..07a19d99c 100644 --- a/.github/workflows/prebuild.yml +++ b/.github/workflows/prebuild.yml @@ -36,7 +36,7 @@ jobs: export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") echo $MMDEPLOY_VERSION echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV - echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV + echo "OUTPUT_DIR=/__w/mmdeploy/prebuild/$MMDEPLOY_VERSION" >> $GITHUB_ENV - name: Build MMDeploy run: | source activate mmdeploy-3.6 @@ -61,10 +61,20 @@ jobs: --system linux --output config.yml --device cuda --build-sdk --build-sdk-monolithic \ --build-sdk-python --sdk-dynamic-net --onnxruntime-dir=$ONNXRUNTIME_GPU_DIR python ../tools/package_tools/mmdeploy_builder.py --config config.yml + - name: Zip mmdeploy sdk + run: | + cd pack/sdk + for folder in * + do + tar czf $folder.tar.gz $folder + done - name: Move artifact run: | - mkdir -p /__w/mmdeploy/prebuild/$OUTPUT_DIR - cp -r pack/* /__w/mmdeploy/prebuild/$OUTPUT_DIR + mkdir -p $OUTPUT_DIR/mmdeploy $OUTPUT_DIR/mmdeploy_runtime $OUTPUT_DIR/sdk + chmod -R 777 $OUTPUT_DIR + mv -vf pack/mmdeploy/* $OUTPUT_DIR/mmdeploy/ + mv -vf pack/mmdeploy_runtime/* $OUTPUT_DIR/mmdeploy_runtime/ + mv -vf pack/sdk/* $OUTPUT_DIR/sdk/ linux_build_cxx11abi: if: inputs.run == true || ${{ github.event_name == 'push' }} @@ -84,7 +94,7 @@ jobs: export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") echo $MMDEPLOY_VERSION echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV - echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV + echo "OUTPUT_DIR=/__w/mmdeploy/prebuild/$MMDEPLOY_VERSION" >> $GITHUB_ENV - name: Build sdk cpu backend run: | mkdir pack; cd pack @@ -99,10 +109,18 @@ jobs: --system linux --output config.yml --device cuda --build-sdk --build-sdk-monolithic \ --sdk-dynamic-net --cxx11abi --onnxruntime-dir=$ONNXRUNTIME_GPU_DIR --cudnn-dir /usr python ../tools/package_tools/mmdeploy_builder.py --config config.yml + - name: Zip mmdeploy sdk + run: | + cd pack/sdk + for folder in * + do + tar czf $folder.tar.gz $folder + done - name: Move artifact run: | - mkdir -p /__w/mmdeploy/prebuild/$OUTPUT_DIR - cp -r pack/* /__w/mmdeploy/prebuild/$OUTPUT_DIR + mkdir -p $OUTPUT_DIR/sdk + chmod -R 777 $OUTPUT_DIR + mv -vf pack/sdk/* $OUTPUT_DIR/sdk/ linux_test: if: inputs.run == true || ${{ github.event_name == 'push' }} @@ -124,60 +142,21 @@ jobs: export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") echo $MMDEPLOY_VERSION echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV - echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV + echo "OUTPUT_DIR=/__w/mmdeploy/prebuild/$MMDEPLOY_VERSION" >> $GITHUB_ENV - name: Test python run: | - cd /__w/mmdeploy/prebuild/$OUTPUT_DIR + cd $OUTPUT_DIR bash $GITHUB_WORKSPACE/tools/package_tools/test/test_sdk_python.sh - name: Test c/cpp run: | - cd /__w/mmdeploy/prebuild/$OUTPUT_DIR + cd $OUTPUT_DIR bash $GITHUB_WORKSPACE/tools/package_tools/test/test_sdk.sh - linux_upload: - if: inputs.run == true || ${{ github.event_name == 'push' }} - runs-on: [self-hosted, linux-3090] - environment: 'prod' - needs: linux_test - env: - PREBUILD_DIR: /data2/actions-runner/prebuild - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Get mmdeploy version - run: | - export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") - echo $MMDEPLOY_VERSION - echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV - echo "OUTPUT_DIR=$MMDEPLOY_VERSION-$GITHUB_RUN_ID" >> $GITHUB_ENV - - name: Upload mmdeploy - run: | - cd $PREBUILD_DIR/$OUTPUT_DIR/mmdeploy - pip install twine - # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }} - twine upload * -u __token__ -p ${{ secrets.pypi_password }} - - name: Upload mmdeploy_runtime - run: | - cd $PREBUILD_DIR/$OUTPUT_DIR/mmdeploy_runtime - # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }} - twine upload * -u __token__ -p ${{ secrets.pypi_password }} - - name: Zip mmdeploy sdk - run: | - cd $PREBUILD_DIR/$OUTPUT_DIR/sdk - for folder in * - do - tar czf $folder.tar.gz $folder - done - - name: Upload mmdeploy sdk - uses: softprops/action-gh-release@v1 - with: - files: | - $PREBUILD_DIR/$OUTPUT_DIR/sdk/*.tar.gz - - windows_build: if: inputs.run == true || ${{ github.event_name == 'push' }} runs-on: [self-hosted, win10-3080] + env: + PREBUILD_DIR: D:\Runners\prebuild steps: - name: Checkout repository uses: actions/checkout@v3 @@ -189,12 +168,13 @@ jobs: $env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") echo $env:MMDEPLOY_VERSION echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV - echo "OUTPUT_DIR=$env:MMDEPLOY_VERSION-$env:GITHUB_RUN_ID" >> $env:GITHUB_ENV + echo "OUTPUT_DIR=$env:PREBUILD_DIR\$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV - name: Build MMDeploy run: | . D:\DEPS\cienv\prebuild_gpu_env.ps1 conda activate mmdeploy-3.6 - mkdir pack; cd pack + New-Item -Path pack -ItemType Directory -Force + cd pack python ../tools/package_tools/generate_build_config.py --backend 'trt;ort' ` --system windows --output config.yml --build-mmdeploy python ../tools/package_tools/mmdeploy_builder.py --config config.yml @@ -216,15 +196,28 @@ jobs: --system windows --output config.yml --device cuda --build-sdk --build-sdk-monolithic ` --build-sdk-python --sdk-dynamic-net python ../tools/package_tools/mmdeploy_builder.py --config config.yml + - name: Zip mmdeploy sdk + run: | + cd pack/sdk + $folders = $(ls).Name + foreach ($folder in $folders) { + Compress-Archive -Path $folder -DestinationPath "$folder.zip" + } - name: Move artifact run: | - New-Item "D:/DEPS/ciartifact/$env:OUTPUT_DIR" -ItemType Directory -Force - Move-Item pack/* "D:/DEPS/ciartifact/$env:OUTPUT_DIR" + New-Item "$env:OUTPUT_DIR\mmdeploy" -ItemType Directory -Force + New-Item "$env:OUTPUT_DIR\mmdeploy_runtime" -ItemType Directory -Force + New-Item "$env:OUTPUT_DIR\sdk" -ItemType Directory -Force + Move-Item pack/mmdeploy/* -Force "$env:OUTPUT_DIR\mmdeploy\" + Move-Item pack/mmdeploy_runtime/* -Force "$env:OUTPUT_DIR\mmdeploy_runtime\" + Move-Item pack/sdk/* -Force "$env:OUTPUT_DIR\sdk\" windows_test: if: inputs.run == true || ${{ github.event_name == 'push' }} runs-on: [self-hosted, win10-3080] needs: windows_build + env: + PREBUILD_DIR: D:\Runners\prebuild steps: - name: Checkout repository uses: actions/checkout@v3 @@ -234,55 +227,15 @@ jobs: $env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") echo $env:MMDEPLOY_VERSION echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV - echo "OUTPUT_DIR=$env:MMDEPLOY_VERSION-$env:GITHUB_RUN_ID" >> $env:GITHUB_ENV + echo "OUTPUT_DIR=$env:PREBUILD_DIR\$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV - name: Test python run: | - cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR" + cd "$env:OUTPUT_DIR" . D:\DEPS\cienv\prebuild_cpu_env.ps1 conda activate ci-test & "$env:GITHUB_WORKSPACE/tools/package_tools/test/test_sdk_python.ps1" - name: Test c/cpp run: | - cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR" + cd "$env:OUTPUT_DIR" . D:\DEPS\cienv\prebuild_cpu_env.ps1 & "$env:GITHUB_WORKSPACE/tools/package_tools/test/test_sdk.ps1" - - windows_upload: - if: inputs.run == true || ${{ github.event_name == 'push' }} - runs-on: [self-hosted, win10-3080] - environment: 'prod' - needs: windows_test - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Get mmdeploy version - run: | - conda activate mmdeploy-3.8 - $env:MMDEPLOY_VERSION=(python -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") - echo $env:MMDEPLOY_VERSION - echo "MMDEPLOY_VERSION=$env:MMDEPLOY_VERSION" >> $env:GITHUB_ENV - echo "OUTPUT_DIR=$env:MMDEPLOY_VERSION-$env:GITHUB_RUN_ID" >> $env:GITHUB_ENV - - name: Upload mmdeploy - run: | - cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR/mmdeploy" - conda activate mmdeploy-3.8 - # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }} - twine upload * -u __token__ -p ${{ secrets.pypi_password }} - - name: Upload mmdeploy_runtime - run: | - cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR/mmdeploy_runtime" - conda activate mmdeploy-3.8 - # twine upload * --repository testpypi -u __token__ -p ${{ secrets.test_pypi_password }} - twine upload * -u __token__ -p ${{ secrets.pypi_password }} - - name: Zip mmdeploy sdk - run: | - cd "D:/DEPS/ciartifact/$env:OUTPUT_DIR/sdk" - $folders = $(ls).Name - foreach ($folder in $folders) { - Compress-Archive -Path $folder -DestinationPath "$folder.zip" - } - - name: Upload mmdeploy sdk - uses: softprops/action-gh-release@v1 - with: - files: | - D:/DEPS/ciartifact/$env:OUTPUT_DIR/sdk/*.zip diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..a720afad8 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,57 @@ +name: publish + +on: + push: + tags: + - "v*.*.*" + + workflow_dispatch: + inputs: + publish: + description: 'Manually publish' + required: false + type: boolean + default: false + +jobs: + publish: + if: inputs.publish == true || ${{ github.event_name == 'push' }} + runs-on: [self-hosted, linux-3090] + environment: 'prod' + env: + PREBUILD_DIR: /data2/actions-runner/prebuild + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Get mmdeploy version + run: | + export MMDEPLOY_VERSION=$(python3 -c "import sys; sys.path.append('mmdeploy');from version import __version__;print(__version__)") + echo $MMDEPLOY_VERSION + echo "MMDEPLOY_VERSION=$MMDEPLOY_VERSION" >> $GITHUB_ENV + echo "OUTPUT_DIR=$PREBUILD_DIR/$MMDEPLOY_VERSION" >> $GITHUB_ENV + pip install twine + - name: Upload mmdeploy + continue-on-error: true + run: | + cd $OUTPUT_DIR/mmdeploy + ls -sha *.whl + # twine upload mmdeploy-${MMDEPLOY_VERSION}-py3-none-manylinux2014_x86_64.whl --repository testpypi -u testmmdeploy -p ${{ secrets.test_pypi_password }} + twine upload *.whl -u __token__ -p ${{ secrets.pypi_password }} + - name: Upload mmdeploy_runtime + continue-on-error: true + run: | + cd $OUTPUT_DIR/mmdeploy_runtime + ls -sha *.whl + # twine upload mmdeploy_runtime-${MMDEPLOY_VERSION}-cp38-none-manylinux2014_x86_64.whl --repository testpypi -u testmmdeploy -p ${{ secrets.test_pypi_password }} + twine upload *.whl -u __token__ -p ${{ secrets.pypi_password }} + - name: Create dummy softlinks to assets + run: | + ln -sf $OUTPUT_DIR/sdk ./prebuild + ls -sha ./prebuild + - name: Upload mmdeploy sdk + uses: softprops/action-gh-release@v1 + with: + tag_name: v${MMDEPLOY_VERSION} + files: | + ./prebuild/*.zip + ./prebuild/*.tar.gz diff --git a/tools/package_tools/test/test_sdk.ps1 b/tools/package_tools/test/test_sdk.ps1 index 1bf290868..299cd88a6 100644 --- a/tools/package_tools/test/test_sdk.ps1 +++ b/tools/package_tools/test/test_sdk.ps1 @@ -15,7 +15,7 @@ $pkgs = $(ls).Name $test_pkg = $pkgs[0] if ($pkgs.Count -gt 1) { foreach ($pkg in $pkgs) { - if ($pkg -like '*cpu*') { + if ($pkg -like '*-windows-amd64') { $test_pkg = $pkg break } diff --git a/tools/package_tools/test/test_sdk_python.ps1 b/tools/package_tools/test/test_sdk_python.ps1 index 894b917ea..12caacb9e 100644 --- a/tools/package_tools/test/test_sdk_python.ps1 +++ b/tools/package_tools/test/test_sdk_python.ps1 @@ -15,7 +15,7 @@ $pkgs = $(ls).Name $test_pkg = "" if ($pkgs.Count -gt 1) { foreach ($pkg in $pkgs) { - if ($pkg -like 'mmdeploy_runtime-*cp38*') { + if ($pkg -like 'mmdeploy_runtime-*cp38*-win_amd64.whl') { $test_pkg = $pkg break } diff --git a/tools/package_tools/test/test_sdk_python.sh b/tools/package_tools/test/test_sdk_python.sh index e2c491605..1bc198e6d 100755 --- a/tools/package_tools/test/test_sdk_python.sh +++ b/tools/package_tools/test/test_sdk_python.sh @@ -12,7 +12,7 @@ cd $WORKSPACE cd $SDK_PYTHON_DIR PY_VERSION=$(python3 -V | awk '{print $2}' | awk '{split($0, a, "."); print a[1]a[2]}') -test_pkg=$(ls | grep mmdeploy_runtime-*cp${PY_VERSION}*) +test_pkg=$(ls | grep mmdeploy_runtime-*cp${PY_VERSION}*x86_64.whl) python3 -m pip install $test_pkg --force-reinstall python3 -m pip install opencv-python