Release #2470
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| push: | |
| branches: | |
| - "release/*" | |
| schedule: | |
| # beijing: 2 am. | |
| # utc: 6 pm. | |
| - cron: "0 18 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| placeholder: | |
| description: "update .github/workflows/release.yml to config your build" | |
| required: false | |
| workflow_call: | |
| inputs: | |
| is_priv: | |
| required: true | |
| type: boolean | |
| branch: | |
| required: false | |
| type: string | |
| default: "main" | |
| upload_override_branch: | |
| required: false | |
| type: string | |
| cuda_cmake_cache: | |
| required: false | |
| type: string | |
| secrets: | |
| ONEFLOW_PRIV_ORG: | |
| required: true | |
| ONEFLOW_PRIV_GH_TOKEN: | |
| required: true | |
| ONEFLOW_PRIV_OSS_BUCKET: | |
| required: true | |
| OSS_ACCESS_KEY_ID: | |
| required: true | |
| OSS_ACCESS_KEY_SECRET: | |
| required: true | |
| ONEFLOW_CI_HTTP_PROXY: | |
| required: false | |
| concurrency: | |
| group: release-${{ github.ref }}-${{ inputs.branch }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| env: | |
| ONEFLOW_SRC: . | |
| jobs: | |
| generate-build-matrix: | |
| name: "Generate build matrix" | |
| runs-on: ubuntu-latest | |
| env: | |
| ONEFLOW_SRC: . | |
| outputs: | |
| matrix: ${{ steps.find-cache.outputs.matrix }} | |
| formatted_date: ${{ steps.date.outputs.formatted_date }} | |
| steps: | |
| - name: Checkout Oneflow-Inc/oneflow | |
| uses: actions/checkout@v2 | |
| if: ${{ !inputs.is_priv }} | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{github.event.pull_request.head.repo.full_name}} | |
| - name: Checkout oneflow | |
| uses: actions/checkout@v2 | |
| if: ${{ inputs.is_priv }} | |
| with: | |
| ref: ${{ inputs.branch }} | |
| repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow | |
| token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }} | |
| - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@whl-skip-nccl | |
| name: Find build cache | |
| id: find-cache | |
| timeout-minutes: 5 | |
| with: | |
| delete-cache: ${{ contains(github.event.pull_request.labels.*.name, 'need-clean-ccache') }} | |
| runner-labels: | | |
| self-hosted | |
| linux | |
| release | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| entries: | | |
| cu122 | |
| cu121 | |
| cu118 | |
| cpu | |
| - name: Get current date | |
| id: date | |
| run: echo "formatted_date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT | |
| staging_release: | |
| env: | |
| MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/release/${{ matrix.entry }} | |
| WHEELHOUSE_DIR: manylinux_wheelhouse | |
| OSS_DIR: branch/${{ github.ref_name }}/${{ matrix.entry }}/${{ github.sha }} | |
| GITHUB_REF_NAME: ${{ github.ref_name }} | |
| GITHUB_SHA: ${{ github.sha }} | |
| ONEFLOW_OSS_BUCKET: oneflow-staging | |
| https_proxy: ${{ secrets.ONEFLOW_CI_HTTP_PROXY }} | |
| needs: [generate-build-matrix] | |
| name: Staging Release | |
| timeout-minutes: 240 | |
| runs-on: [self-hosted, linux, release] | |
| if: github.repository == 'Oneflow-Inc/oneflow' || inputs.is_priv | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 6 | |
| matrix: ${{ fromJson(needs.generate-build-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Fix permissions | |
| run: | | |
| docker run --rm -v $PWD:/p -w /p busybox rm -rf * | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| python3 -m pip install -U setuptools wheel --user | |
| python3 -m pip install oss2 --user | |
| - name: Checkout Oneflow-Inc/oneflow | |
| uses: actions/checkout@v2 | |
| if: ${{ !inputs.is_priv }} | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{github.event.pull_request.head.repo.full_name}} | |
| - name: Checkout private oneflow | |
| uses: actions/checkout@v2 | |
| if: ${{ inputs.is_priv }} | |
| with: | |
| ref: ${{ inputs.branch }} | |
| repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/oneflow | |
| token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }} | |
| - name: Checkout cutlass_extension | |
| uses: actions/checkout@v2 | |
| if: ${{ inputs.is_priv }} | |
| with: | |
| repository: ${{ secrets.ONEFLOW_PRIV_ORG }}/cutlass-extension | |
| token: ${{ secrets.ONEFLOW_PRIV_GH_TOKEN }} | |
| path: cutlass-extension | |
| - name: Set Private env | |
| if: ${{ inputs.is_priv }} | |
| run: | | |
| GITHUB_SHA=$(git rev-parse HEAD) | |
| echo "OSS_DIR=branch/${{ inputs.upload_override_branch || inputs.branch }}/${{ matrix.entry }}/${GITHUB_SHA}" >> $GITHUB_ENV | |
| echo "GITHUB_REF_NAME=${{ inputs.upload_override_branch || inputs.branch }}" >> $GITHUB_ENV | |
| echo "GITHUB_SHA=${GITHUB_SHA}" >> $GITHUB_ENV | |
| echo "ONEFLOW_OSS_BUCKET=${{ secrets.ONEFLOW_PRIV_OSS_BUCKET }}" >> $GITHUB_ENV | |
| - name: Print env | |
| if: ${{ inputs.is_priv }} | |
| run: | | |
| env | |
| - uses: Oneflow-Inc/get-oneflow@whl-skip-nccl | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ matrix.entry =='cu118' || startsWith(matrix.entry, 'cu12') }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/${{ inputs.cuda_cmake_cache || 'cmake/caches/ci/release/cu118.cmake' }} | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc9.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: true | |
| nightly: ${{ inputs.is_priv || github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| use-nvidia-wheels: ${{ matrix.entry !='cu112' }} | |
| python-versions: | | |
| 3.12 | |
| 3.11 | |
| 3.10 | |
| 3.9 | |
| 3.8 | |
| - uses: Oneflow-Inc/get-oneflow@whl-skip-nccl | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ startsWith(matrix.entry, 'cu') && matrix.entry !='cu118' && !startsWith(matrix.entry, 'cu12') }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cuda.cmake | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc9.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: true | |
| nightly: ${{ inputs.is_priv || github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| use-nvidia-wheels: ${{ matrix.entry !='cu112' }} | |
| python-versions: | | |
| 3.12 | |
| 3.11 | |
| 3.10 | |
| 3.9 | |
| 3.8 | |
| - uses: Oneflow-Inc/get-oneflow@whl-skip-nccl | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ matrix.entry =='cpu' }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cpu.cmake | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: false | |
| nightly: ${{ inputs.is_priv || github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| python-versions: | | |
| 3.12 | |
| 3.11 | |
| 3.10 | |
| 3.9 | |
| 3.8 | |
| - name: Upload wheel | |
| uses: ./.github/actions/upload_oss | |
| with: | |
| src_path: ${{ env.WHEELHOUSE_DIR }} | |
| oss_dst_path: oss://${{ env.ONEFLOW_OSS_BUCKET }}/${{ env.OSS_DIR }} | |
| oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| - name: Update pip index | |
| env: | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| run: | | |
| python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| python3 -m pip install oss2 beautifulsoup4 --user | |
| python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b ${{ env.ONEFLOW_OSS_BUCKET }} \ | |
| --index_key=branch/${{ env.GITHUB_REF_NAME }}/${{ matrix.entry }}/index.html \ | |
| --index_key=branch/${{ env.GITHUB_REF_NAME }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \ | |
| --index_key=${{ env.OSS_DIR }}/index.html \ | |
| --index_key=commit/${{ env.GITHUB_SHA }}/${{ matrix.entry }}/index.html | |
| - name: Update API docs | |
| if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu' && !inputs.is_priv | |
| env: | |
| READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }} | |
| run: | | |
| curl -X POST -d "branches=master" -d "token=${READTHEDOCS_TOKEN}" https://readthedocs.org/api/v2/webhook/oneflow/135376/ |