diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml new file mode 100644 index 000000000..9d1a97b9f --- /dev/null +++ b/.github/workflows/rc.yml @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RC +on: + push: + branches: + - '**' + - '!dependabot/**' + tags: + - '*-rc*' + pull_request: +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: read +jobs: + archive: + name: Archive + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - name: Prepare for tag + if: github.ref_type == 'tag' + run: | + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Prepare for branch + if: github.ref_type == 'branch' + run: | + version=$(grep -o '^ .*' "pom.xml" | + sed \ + -e 's,^ ,,' \ + -e 's,$,,') + rc=$(date +%Y%m%d) + echo "VERSION=${version}" >> ${GITHUB_ENV} + echo "RC=${rc}" >> ${GITHUB_ENV} + - name: Archive + run: | + id="apache-arrow-java-${VERSION}" + tar_gz="${id}.tar.gz" + echo "TAR_GZ=${tar_gz}" >> ${GITHUB_ENV} + git archive HEAD --prefix "${id}/" --output "${tar_gz}" + sha256sum "${tar_gz}" > "${tar_gz}.sha256" + sha512sum "${tar_gz}" > "${tar_gz}.sha512" + - name: Audit + run: | + dev/release/run_rat.sh "${TAR_GZ}" + - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: archive + path: | + apache-arrow-java-* + verify: + name: Verify + needs: + - archive + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Verify + run: | + tar_gz=$(echo apache-arrow-java-*.tar.gz) + version=${tar_gz#apache-arrow-java-} + version=${version%.tar.gz} + # rc isn't used with VERIFY_DOWNLOAD=0 + if [ "${GITHUB_REF_TYPE}" = "tag" ]; then + rc="${GITHUB_REF_NAME#*-rc}" + else + rc=$(date +%Y%m%d) + fi + VERIFY_DEFAULT=0 \ + VERIFY_SOURCE=1 \ + dev/release/verify_rc.sh "${version}" "${rc}" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + upload: + name: Upload + if: github.ref_type == 'tag' + needs: + - verify + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: archive + - name: Upload + run: | + # TODO: Add support for release note + version=${GITHUB_REF_NAME%-rc*} + version=${version#v} + rc=${GITHUB_REF_NAME#*-rc} + gh release create ${GITHUB_REF_NAME} \ + --notes "TODO" \ + --prerelease \ + --title "Apache Arrow Java ${version} RC${rc}" \ + --verify-tag \ + apache-arrow-java-*.tar.gz \ + apache-arrow-java-*.tar.gz.sha* + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..e0b866e84 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: RC +on: + push: + tags: + - '*' + - '!*-rc*' + pull_request: +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true +permissions: + contents: write +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +jobs: + publish: + name: Publish + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Download RC contents + run: | + set -x + latest_rc_tag=$(gh release list --json tagName --jq '.[].tagName' | \ + grep -F "${GITHUB_REF_NAME}-rc" | \ + head -n1) + gh release download ${latest_rc_tag} --dir dists + - name: Create GitHub Release + run: | + version=${GITHUB_REF_NAME#v} + gh release create ${GITHUB_REF_NAME} \ + --notes "TODO" \ + --title "Apache Arrow Java ${version}" \ + --verify-tag \ + dists/* diff --git a/dev/release/README.md b/dev/release/README.md new file mode 100644 index 000000000..27142406f --- /dev/null +++ b/dev/release/README.md @@ -0,0 +1,122 @@ + + +# Release + +## Overview + + 1. Test the revision to be released + 2. Bump version by `dev/release/bump_version.sh X.Y.Z` + 3. Prepare RC and vote (detailed later) + 4. Publish (detailed later) + 5. Bump version by `dev/release/bump_version.sh X.Y.Z-SNAPSHOT` + +### Prepare RC and vote + +Run `dev/release/release_rc.sh` on a working copy of +`git@github.com:apache/arrow-java` not your fork: + +```console +$ git clone git@github.com:apache/arrow-java.git +$ dev/release/release_rc.sh ${RC} +(Send a vote email to dev@arrow.apache.org. + You can use a draft shown by release_rc.sh for the email.) +``` + +Here is an example to release RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release_rc.sh 1 +``` + +The argument of `release_rc.sh` is the RC number. If RC1 has a +problem, we'll increment the RC number such as RC2, RC3 and so on. + +Requirements to run `release_rc.sh`: + + * You must be an Apache Arrow committer or PMC member + * You must prepare your PGP key for signing + +If you don't have a PGP key, +https://infra.apache.org/release-signing.html#generate may be helpful. + +Your PGP key must be registered to the followings: + + * https://dist.apache.org/repos/dist/dev/arrow/KEYS + * https://dist.apache.org/repos/dist/release/arrow/KEYS + +See the header comment of them how to add a PGP key. + +Apache arrow committers can update them by Subversion client with +their ASF account. e.g.: + +```console +$ svn co https://dist.apache.org/repos/dist/dev/arrow +$ cd arrow +$ editor KEYS +$ svn ci KEYS +``` + +### Publish + +We need to do the followings to publish a new release: + + * Publish to apache.org + +Run `dev/release/release.sh` on a working copy of +`git@github.com:apache/arrow-java` not your fork to publish to +apache.org: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh ${VERSION} ${RC} +``` + +Here is an example to release 19.0.0 RC1: + +```console +$ GH_TOKEN=${YOUR_GITHUB_TOKEN} dev/release/release.sh 19.0.0 1 +``` + +Add the release to ASF's report database via [Apache Committee Report +Helper](https://reporter.apache.org/addrelease.html?arrow). + +### Verify + +We have a script to verify a RC. + +You must install the following commands to use the script: + + * `curl` + * `gpg` + * `shasum` or `sha256sum`/`sha512sum` + * `tar` + +To verify a RC, run the following command line: + +```console +$ dev/release/verify_rc.sh ${VERSION} ${RC} +``` + +Here is an example to verify the release 19.0.0 RC1: + +```console +$ dev/release/verify_rc.sh 19.0.0 1 +``` + +If the verification is successful, the message `RC looks good!` is shown. diff --git a/dev/release/bump_version.sh b/dev/release/bump_version.sh new file mode 100755 index 000000000..84eded468 --- /dev/null +++ b/dev/release/bump_version.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " e.g.: $0 19.0.1" + echo " e.g.: $0 20.0.0-SNAPSHOT" + exit 1 +fi + +version=$1 + +cd "${SOURCE_TOP_DIR}" + +git switch -c bump-version-${version} +mvn versions:set -DnewVersion=${version} -DprocessAllModules -DgenerateBackupPoms=false +case "${version}" in + *-SNAPSHOT) + tag=main + ;; + *) + tag=v${version} + ;; +esac +mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-java-root +mvn versions:set-scm-tag -DnewTag=${tag} -DgenerateBackupPoms=false -pl :arrow-bom +git add "pom.xml" +git add "**/pom.xml" +git commit -m "MINOR: Bump version to ${version}" +gh pr create --fill diff --git a/dev/release/release.sh b/dev/release/release.sh new file mode 100755 index 000000000..ff3f854b1 --- /dev/null +++ b/dev/release/release.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo " e.g.: $0 19.0.1 1" + exit 1 +fi + +version=$1 +rc=$2 + +git_origin_url="$(git remote get-url origin)" +repository="${git_origin_url#*github.com?}" +repository="${repository%.git}" +if [ "${git_origin_url}" != "git@github.com:apache/arrow-java.git" ]; then + echo "This script must be ran with working copy of apache/arrow-java." + echo "The origin's URL: ${git_origin_url}" + exit 1 +fi + +tag="v${version}" +rc_tag="${tag}-rc${rc}" +echo "Tagging for release: ${tag}" +git tag -a -m "${version}" "${tag}" "${rc_tag}" +git push origin "${tag}" + +dist_url="https://dist.apache.org/repos/dist/release/arrow" +dist_dir="dev/release/dist" +echo "Checking out ${dist_url}" +rm -rf "${dist_dir}" +svn co --depth=empty "${dist_url}" "${dist_dir}" +gh release download "${rc_tag}" \ + --repo "${repository}" \ + --dir "${dist_dir}" \ + --skip-existing + +release_id="apache-arrow-java-${version}" +echo "Uploading to release/" +pushd "${dist_dir}" +svn add . +svn ci -m "Apache Arrow Java ${version}" +popd +rm -rf "${dist_dir}" + +echo "Keep only the latest versions" +old_releases=$( + svn ls https://dist.apache.org/repos/dist/release/arrow/ | + grep -E '^apache-arrow-java-' | + sort --version-sort --reverse | + tail -n +2 +) +for old_release_version in ${old_releases}; do + echo "Remove old release ${old_release_version}" + svn \ + delete \ + -m "Remove old Apache Arrow Java release: ${old_release_version}" \ + "https://dist.apache.org/repos/dist/release/arrow/${old_release_version}" +done + +echo "Success! The release is available here:" +echo " https://dist.apache.org/repos/dist/release/arrow/${release_id}" +echo +echo "Add this release to ASF's report database:" +echo " https://reporter.apache.org/addrelease.html?arrow" diff --git a/dev/release/release_rc.sh b/dev/release/release_rc.sh new file mode 100755 index 000000000..7e820aab8 --- /dev/null +++ b/dev/release/release_rc.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -eu + +SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + echo " e.g.: $0 1" + exit 1 +fi + +rc=$1 + +: "${RELEASE_DEFAULT:=1}" +: "${RELEASE_PULL:=${RELEASE_DEFAULT}}" +: "${RELEASE_PUSH_TAG:=${RELEASE_DEFAULT}}" +: "${RELEASE_SIGN:=${RELEASE_DEFAULT}}" +: "${RELEASE_UPLOAD:=${RELEASE_DEFAULT}}" + +cd "${SOURCE_TOP_DIR}" + +if [ "${RELEASE_PULL}" -gt 0 ] || [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + if [ "${git_origin_url}" != "git@github.com:apache/arrow-java.git" ]; then + echo "This script must be ran with working copy of apache/arrow-java." + echo "The origin's URL: ${git_origin_url}" + exit 1 + fi +fi + +if [ "${RELEASE_PULL}" -gt 0 ]; then + echo "Ensure using the latest commit" + git checkout main + git pull --rebase --prune +fi + +version=$(grep -o '^ .*' "pom.xml" | + sed \ + -e 's,^ ,,' \ + -e 's,$,,') + +case "${version}" in + *-SNAPSHOT) + echo "Version isn't bumped: ${version}" + echo "Run dev/release/bump_version.sh before you run this script." + exit 1 + ;; +esac + +rc_tag="v${version}-rc${rc}" +if [ "${RELEASE_PUSH_TAG}" -gt 0 ]; then + echo "Tagging for RC: ${rc_tag}" + git tag -a -m "${version} RC${rc}" "${rc_tag}" + git push origin "${rc_tag}" +fi + +rc_hash="$(git rev-list --max-count=1 "${rc_tag}")" + +id="apache-arrow-java-${version}" +tar_gz="${id}.tar.gz" + +if [ "${RELEASE_SIGN}" -gt 0 ]; then + git_origin_url="$(git remote get-url origin)" + repository="${git_origin_url#*github.com?}" + repository="${repository%.git}" + + echo "Looking for GitHub Actions workflow on ${repository}:${rc_tag}" + run_id="" + while [ -z "${run_id}" ]; do + echo "Waiting for run to start..." + run_id=$(gh run list \ + --branch "${rc_tag}" \ + --jq ".[].databaseId" \ + --json 'databaseId' \ + --limit 1 \ + --repo "${repository}" \ + --workflow rc.yml) + sleep 1 + done + + echo "Found GitHub Actions workflow with ID: ${run_id}" + gh run watch --repo "${repository}" --exit-status "${run_id}" + + echo "Downloading .tar.gz from GitHub Releases" + gh release download "${rc_tag}" \ + --dir . \ + --pattern "${tar_gz}" \ + --repo "${repository}" \ + --skip-existing + + echo "Signing tar.gz and creating checksums" + gpg --armor --output "${tar_gz}.asc" --detach-sig "${tar_gz}" +fi + +if [ "${RELEASE_UPLOAD}" -gt 0 ]; then + echo "Uploading signature" + gh release upload "${rc_tag}" \ + --clobber \ + --repo "${repository}" \ + "${tar_gz}.asc" +fi + +echo "Draft email for dev@arrow.apache.org mailing list" +echo "" +echo "---------------------------------------------------------" +cat < " + echo " e.g.: $0 19.0.1 1" + exit 1 +fi + +set -o pipefail +set -x + +VERSION="$1" +RC="$2" + +ARROW_DIST_BASE_URL="https://dist.apache.org/repos/dist/dev/arrow" +DOWNLOAD_RC_BASE_URL="https://github.com/apache/arrow-java/releases/download/v${VERSION}-rc${RC}" +ARCHIVE_BASE_NAME="apache-arrow-java-${VERSION}" + +: "${VERIFY_DEFAULT:=1}" +: "${VERIFY_DOWNLOAD:=${VERIFY_DEFAULT}}" +: "${VERIFY_SIGN:=${VERIFY_DEFAULT}}" +: "${VERIFY_SOURCE:=${VERIFY_DEFAULT}}" +: "${VERIFY_BINARY:=${VERIFY_DEFAULT}}" + +VERIFY_SUCCESS=no + +setup_tmpdir() { + cleanup() { + if [ "${VERIFY_SUCCESS}" = "yes" ]; then + rm -rf "${VERIFY_TMPDIR}" + else + echo "Failed to verify release candidate. See ${VERIFY_TMPDIR} for details." + fi + } + + if [ -z "${VERIFY_TMPDIR:-}" ]; then + VERIFY_TMPDIR="$(mktemp -d -t "$1.XXXXX")" + trap cleanup EXIT + else + mkdir -p "${VERIFY_TMPDIR}" + fi +} + +download() { + curl \ + --fail \ + --location \ + --remote-name \ + --show-error \ + --silent \ + "$1" +} + +download_rc_file() { + if [ "${VERIFY_DOWNLOAD}" -gt 0 ]; then + download "${DOWNLOAD_RC_BASE_URL}/$1" + else + cp "${TOP_SOURCE_DIR}/$1" "$1" + fi +} + +import_gpg_keys() { + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download "${ARROW_DIST_BASE_URL}/KEYS" + gpg --import KEYS + fi +} + +if type shasum >/dev/null 2>&1; then + sha256_verify="shasum -a 256 -c" + sha512_verify="shasum -a 512 -c" +else + sha256_verify="sha256sum -c" + sha512_verify="sha512sum -c" +fi + +fetch_archive() { + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz" + if [ "${VERIFY_SIGN}" -gt 0 ]; then + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.asc" + gpg --verify "${ARCHIVE_BASE_NAME}.tar.gz.asc" "${ARCHIVE_BASE_NAME}.tar.gz" + fi + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + ${sha256_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha256" + download_rc_file "${ARCHIVE_BASE_NAME}.tar.gz.sha512" + ${sha512_verify} "${ARCHIVE_BASE_NAME}.tar.gz.sha512" +} + +ensure_source_directory() { + tar xf "${ARCHIVE_BASE_NAME}".tar.gz + + if [ -d "${TOP_SOURCE_DIR}/testing/data" ]; then + cp -a "${TOP_SOURCE_DIR}/testing" "${ARCHIVE_BASE_NAME}/" + else + git clone \ + https://github.com/apache/arrow-testing.git \ + "${ARCHIVE_BASE_NAME}/testing" + fi + + ARROW_TEST_DATA="${ARCHIVE_BASE_NAME}/testing/data" + export ARROW_TEST_DATA +} + +test_source_distribution() { + if [ ${VERIFY_SOURCE} -le 0 ]; then + return 0 + fi + + "${TOP_SOURCE_DIR}/ci/scripts/build.sh" "$(pwd)" + "${TOP_SOURCE_DIR}/ci/scripts/test.sh" "$(pwd)" + # TODO: JNI test + # TODO: Integration test +} + +test_binary_distribution() { + if [ ${VERIFY_SOURCE} -le 0 ]; then + return 0 + fi + + # TODO: jar test +} + +setup_tmpdir "arrow-java-${VERSION}-${RC}" +echo "Working in sandbox ${VERIFY_TMPDIR}" +cd "${VERIFY_TMPDIR}" + +import_gpg_keys +fetch_archive +ensure_source_directory +pushd "${ARCHIVE_BASE_NAME}" +test_source_distribution +test_binary_distribution +popd + +VERIFY_SUCCESS=yes +echo "RC looks good!"