Build PyPop #2316
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build PyPop | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
cancel-in-progress: true | |
on: | |
workflow_dispatch: | |
pull_request: | |
paths-ignore: | |
- '**.md' | |
- '**.rst' | |
- 'CITATION.cff' | |
- '.zenodo.extras.json' | |
- 'website/**' | |
- '.github/dependabot.yml' | |
- '.github/release-drafter.yml' | |
- '.github/ISSUE_TEMPLATE/**' | |
- '.github/workflows/documentation.yaml' | |
- '.github/workflows/buildjet_arm64.yml' | |
- '.github/workflows/release-drafter.yml' | |
- '.github/workflows/codeql.yml' | |
- '.gitattributes' | |
push: | |
paths-ignore: | |
- '**.md' | |
- '**.rst' | |
- 'CITATION.cff' | |
- '.zenodo.extras.json' | |
- 'website/**' | |
- '.github/dependabot.yml' | |
- '.github/release-drafter.yml' | |
- '.github/ISSUE_TEMPLATE/**' | |
- '.github/workflows/documentation.yaml' | |
- '.github/workflows/buildjet_arm64.yml' | |
- '.github/workflows/release-drafter.yml' | |
- '.github/workflows/codeql.yml' | |
- '.gitattributes' | |
release: | |
types: | |
- published | |
jobs: | |
skip_publish_zenodo: | |
name: Non publication, Zenodo skip | |
runs-on: ubuntu-latest | |
# FIXME: this is the default action, the logical inverse of the | |
# below "if" in publish_zenodo: a bit hacky, but it works | |
if: (github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false) == false | |
steps: | |
- name: Skip Zenodo step unless publishing release | |
run: | | |
echo "Not a publication release, skipping Zenodo" | |
echo "github.ref: ${{ toJson(github.ref) }}" | |
echo "github.event_name: ${{ toJson(github.event_name) }}" | |
echo "github.event.action: ${{ toJson(github.event.action) }}" | |
publish_zenodo: | |
name: Production release, Zenodo publication | |
runs-on: ubuntu-latest | |
# "else" we *are* publishing something, only works for non-pre-releases on the main branch | |
if: github.event.release.target_commitish == 'citation-cff-zenodo' && github.event_name == 'release' && github.event.action == 'published' && github.event.release.prerelease == false | |
steps: | |
- name: Checkout the contents of your repository | |
uses: actions/checkout@v4 | |
- name: Upsert the version and related ids in the metadata in CITATION.cff | |
run: | | |
echo "GITHUB_REF_NAME = $GITHUB_REF_NAME | GITHUB_REF = $GITHUB_REF | GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH" | |
echo "github.event.release.target_commitish = ${{ github.event.release.target_commitish }}" | |
# convert semver to Python-style version | |
VERSION_PYTHON_STYLE=$(GITHUB_REF_NAME=$GITHUB_REF_NAME python -c 'import os; from packaging import version; print(str(version.Version(os.getenv("GITHUB_REF_NAME"))))') | |
echo "VERSION_PYTHON_STYLE = $VERSION_PYTHON_STYLE" | |
# insert version | |
yq -i ".version = \"$GITHUB_REF_NAME\"" CITATION.cff | |
# update the 'related_identifiers' to use exact tree and pypi packaage | |
yq -i "(.identifiers[] | select(.value == \"https://github.com/alexlancaster/pypop/tree*\") | .value) |= \"https://github.com/alexlancaster/pypop/tree/$GITHUB_REF_NAME\"" CITATION.cff | |
yq -i "(.identifiers[] | select(.value == \"https://pypi.org/project/pypop-genomics/*\") | .value) |= \"https://pypi.org/project/pypop-genomics/$VERSION_PYTHON_STYLE\"" CITATION.cff | |
# temporarily remove DOI | |
yq -i eval "del(.doi)" CITATION.cff | |
# use pre-prelease of cffconvert 3.0.0 | |
pip install -U pip | |
pip install 'git+https://github.com/alexlancaster/cffconvert.git@combine_features#egg=cffconvert' | |
# convert to json, but remove creators | |
cffconvert --format zenodo --infile CITATION.cff | jq 'del(.creators)' > .zenodo.citation.json | |
# save creators separately, if they are included in the merge they get re-sorted | |
cffconvert --format zenodo --infile CITATION.cff | jq '{creators, creators}' > .zenodo.creators.json | |
# need to do a "deep merge" to make sure 'related_identifiers' get merged properly | |
# including updating the version number, followed by merging back with creators | |
cat .zenodo.citation.json .zenodo.extras.json |jq -s 'def deepmerge(a;b): | |
reduce b[] as $item (a; | |
reduce ($item | keys_unsorted[]) as $key (.; | |
$item[$key] as $val | ($val | type) as $type | .[$key] = if ($type == "object") then | |
deepmerge({}; [if .[$key] == null then {} else .[$key] end, $val]) | |
elif ($type == "array") then | |
(.[$key] + $val | unique) | |
else | |
$val | |
end) | |
); | |
deepmerge({}; .)' -S --indent 4 | cat .zenodo.creators.json - | jq -s add --indent 4 > .zenodo.json | |
# debugging only | |
cat .zenodo.citation.json | |
cat .zenodo.creators.json | |
cat .zenodo.json | |
rm .zenodo.citation.json .zenodo.creators.json | |
- name: Push changes back to repo | |
run: | | |
git fetch origin | |
git config --global user.name "ci-pypop" | |
git config --global user.email "[email protected]" | |
git checkout -b $GITHUB_REF_NAME-with-upserting-changes-pre-upload | |
git add CITATION.cff | |
git commit -m "Upserted version into CITATION.cff" | |
git checkout ${{ github.event.release.target_commitish }} | |
git merge $GITHUB_REF_NAME-with-upserting-changes-pre-upload | |
git push origin ${{ github.event.release.target_commitish }} | |
- name: Push generated citation files from CITATION.cff to repo | |
uses: ./.github/actions/update_citations | |
with: | |
CITATION_DIR: 'src/PyPop/citation' | |
INIT_FILE: 'src/PyPop/__init__.py' | |
PUSH: "origin ${{ github.event.release.target_commitish }}" | |
- name: Clean-up and remove files not for export | |
run: | | |
# FIXME: removing files marked with 'export-ignore' 'set' in .gitattributes | |
# before zenodraft creates the zipball using https://stackoverflow.com/a/30573319 | |
# to ensure that we handle filenames with potential ':' or newlines | |
git ls-files -z|git check-attr --stdin -z export-ignore |sed -zne 'x;n;n;s/^set$//;t print;b;:print;x;p'|xargs --null rm | |
# FIXME: need to also remove orphaned empty directories resulting from above | |
find . -empty -type d -delete | |
- name: Create a draft snapshot of your repository contents as a new | |
version using metadata from generated .zenodo.json | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }} | |
uses: zenodraft/[email protected] | |
# uses: alexlancaster/[email protected] | |
with: | |
concept: 10080667 | |
metadata: .zenodo.json | |
publish: false | |
sandbox: false | |
compression: zip | |
upsert-doi: true | |
upsert-location: doi | |
verbose: true | |
generate-wheels-matrix: | |
# Create a matrix of all architectures & versions to build. | |
# This enables the next step to run cibuildwheel in parallel. | |
# From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210 | |
name: Generate wheels matrix | |
runs-on: ubuntu-latest | |
outputs: | |
include: ${{ steps.set-matrix.outputs.include }} | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Install cibuildwheel | |
# Nb. keep cibuildwheel version pin consistent with job below | |
run: pipx install cibuildwheel==2.21.3 | |
- id: set-matrix | |
run: | | |
echo "CI_ONLY" $CI_ONLY | |
if [ "$CI_ONLY" == "true" ]; then | |
MATRIX="[{'only':'cp313-manylinux_x86_64','os':'ubuntu-20.04'},{'only':'cp313-win_amd64','os':'windows-2019'}, {'only':'cp313-macosx_x86_64','os':'macos-13'}, {'only':'cp313-macosx_arm64','os':'macos-14'}]" | |
else | |
MATRIX=$( | |
{ | |
cibuildwheel --print-build-identifiers --platform linux \ | |
| jq -nRc '{"only": inputs, "os": "ubuntu-20.04"}' \ | |
&& cibuildwheel --print-build-identifiers --platform macos --archs x86_64 \ | |
| jq -nRc '{"only": inputs, "os": "macos-13"}' \ | |
&& cibuildwheel --print-build-identifiers --platform macos --archs arm64 \ | |
| jq -nRc '{"only": inputs, "os": "macos-14"}' \ | |
&& cibuildwheel --print-build-identifiers --platform windows \ | |
| jq -nRc '{"only": inputs, "os": "windows-2019"}' | |
} | jq -sc | |
) | |
fi | |
echo "include=$MATRIX" >> $GITHUB_OUTPUT | |
echo "platform matrix:" $MATRIX | |
env: | |
# only generate all wheels on releases, pull requests or commits to the main branch | |
# otherwise just generate a test wheel Python 3.12 for 4 platform | |
CI_ONLY: ${{ github.event_name != 'release' && github.event_name != 'pull_request' && github.ref_name != 'main' && github.event_name != 'workflow_dispatch' }} | |
build_wheels: | |
name: Build wheels on ${{ matrix.only }} | |
# depend on zenodo, to make sure updated CITATION.cff gets into builds | |
# this only results in changes upon production releases | |
needs: [publish_zenodo, skip_publish_zenodo, generate-wheels-matrix] | |
# trying solution here: https://github.com/actions/runner/issues/491#issuecomment-1507495166 | |
# so that it works if either job in "needs" completes | |
if: always() && !cancelled() && !failure() | |
runs-on: ${{ matrix.os }} | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} | |
steps: | |
- uses: actions/checkout@v4 | |
with: | |
fetch-depth: 0 | |
# FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky) | |
ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }} | |
- name: Set up QEMU | |
if: runner.os == 'Linux' | |
uses: docker/setup-qemu-action@v3 | |
with: | |
platforms: all | |
- name: Install toml and remove cffconvert from pyproject.toml | |
run: | | |
python -m pip install toml | |
python -c " | |
import toml | |
with open('pyproject.toml', 'r') as f: | |
config = toml.load(f) | |
if 'build-system' in config and 'requires' in config['build-system']: | |
config['build-system']['requires'] = [ | |
dep for dep in config['build-system']['requires'] if 'cffconvert' not in dep.lower() | |
] | |
with open('pyproject.toml', 'w') as f: | |
toml.dump(config, f) | |
" | |
- name: Install a recent stable Python to handle cffconvert | |
uses: actions/setup-python@v4 | |
with: | |
python-version: 3.12 | |
- name: Generate citation formats | |
run: | | |
python --version | |
python -m pip install git+https://github.com/alexlancaster/cffconvert.git@combine_features#egg=cffconvert | |
python src/PyPop/citation.py | |
- name: Build and test wheels | |
uses: pypa/[email protected] | |
env: | |
# FIXME: only run the slow tests when doing regular pushes, or manual - not for PRs | |
CIBW_TEST_COMMAND: "pytest -v {package}/tests ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && '--runslow' || '' }}" | |
with: | |
only: ${{ matrix.only }} | |
package-dir: . | |
output-dir: wheelhouse | |
config-file: "{package}/pyproject.toml" | |
- uses: actions/upload-artifact@v4 | |
with: | |
name: wheels-${{ matrix.only }} | |
path: ./wheelhouse/*.whl | |
# path: ./wheelhouse/* use if more than wheels are wanted | |
build_sdist: | |
name: Build source distribution | |
# depend on zenodo, to make sure updated CITATION.cff gets into builds | |
needs: [publish_zenodo, skip_publish_zenodo] | |
if: always() && !cancelled() && !failure() | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v4 | |
with: # FIXME: if on a release, use the tag name, not original SHA because it might be changed (somewhat hacky) | |
ref: ${{ (github.event_name == 'release' && github.event.action == 'published') && github.ref_name || '' }} | |
- name: Build sdist | |
run: pipx run build --sdist | |
- uses: actions/upload-artifact@v4 | |
with: | |
name: sdist | |
path: dist/*.tar.gz | |
upload_gh_release: | |
name: Upload binary wheels and sdist to GH release page | |
needs: [build_wheels, build_sdist] | |
runs-on: ubuntu-latest | |
# alternatively, to publish when a GitHub Release is created, use the following rule: | |
if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
steps: | |
- uses: actions/download-artifact@v4 | |
with: | |
# unpacks default artifact into dist/ | |
path: dist | |
merge-multiple: true | |
- uses: softprops/action-gh-release@v2 | |
name: Uploading binaries to release page | |
with: | |
files: dist/* | |
upload_test_pypi: | |
name: Upload to Test_PyPI | |
needs: [build_wheels, build_sdist] | |
runs-on: ubuntu-latest | |
environment: test_pypi | |
permissions: | |
id-token: write | |
# drop requirement of 'main', release to test_pypi to test releases | |
if: github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
steps: | |
- uses: actions/download-artifact@v4 | |
with: | |
# unpacks default artifact into dist/ | |
path: dist | |
merge-multiple: true | |
- uses: pypa/gh-action-pypi-publish@release/v1 | |
with: | |
repository_url: https://test.pypi.org/legacy/ | |
upload_pypi: | |
name: Upload to PyPI | |
needs: [build_wheels, build_sdist] | |
runs-on: ubuntu-latest | |
environment: pypi | |
permissions: | |
id-token: write | |
if: github.event.release.target_commitish == 'main' && github.event_name == 'release' && github.event.action == 'published' && always() && !cancelled() && !failure() | |
steps: | |
- uses: actions/download-artifact@v4 | |
with: | |
# unpacks default artifact into dist/ | |
path: dist | |
merge-multiple: true | |
- name: Publish package distributions to PyPI | |
uses: pypa/gh-action-pypi-publish@release/v1 |