From dd09da1af730f860f5feb80c3f315993c68a8a9c Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Sun, 21 Jul 2024 21:50:14 +1000 Subject: [PATCH 01/10] Add examples artifact job --- .github/workflows/examples-artifact.yml | 41 +++++++++++++++++++++++++ .github/workflows/test.yml | 3 ++ 2 files changed, 44 insertions(+) create mode 100644 .github/workflows/examples-artifact.yml diff --git a/.github/workflows/examples-artifact.yml b/.github/workflows/examples-artifact.yml new file mode 100644 index 0000000..2ac9d79 --- /dev/null +++ b/.github/workflows/examples-artifact.yml @@ -0,0 +1,41 @@ +name: Notebook examples zip archive + +on: + workflow_call: + +permissions: + contents: read + +jobs: + examples-artifact: + + strategy: + matrix: + os: [ubuntu-latest] + python: ["3.11"] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install tooling + run: | + python -m pip install --upgrade pip + - name: Install dependencies + run: | + python -m pip install jupyter-client~=8.6 jupytext~=1.16 + - name: Convert notebooks + run: | + python -m jupytext --opt notebook_metadata_filter=-kernelspec \ + --to ipynb docs/source/examples/*.md + - name: Archive notebook and data artifact + uses: actions/upload-artifact@v4 + with: + name: notebook-examples + path: | + docs/source/examples/*.ipynb + docs/source/examples/data diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92eebce..aa5d0c9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,3 +19,6 @@ jobs: notebook-examples: uses: ./.github/workflows/notebook-examples.yml needs: [unit-tests] + examples-artifact: + uses: ./.github/workflows/examples-artifact.yml + needs: [unit-tests] From 348deebfdf773154690fb15873446d6852a853b3 Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Sun, 21 Jul 2024 23:02:58 +1000 Subject: [PATCH 02/10] Add artifact download script --- scripts/artifacts.py | 82 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 scripts/artifacts.py diff --git a/scripts/artifacts.py b/scripts/artifacts.py new file mode 100644 index 0000000..b205cea --- /dev/null +++ b/scripts/artifacts.py @@ -0,0 +1,82 @@ +# Run as a pre-build step on RTD. Use a fine-grained personal access token +# with no extra permissions (for public repos). +# +# Local test: +# +# READTHEDOCS_GIT_COMMIT_HASH=$(git rev-parse HEAD) READTHEDOCS=True \ +# GH_API_TOKEN= python artifacts.py + +import os +import pathlib +import sys + +import requests + +docs_source = pathlib.Path(__file__).parent.parent.joinpath("docs/source").resolve() + + +def download_executed_notebooks(runs_url, gh_token, head_sha): + headers = { + "X-GitHub-Api-Version": "2022-11-28", + "Authorization": f"Bearer {gh_token}", + } + + params = {"head_sha": head_sha} + response = requests.get(runs_url, headers=headers, params=params) + response.raise_for_status() + runs_data = response.json() + + for run in runs_data["workflow_runs"]: + print("Run id={id} event={event} status={status} path={path}".format(**run)) + + if run["path"] != ".github/workflows/test.yml": + continue + if run["status"] != "completed": + continue + if run["conclusion"] != "success": + continue + + artifacts_url = run["artifacts_url"] + response = requests.get(artifacts_url, headers=headers) + response.raise_for_status() + artifacts_data = response.json() + + for artifact in artifacts_data["artifacts"]: + print("Artifact id={id} name={name}".format(**artifact)) + + if artifact["name"] != "notebook-examples": + continue + + download_url = artifact["archive_download_url"] + response = requests.get(download_url, headers=headers) + response.raise_for_status() + + target = docs_source.joinpath("artifact/gurobipy-pandas-examples.zip") + os.makedirs(target.parent, exist_ok=True) + if target.exists(): + target.unlink() + with target.open("wb") as outfile: + outfile.write(response.content) + + print(f"Downloaded {target}") + return True + + return False + + +assert os.environ.get("READTHEDOCS") == "True" + +success = download_executed_notebooks( + runs_url="https://api.github.com/repos/Gurobi/gurobipy-pandas/actions/runs", + gh_token=os.environ["GH_API_TOKEN"], + head_sha=os.environ["READTHEDOCS_GIT_COMMIT_HASH"], +) + +if success: + # Success, RTD build can continue + sys.exit(0) +else: + # Cancels the RTD build (rely on a later trigger to rebuild) + sys.exit(183) + +# Any exception would be a build failure From 6d0ea4f77685db0b5b3a526b84fd1185be7d1aa5 Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Sun, 21 Jul 2024 23:12:21 +1000 Subject: [PATCH 03/10] Fetch artifact as docs pre-build step --- .readthedocs.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index c7299ce..85996c7 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,6 +4,9 @@ build: os: "ubuntu-22.04" tools: python: "3.11" + jobs: + pre_build: + - python scripts/artifacts.py sphinx: fail_on_warning: true From 1baa237eef20d85fca4d46323274695ac7ba26fe Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Sun, 21 Jul 2024 23:19:00 +1000 Subject: [PATCH 04/10] Include artifact in docs build --- docs/source/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1891dcb..128d40b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -45,6 +45,8 @@ } html_favicon = "https://www.gurobi.com/favicon.ico" +html_extra_path = ["artifact/gurobipy-pandas-examples.zip"] + nbsphinx_kernel_name = "python3" intersphinx_mapping = { From 9500ffeea01b76ae296af9487dba80c1e0c75178 Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Sun, 21 Jul 2024 23:25:29 +1000 Subject: [PATCH 05/10] Update .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f0efd5c..9f8c0cf 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,6 @@ docs/source/examples/*.ipynb # Vim swap files *.swp + +# Downloaded zip artifact +docs/source/artifact/*.zip From ed66e646feef9a7667d77ca087bce773be31913e Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Sun, 21 Jul 2024 23:46:38 +1000 Subject: [PATCH 06/10] Add example notebooks download banner --- docs/source/conf.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 128d40b..dce167a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -83,3 +83,14 @@ # -- Options for EPUB output epub_show_urls = "footnote" + +# -- Note pointing to notebook downloads + +nbsphinx_prolog = """ + +.. note:: + + This is example is available as a Jupyter notebook. Download it and all + necessary data files :download:`here `. + +""" From 920b9269ccbbdbccfaf691219e016e358fe3707e Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Mon, 22 Jul 2024 00:00:50 +1000 Subject: [PATCH 07/10] Add readthedocs rebuild trigger --- .github/workflows/test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa5d0c9..b96e371 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,3 +22,14 @@ jobs: examples-artifact: uses: ./.github/workflows/examples-artifact.yml needs: [unit-tests] + trigger-rtd-build: + runs-on: ubuntu-latest + needs: [examples-artifact] + steps: + - name: Trigger RTD build + env: + RTD_HOOK_TOKEN: ${{ secrets.RTD_HOOK_TOKEN }} + run: | + curl -X POST -d "branches=main" -d "token=$RTD_HOOK_TOKEN" \ + -d "default_branch=main" \ + https://readthedocs.com/api/v2/webhook/gurobi-optimization-gurobipy-pandas/11030/ From b980ebb894459a245275f22240d53122cbaf6385 Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Mon, 22 Jul 2024 00:09:36 +1000 Subject: [PATCH 08/10] Fix file download --- docs/source/conf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index dce167a..3d80f16 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -45,8 +45,6 @@ } html_favicon = "https://www.gurobi.com/favicon.ico" -html_extra_path = ["artifact/gurobipy-pandas-examples.zip"] - nbsphinx_kernel_name = "python3" intersphinx_mapping = { @@ -91,6 +89,6 @@ .. note:: This is example is available as a Jupyter notebook. Download it and all - necessary data files :download:`here `. + necessary data files :download:`here `. """ From 0a8cb3bb4eada975ae692192b7c85a3af56049af Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Mon, 22 Jul 2024 00:30:03 +1000 Subject: [PATCH 09/10] Update artifacts script Exit 0 if GH_API_TOKEN doesn't exist. This allows pull request builds to go through without an artifact --- scripts/artifacts.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/artifacts.py b/scripts/artifacts.py index b205cea..8d6edee 100644 --- a/scripts/artifacts.py +++ b/scripts/artifacts.py @@ -13,6 +13,7 @@ import requests docs_source = pathlib.Path(__file__).parent.parent.joinpath("docs/source").resolve() +target = docs_source.joinpath("artifact/gurobipy-pandas-examples.zip") def download_executed_notebooks(runs_url, gh_token, head_sha): @@ -51,7 +52,6 @@ def download_executed_notebooks(runs_url, gh_token, head_sha): response = requests.get(download_url, headers=headers) response.raise_for_status() - target = docs_source.joinpath("artifact/gurobipy-pandas-examples.zip") os.makedirs(target.parent, exist_ok=True) if target.exists(): target.unlink() @@ -64,7 +64,13 @@ def download_executed_notebooks(runs_url, gh_token, head_sha): return False -assert os.environ.get("READTHEDOCS") == "True" +if not os.environ.get("GH_API_TOKEN"): + # Pull requests run in this configuration + print("No API token, can't fetch artifacts. Continuing build with dummy file.") + os.makedirs(target.parent, exist_ok=True) + with target.open("wb") as outfile: + outfile.write(b"") + sys.exit(0) success = download_executed_notebooks( runs_url="https://api.github.com/repos/Gurobi/gurobipy-pandas/actions/runs", @@ -74,9 +80,11 @@ def download_executed_notebooks(runs_url, gh_token, head_sha): if success: # Success, RTD build can continue + print("Artifact retrieval succeeded.") sys.exit(0) else: # Cancels the RTD build (rely on a later trigger to rebuild) + print("Aritfact not found. Cancelling build.") sys.exit(183) # Any exception would be a build failure From d92097858d416ee7e89cb68f7fe8290f58ebecea Mon Sep 17 00:00:00 2001 From: Simon Bowly Date: Mon, 22 Jul 2024 00:33:19 +1000 Subject: [PATCH 10/10] Split main branch and PR workflows --- .github/workflows/{test.yml => main.yml} | 9 +++------ .github/workflows/pr.yml | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) rename .github/workflows/{test.yml => main.yml} (83%) create mode 100644 .github/workflows/pr.yml diff --git a/.github/workflows/test.yml b/.github/workflows/main.yml similarity index 83% rename from .github/workflows/test.yml rename to .github/workflows/main.yml index b96e371..80fe92a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -# Regular QA workflow (pull requests, main branch) +# Regular QA workflow for main branch name: Test @@ -6,9 +6,6 @@ on: push: branches: - main - pull_request_target: - branches: - - main jobs: code-quality: @@ -18,10 +15,10 @@ jobs: needs: [code-quality] notebook-examples: uses: ./.github/workflows/notebook-examples.yml - needs: [unit-tests] + needs: [code-quality] examples-artifact: uses: ./.github/workflows/examples-artifact.yml - needs: [unit-tests] + needs: [code-quality] trigger-rtd-build: runs-on: ubuntu-latest needs: [examples-artifact] diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..c140992 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,21 @@ +# QA workflow for pull requests (no RTD trigger) + +name: Pull request + +on: + pull_request_target: + branches: + - main + +jobs: + code-quality: + uses: ./.github/workflows/code-quality.yml + unit-tests: + uses: ./.github/workflows/unit-tests.yml + needs: [code-quality] + notebook-examples: + uses: ./.github/workflows/notebook-examples.yml + needs: [unit-tests] + examples-artifact: + uses: ./.github/workflows/examples-artifact.yml + needs: [notebook-examples]