Skip to content

Commit 6a10f81

Browse files
smcguire-cmusmcguire-cmu
authored
Merge pull request #26 from astronomy-commons/sean/benchmarking
Add benchmarking
2 parents ea8f017 + 6492eb9 commit 6a10f81

14 files changed

+447
-11
lines changed

‎.copier-answers.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
# Changes here will be overwritten by Copier
2-
_commit: v1.4.1
2+
_commit: v1.4.2
33
_src_path: gh:lincc-frameworks/python-project-template
44
author_email: [email protected]
55
author_name: LINCC Frameworks
66
create_example_module: false
77
custom_install: true
8+
include_benchmarks: true
89
include_docs: true
9-
include_notebooks: false
10+
include_notebooks: true
1011
mypy_type_checking: basic
1112
package_name: lsdb
1213
preferred_linter: pylint

‎.github/workflows/asv-main.yml

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# This workflow will run benchmarks with airspeed velocity (asv),
2+
# store the new results in the "benchmarks" branch and publish them
3+
# to a dashboard on GH Pages.
4+
5+
name: Run ASV benchmarks for main
6+
7+
on:
8+
push:
9+
branches: [ main ]
10+
11+
env:
12+
PYTHON_VERSION: "3.10"
13+
WORKING_DIR: ${{ github.workspace }}/benchmarks
14+
15+
jobs:
16+
17+
consecutiveness:
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Set workflows on main to run consecutively
22+
uses: mktcode/consecutive-workflow-action@eb43c6b5852dd0e33efa797a1817196d06daa4b2
23+
with:
24+
token: ${{ secrets.GITHUB_TOKEN }}
25+
26+
setup-python:
27+
runs-on: ubuntu-latest
28+
needs: consecutiveness
29+
30+
steps:
31+
- name: Cache Python ${{ env.PYTHON_VERSION }}
32+
uses: actions/cache@v3
33+
with:
34+
path: ~/.cache/pip
35+
key: python-${{ env.PYTHON_VERSION }}
36+
37+
- name: Set up Python ${{ env.PYTHON_VERSION }}
38+
uses: actions/setup-python@v4
39+
with:
40+
python-version: "${{ env.PYTHON_VERSION }}"
41+
42+
asv-main:
43+
runs-on: ubuntu-latest
44+
needs: setup-python
45+
46+
permissions:
47+
contents: write
48+
49+
defaults:
50+
run:
51+
working-directory: ${{ env.WORKING_DIR }}
52+
53+
steps:
54+
- name: Checkout main branch of the repository
55+
uses: actions/checkout@v3
56+
with:
57+
fetch-depth: 0
58+
59+
- name: Cache Python ${{ env.PYTHON_VERSION }}
60+
uses: actions/cache@v3
61+
with:
62+
path: ~/.cache/pip
63+
key: python-${{ env.PYTHON_VERSION }}
64+
65+
- name: Install dependencies
66+
run: |
67+
sudo apt-get update
68+
python -m pip install --upgrade pip
69+
pip install asv==0.5.1 virtualenv tabulate
70+
71+
- name: Configure git
72+
run: |
73+
git config user.name "github-actions[bot]"
74+
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
75+
76+
- name: Create ASV machine config file
77+
run: asv machine --machine gh-runner --yes
78+
79+
- name: Fetch previous results from the "benchmarks" branch
80+
run: |
81+
if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
82+
git merge origin/benchmarks \
83+
--allow-unrelated-histories \
84+
--no-commit
85+
mv ../_results .
86+
fi
87+
88+
- name: Run ASV for the main branch
89+
run: asv run ALL --skip-existing
90+
91+
- name: Submit new results to the "benchmarks" branch
92+
uses: JamesIves/github-pages-deploy-action@v4
93+
with:
94+
branch: benchmarks
95+
folder: ${{ env.WORKING_DIR }}/_results
96+
target-folder: _results
97+
98+
- name: Generate dashboard HTML
99+
run: |
100+
asv show
101+
asv publish
102+
103+
- name: Deploy to Github pages
104+
uses: JamesIves/github-pages-deploy-action@v4
105+
with:
106+
branch: gh-pages
107+
folder: ${{ env.WORKING_DIR }}/_html

‎.github/workflows/asv-nightly.yml

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# This workflow will run daily at 06:45.
2+
# It will run benchmarks with airspeed velocity (asv)
3+
# and compare performance with the previous nightly build.
4+
5+
name: Run benchmarks nightly job
6+
7+
on:
8+
schedule:
9+
- cron: 45 6 * * *
10+
11+
env:
12+
PYTHON_VERSION: "3.10"
13+
WORKING_DIR: ${{ github.workspace }}/benchmarks
14+
NIGHTLY_HASH_FILE: nightly-hash
15+
16+
jobs:
17+
18+
asv-nightly:
19+
runs-on: ubuntu-latest
20+
21+
defaults:
22+
run:
23+
working-directory: ${{ env.WORKING_DIR }}
24+
25+
steps:
26+
- name: Checkout main branch of the repository
27+
uses: actions/checkout@v3
28+
with:
29+
fetch-depth: 0
30+
31+
- name: Cache Python ${{ env.PYTHON_VERSION }}
32+
uses: actions/cache@v3
33+
with:
34+
path: ~/.cache/pip
35+
key: python-${{ env.PYTHON_VERSION }}
36+
37+
- name: Set up Python ${{ env.PYTHON_VERSION }}
38+
uses: actions/setup-python@v4
39+
with:
40+
python-version: "${{ env.PYTHON_VERSION }}"
41+
42+
- name: Install dependencies
43+
run: |
44+
sudo apt-get update
45+
python -m pip install --upgrade pip
46+
pip install asv==0.5.1 virtualenv
47+
48+
- name: Create ASV machine config file
49+
run: asv machine --machine gh-runner --yes
50+
51+
- name: Get nightly dates under comparison
52+
id: nightly-dates
53+
run: |
54+
echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT
55+
echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
56+
57+
- name: Use last nightly commit hash from cache
58+
uses: actions/cache@v3
59+
with:
60+
path: ${{ env.WORKING_DIR }}
61+
key: nightly-results-${{ steps.nightly-dates.outputs.yesterday }}
62+
63+
- name: Run comparison of main against last nightly build
64+
run: |
65+
HASH_FILE=${{ env.NIGHTLY_HASH_FILE }}
66+
CURRENT_HASH=${{ github.sha }}
67+
68+
if [ -f $HASH_FILE ]; then
69+
PREV_HASH=$(cat $HASH_FILE)
70+
asv continuous $PREV_HASH $CURRENT_HASH || true
71+
asv compare $PREV_HASH $CURRENT_HASH --sort ratio
72+
fi
73+
74+
echo $CURRENT_HASH > $HASH_FILE
75+
76+
- name: Update last nightly hash in cache
77+
uses: actions/cache@v3
78+
with:
79+
path: ${{ env.WORKING_DIR }}
80+
key: nightly-results-${{ steps.nightly-dates.outputs.today }}

‎.github/workflows/asv-pr.yml

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# This workflow will run benchmarks with airspeed velocity (asv) for pull requests.
2+
# It will compare the performance of the main branch with the performance of the merge
3+
# with the new changes and publish a comment with this assessment.
4+
5+
name: Run ASV benchmarks for PR
6+
7+
on:
8+
pull_request:
9+
branches: [ main ]
10+
11+
env:
12+
PYTHON_VERSION: "3.10"
13+
WORKING_DIR: ${{ github.workspace }}/benchmarks
14+
15+
jobs:
16+
17+
setup-python:
18+
runs-on: ubuntu-latest
19+
20+
steps:
21+
- name: Cache Python ${{ env.PYTHON_VERSION }}
22+
uses: actions/cache@v3
23+
with:
24+
path: ~/.cache/pip
25+
key: python-${{ env.PYTHON_VERSION }}
26+
27+
- name: Set up Python ${{ env.PYTHON_VERSION }}
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: "${{ env.PYTHON_VERSION }}"
31+
32+
asv-pr:
33+
runs-on: ubuntu-latest
34+
needs: setup-python
35+
36+
permissions:
37+
actions: read
38+
pull-requests: write
39+
40+
defaults:
41+
run:
42+
working-directory: ${{ env.WORKING_DIR }}
43+
44+
steps:
45+
- name: Checkout PR branch of the repository
46+
uses: actions/checkout@v3
47+
with:
48+
fetch-depth: 0
49+
50+
- name: Cache Python ${{ env.PYTHON_VERSION }}
51+
uses: actions/cache@v3
52+
with:
53+
path: ~/.cache/pip
54+
key: python-${{ env.PYTHON_VERSION }}
55+
56+
- name: Install dependencies
57+
run: |
58+
sudo apt-get update
59+
python -m pip install --upgrade pip
60+
pip install asv==0.5.1 virtualenv tabulate lf-asv-formatter
61+
62+
- name: Get current job logs URL
63+
uses: Tiryoh/gha-jobid-action@v0
64+
id: jobs
65+
with:
66+
github_token: ${{ secrets.GITHUB_TOKEN }}
67+
job_name: ${{ github.job }}
68+
69+
- name: Create ASV machine config file
70+
run: asv machine --machine gh-runner --yes
71+
72+
- name: Run comparison of PR against main branch
73+
run: |
74+
git remote add upstream https://github.com/${{ github.repository }}.git
75+
git fetch upstream
76+
asv continuous upstream/main HEAD || true
77+
asv compare upstream/main HEAD --sort ratio | tee output
78+
python -m lf_asv_formatter
79+
printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output
80+
env:
81+
STEP_URL: "${{ steps.jobs.outputs.html_url }}#step:8:1"
82+
83+
- name: Publish comment to PR
84+
uses: actions/github-script@v6
85+
with:
86+
github-token: ${{ secrets.GITHUB_TOKEN }}
87+
script: |
88+
const fs = require('fs');
89+
const path = require('path');
90+
91+
const workingDir = process.env.WORKING_DIR;
92+
try {
93+
process.chdir(workingDir);
94+
const comment = fs.readFileSync('output', 'utf-8');
95+
const { data } = await github.rest.issues.createComment({
96+
...context.repo,
97+
issue_number: context.issue.number,
98+
body: comment,
99+
});
100+
console.log('Comment published:', data.html_url);
101+
} catch (err) {
102+
console.error(err);
103+
}

‎.github/workflows/build-documentation.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ jobs:
2626
python -m pip install --upgrade pip
2727
if [ -f docs/requirements.txt ]; then pip install -r docs/requirements.txt; fi
2828
pip install .
29+
- name: Install notebook requirements
30+
run: |
31+
sudo apt-get install pandoc
2932
- name: Build docs
3033
run: |
3134
sphinx-build -T -E -b html -d docs/build/doctrees ./docs docs/build/html

‎.gitignore

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,3 +138,13 @@ dask-worker-space/
138138

139139
# tmp directory
140140
tmp/
141+
142+
# benchmarking
143+
_results/
144+
_html/
145+
146+
# Mac OS
147+
.DS_Store
148+
149+
# IntelliJ
150+
.idea

‎.pre-commit-config.yaml

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,10 @@ repos:
116116

117117
]
118118

119-
# Make sure Sphinx can build the documentation without issues.
119+
# Make sure Sphinx can build the documentation while explicitly omitting
120+
# notebooks from the docs, so users don't have to wait through the execution
121+
# of each notebook or each commit. By default, these will be checked in the
122+
# GitHub workflows.
120123
- repo: local
121124
hooks:
122125
- id: sphinx-build
@@ -127,12 +130,15 @@ repos:
127130
exclude_types: [file, symlink]
128131
args:
129132
[
133+
"-M", # Run sphinx in make mode, so we can use -D flag later
134+
# Note: -M requires next 3 args to be builder, source, output
135+
"html", # Specify builder
136+
"./docs", # Source directory of documents
137+
"./_readthedocs", # Output directory for rendered documents
130138
"-T", # Show full trace back on exception
131-
"-E", # Don't use saved env. always read all files.
132-
"-b", # Flag to select which builder to use
133-
"html", # Use the HTML builder
139+
"-E", # Don't use saved env; always read all files
134140
"-d", # Flag for cached environment and doctrees
135-
"./docs/_build/doctrees", # directory
136-
"./docs", # Source directory of documents
137-
"./_readthedocs", # Output directory for rendered documents.
141+
"./docs/_build/doctrees", # Directory
142+
"-D", # Flag to override settings in conf.py
143+
"exclude_patterns=notebooks/*", # Exclude our notebooks from pre-commit
138144
]

‎benchmarks/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)