Merge pull request #26 from astronomy-commons/sean/benchmarking

Add benchmarking

.copier-answers.yml

@@ -1,12 +1,13 @@
 # Changes here will be overwritten by Copier
-_commit: v1.4.1
+_commit: v1.4.2
 _src_path: gh:lincc-frameworks/python-project-template
 author_email: [email protected]
 author_name: LINCC Frameworks
 create_example_module: false
 custom_install: true
+include_benchmarks: true
 include_docs: true
-include_notebooks: false
+include_notebooks: true
 mypy_type_checking: basic
 package_name: lsdb
 preferred_linter: pylint

.github/workflows/asv-main.yml

@@ -0,0 +1,107 @@
+# This workflow will run benchmarks with airspeed velocity (asv), 
+# store the new results in the "benchmarks" branch and publish them
+# to a dashboard on GH Pages.
+
+name: Run ASV benchmarks for main
+
+on:
+  push:
+    branches: [ main ]
+
+env:
+  PYTHON_VERSION: "3.10"
+  WORKING_DIR: ${{ github.workspace }}/benchmarks
+
+jobs:
+
+  consecutiveness:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set workflows on main to run consecutively
+        uses: mktcode/consecutive-workflow-action@eb43c6b5852dd0e33efa797a1817196d06daa4b2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+  setup-python:
+    runs-on: ubuntu-latest
+    needs: consecutiveness
+
+    steps:
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+  asv-main:
+    runs-on: ubuntu-latest
+    needs: setup-python
+
+    permissions:
+      contents: write
+
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+
+    steps:
+      - name: Checkout main branch of the repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          python -m pip install --upgrade pip
+          pip install asv==0.5.1 virtualenv tabulate
+
+      - name: Configure git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Create ASV machine config file
+        run: asv machine --machine gh-runner --yes
+
+      - name: Fetch previous results from the "benchmarks" branch
+        run: |
+          if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
+            git merge origin/benchmarks \
+              --allow-unrelated-histories \
+              --no-commit
+            mv ../_results .
+          fi
+
+      - name: Run ASV for the main branch
+        run: asv run ALL --skip-existing
+
+      - name: Submit new results to the "benchmarks" branch
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          branch: benchmarks
+          folder: ${{ env.WORKING_DIR }}/_results
+          target-folder: _results
+
+      - name: Generate dashboard HTML
+        run: |
+          asv show
+          asv publish
+
+      - name: Deploy to Github pages
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          branch: gh-pages
+          folder: ${{ env.WORKING_DIR }}/_html

.github/workflows/asv-nightly.yml

@@ -0,0 +1,80 @@
+# This workflow will run daily at 06:45.
+# It will run benchmarks with airspeed velocity (asv)
+# and compare performance with the previous nightly build.
+
+name: Run benchmarks nightly job
+
+on:
+  schedule:
+    - cron: 45 6 * * *
+
+env:
+  PYTHON_VERSION: "3.10"
+  WORKING_DIR: ${{ github.workspace }}/benchmarks
+  NIGHTLY_HASH_FILE: nightly-hash
+
+jobs:
+
+  asv-nightly:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+
+    steps:
+      - name: Checkout main branch of the repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          python -m pip install --upgrade pip
+          pip install asv==0.5.1 virtualenv
+
+      - name: Create ASV machine config file
+        run: asv machine --machine gh-runner --yes
+
+      - name: Get nightly dates under comparison
+        id: nightly-dates
+        run: |
+          echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+          echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+      - name: Use last nightly commit hash from cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.WORKING_DIR }}
+          key: nightly-results-${{ steps.nightly-dates.outputs.yesterday }}
+
+      - name: Run comparison of main against last nightly build
+        run: |
+          HASH_FILE=${{ env.NIGHTLY_HASH_FILE }}
+          CURRENT_HASH=${{ github.sha }}
+
+          if [ -f $HASH_FILE ]; then          
+            PREV_HASH=$(cat $HASH_FILE)
+            asv continuous $PREV_HASH $CURRENT_HASH || true
+            asv compare $PREV_HASH $CURRENT_HASH --sort ratio
+          fi
+
+          echo $CURRENT_HASH > $HASH_FILE
+
+      - name: Update last nightly hash in cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.WORKING_DIR }}
+          key: nightly-results-${{ steps.nightly-dates.outputs.today }}

.github/workflows/asv-pr.yml

@@ -0,0 +1,103 @@
+# This workflow will run benchmarks with airspeed velocity (asv) for pull requests.
+# It will compare the performance of the main branch with the performance of the merge
+# with the new changes and publish a comment with this assessment.  
+
+name: Run ASV benchmarks for PR
+
+on:
+  pull_request:
+    branches: [ main ]
+
+env:
+  PYTHON_VERSION: "3.10"
+  WORKING_DIR: ${{ github.workspace }}/benchmarks
+
+jobs:
+
+  setup-python:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Set up Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: "${{ env.PYTHON_VERSION }}"
+
+  asv-pr:
+    runs-on: ubuntu-latest
+    needs: setup-python
+
+    permissions:
+      actions: read
+      pull-requests: write
+
+    defaults:
+      run:
+        working-directory: ${{ env.WORKING_DIR }}
+
+    steps:
+      - name: Checkout PR branch of the repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Cache Python ${{ env.PYTHON_VERSION }}
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: python-${{ env.PYTHON_VERSION }}
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          python -m pip install --upgrade pip
+          pip install asv==0.5.1 virtualenv tabulate lf-asv-formatter
+
+      - name: Get current job logs URL
+        uses: Tiryoh/gha-jobid-action@v0
+        id: jobs
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          job_name: ${{ github.job }}
+
+      - name: Create ASV machine config file
+        run: asv machine --machine gh-runner --yes
+
+      - name: Run comparison of PR against main branch
+        run: |
+          git remote add upstream https://github.com/${{ github.repository }}.git
+          git fetch upstream
+          asv continuous upstream/main HEAD || true
+          asv compare upstream/main HEAD --sort ratio | tee output
+          python -m lf_asv_formatter
+          printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output
+        env:
+          STEP_URL: "${{ steps.jobs.outputs.html_url }}#step:8:1"
+
+      - name: Publish comment to PR
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const workingDir = process.env.WORKING_DIR;
+            try {
+              process.chdir(workingDir);
+              const comment = fs.readFileSync('output', 'utf-8');
+              const { data } = await github.rest.issues.createComment({
+                ...context.repo,
+                issue_number: context.issue.number,
+                body: comment,
+              });
+              console.log('Comment published:', data.html_url);
+            } catch (err) {
+              console.error(err);
+            }

.github/workflows/build-documentation.yml

@@ -26,6 +26,9 @@ jobs:
         python -m pip install --upgrade pip
         if [ -f docs/requirements.txt ]; then pip install -r docs/requirements.txt; fi
         pip install .
+    - name: Install notebook requirements
+      run: |
+        sudo apt-get install pandoc
     - name: Build docs
       run: |
         sphinx-build -T -E -b html -d docs/build/doctrees ./docs docs/build/html

.gitignore

@@ -138,3 +138,13 @@ dask-worker-space/
 
 # tmp directory
 tmp/
+
+# benchmarking
+_results/
+_html/
+
+# Mac OS
+.DS_Store
+
+# IntelliJ
+.idea

.pre-commit-config.yaml

@@ -116,7 +116,10 @@ repos:
 
           ]
 
-    # Make sure Sphinx can build the documentation without issues.
+    # Make sure Sphinx can build the documentation while explicitly omitting 
+    # notebooks from the docs, so users don't have to wait through the execution 
+    # of each notebook or each commit. By default, these will be checked in the 
+    # GitHub workflows.
   - repo: local
     hooks:
       - id: sphinx-build
@@ -127,12 +130,15 @@ repos:
         exclude_types: [file, symlink]
         args:
           [
+            "-M", # Run sphinx in make mode, so we can use -D flag later
+                  # Note: -M requires next 3 args to be builder, source, output
+            "html", # Specify builder
+            "./docs", # Source directory of documents
+            "./_readthedocs", # Output directory for rendered documents
             "-T", # Show full trace back on exception
-            "-E", # Don't use saved env. always read all files.
-            "-b", # Flag to select which builder to use
-            "html", # Use the HTML builder
+            "-E", # Don't use saved env; always read all files
             "-d", # Flag for cached environment and doctrees
-            "./docs/_build/doctrees", # directory
-            "./docs", # Source directory of documents
-            "./_readthedocs", # Output directory for rendered documents.
+            "./docs/_build/doctrees", # Directory
+            "-D", # Flag to override settings in conf.py
+            "exclude_patterns=notebooks/*", # Exclude our notebooks from pre-commit
           ]