refactor: only increment dataloader _counter when on object #1746
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Monty | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| env: | |
| UV_PYTHON: 3.9.22 # version of Python to use in `uv` environment | |
| jobs: | |
| build_sphinx_monty: | |
| name: build-sphinx-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu2204-2core | |
| needs: | |
| - check_dependencies_monty # Don't run if dependency check fails | |
| - check_license_monty # Don't run if license check fails | |
| - check_style_monty # Don't run if style check fails | |
| - check_types_monty # Don't run if type check fails | |
| - install_monty | |
| - should_run_monty | |
| steps: | |
| - name: Restore cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ needs.install_monty.outputs.conda_env_cache_key_sha }} | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| lfs: true | |
| path: tbp.monty | |
| - name: Build API docs | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| export PATH="$HOME/miniconda/bin:$PATH" | |
| source activate tbp.monty | |
| pip install -e .[generate_api_docs_tool] | |
| cd tools/generate_api_docs | |
| make apidoc html | |
| - name: Store API docs artifact | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sphinx-html-${{ github.sha }} | |
| path: tbp.monty/tools/generate_api_docs/build/html | |
| build_wheel_monty: | |
| name: build-wheel-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu2204-2core | |
| needs: | |
| - install_monty # Needed for the cache key | |
| - should_run_monty | |
| - test_monty | |
| steps: | |
| - name: Restore cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ needs.install_monty.outputs.conda_env_cache_key_sha }} | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Build wheel | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| export PATH="$HOME/miniconda/bin:$PATH" | |
| source activate tbp.monty | |
| pip install -e .[build] | |
| python -m build | |
| - name: Store wheel | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dist | |
| path: tbp.monty/dist | |
| check_dependencies_monty: | |
| name: check-dependencies-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu2204-2core | |
| needs: | |
| - install_monty | |
| - should_run_monty | |
| steps: | |
| - name: Restore cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ needs.install_monty.outputs.conda_env_cache_key_sha }} | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Check dependencies | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| export PATH="$HOME/miniconda/bin:$PATH" | |
| source activate tbp.monty | |
| deptry benchmarks src tests | |
| check_license_monty: | |
| name: check-license-monty | |
| runs-on: ubuntu-latest | |
| needs: should_run_monty | |
| steps: | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Check license | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| FILES=$(grep -l -r ' GNU \| MPL \| Mozilla ' * ) || true | |
| if [ -n "$FILES" ]; then | |
| echo "Found Copyleft Licensed files: $FILES" | |
| exit 1 | |
| fi | |
| check_style_monty: | |
| name: check-style-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu2204-2core | |
| needs: | |
| - install_monty | |
| - should_run_monty | |
| steps: | |
| - name: Restore cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ needs.install_monty.outputs.conda_env_cache_key_sha }} | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Check style | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| export PATH="$HOME/miniconda/bin:$PATH" | |
| source activate tbp.monty | |
| mkdir -p test_results/ruff | |
| ruff check src tests benchmarks | |
| ruff format --check src tests benchmarks | |
| check_types_monty: | |
| name: check-types-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu2204-2core | |
| needs: | |
| - install_monty | |
| - should_run_monty | |
| steps: | |
| - name: Restore cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ needs.install_monty.outputs.conda_env_cache_key_sha }} | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Check types | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| continue-on-error: true # Accept type errors for now | |
| working-directory: tbp.monty | |
| run: | | |
| export PATH="$HOME/miniconda/bin:$PATH" | |
| source activate tbp.monty | |
| mkdir -p test_results/mypy | |
| mypy src tests benchmarks | |
| install_monty: | |
| name: install-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu2204-2core | |
| needs: | |
| - check_license_monty # Don't run if license check fails | |
| - should_run_monty | |
| outputs: | |
| conda_env_cache_key_sha: ${{ steps.generate_cache_key.outputs.conda_env_cache_key_sha }} | |
| steps: | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Generate cache key | |
| id: generate_cache_key | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| run: | | |
| mkdir -p ~/tbp | |
| ln -s $GITHUB_WORKSPACE/tbp.monty ~/tbp/tbp.monty | |
| shasum -a 256 ~/tbp/tbp.monty/environment_arm64.yml | awk '{print $1}' > ~/tbp/environment_arm64.sha | |
| shasum -a 256 ~/tbp/tbp.monty/environment.yml | awk '{print $1}' > ~/tbp/environment.sha | |
| shasum -a 256 ~/tbp/tbp.monty/pyproject.toml | awk '{print $1}' > ~/tbp/pyproject.toml.sha | |
| echo "monty-${RUNNER_OS}-$(cat ~/tbp/environment_arm64.sha)-$(cat ~/tbp/environment.sha)-$(cat ~/tbp/pyproject.toml.sha)" > ~/tbp/conda_env_cache_key.txt | |
| echo "conda_env_cache_key_sha=$(cat ~/tbp/conda_env_cache_key.txt | shasum -a 256 | awk '{print $1}')" >> $GITHUB_OUTPUT | |
| - name: Set up Python 3.8 | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.8" | |
| - name: Restore cache | |
| id: restore_cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ steps.generate_cache_key.outputs.conda_env_cache_key_sha }} | |
| lookup-only: true | |
| - name: Install miniconda | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' && steps.restore_cache.outputs.cache-hit != 'true' }} | |
| run: | | |
| if [ ! -d ~/miniconda ] | |
| then | |
| wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh | |
| bash ~/miniconda.sh -b -p ~/miniconda | |
| rm ~/miniconda.sh | |
| fi | |
| - name: Create conda environment | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' && steps.restore_cache.outputs.cache-hit != 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| export PATH="$HOME/miniconda/bin:$PATH" | |
| (conda env list | grep tbp.monty) && conda remove --name tbp.monty --all --yes || true | |
| conda env create | |
| source activate tbp.monty | |
| pip install -e .[dev] | |
| pip list | |
| conda list | |
| - name: Save cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' && steps.restore_cache.outputs.cache-hit != 'true' }} | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ steps.restore_cache.outputs.cache-primary-key }} | |
| should_run_monty: | |
| name: should-run-monty | |
| runs-on: ubuntu-latest | |
| if: ${{ github.repository_owner == 'thousandbrainsproject' }} | |
| outputs: | |
| should_run_monty: ${{ steps.should_run_monty.outputs.should_run_monty }} | |
| steps: | |
| - name: Checkout tbp.monty | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| lfs: true | |
| path: tbp.monty | |
| - name: Should Run Monty | |
| id: should_run_monty | |
| uses: ./tbp.monty/.github/actions/should_run_monty | |
| with: | |
| git_base_ref: ${{ github.base_ref }} | |
| git_sha: ${{ github.sha }} | |
| github_event_name: ${{ github.event_name }} | |
| working_directory: tbp.monty | |
| test_monty: | |
| name: test-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu-nvidia-4core-t4gpu | |
| timeout-minutes: 120 | |
| needs: | |
| - check_dependencies_monty # Don't run if dependency check fails | |
| - check_style_monty # Don't run if style check fails | |
| - check_types_monty # Don't run if type check fails | |
| - install_monty | |
| - should_run_monty | |
| steps: | |
| - name: Restore cache | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/miniconda | |
| key: ${{ needs.install_monty.outputs.conda_env_cache_key_sha }} | |
| - name: Install dependencies | |
| # Installing before checkout because GitLFS is missing from the image | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| run: | | |
| sudo apt update -y | |
| sudo apt install -y --no-install-recommends xvfb libegl1-mesa-dev git-lfs | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Run python tests | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| export PATH="$HOME/miniconda/bin:$PATH" | |
| source activate tbp.monty | |
| set -e | |
| mkdir -p test_results/pytest | |
| xvfb-run pytest --cov --cov-report html --cov-report term --junitxml=test_results/pytest/results.xml --verbose | |
| - name: Store test results | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test_results | |
| path: tbp.monty/test_results/pytest | |
| - name: Store coverage | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test_coverage | |
| path: tbp.monty/htmlcov | |
| test_uv_monty: | |
| name: test-uv-monty | |
| runs-on: | |
| group: tbp.monty | |
| labels: tbp-linux-x64-ubuntu2204-8core | |
| timeout-minutes: 120 | |
| needs: | |
| # TODO: check all these using `uv` once we move off Conda | |
| - check_dependencies_monty # Don't run if dependency check fails | |
| - check_style_monty # Don't run if style check fails | |
| - check_types_monty # Don't run if type check fails | |
| - should_run_monty | |
| steps: | |
| - name: Checkout tbp.monty | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/checkout@v4 | |
| with: | |
| lfs: true | |
| path: tbp.monty | |
| - name: Install uv | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: '**/uv.lock' | |
| - name: Create uv environment | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: uv sync --locked --extra dev --extra simulator_mujoco | |
| - name: Run tests | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| working-directory: tbp.monty | |
| run: | | |
| mkdir -p test_results/pytest | |
| uv run pytest --cov --cov-report html --cov-report term --junitxml=test_results/pytest/results.xml --verbose | |
| - name: Store test results | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test_results_uv | |
| path: tbp.monty/test_results/pytest | |
| - name: Store coverage | |
| if: ${{ needs.should_run_monty.outputs.should_run_monty == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test_coverage_uv | |
| path: tbp.monty/htmlcov |