Skip to content

Commit 9b61df0

Browse files
tandavtandav
committed
Merge branch 'develop' into 'master'
support python3.12 and update CUDA base image See merge request pitchtrack/pitch-detectors!59
2 parents 56d3f9a + a12c7cf commit 9b61df0

25 files changed

+406
-267
lines changed

‎.github/workflows/ci.yml

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,36 +3,36 @@ name: ci
33
on: push
44

55
jobs:
6-
test:
7-
runs-on: [self-hosted, gpu]
8-
steps:
9-
- uses: actions/checkout@v3
6+
# test:
7+
# runs-on: [self-hosted, gpu]
8+
# steps:
9+
# - uses: actions/checkout@v4
1010

11-
- name: test gpu is available
12-
run: nvidia-smi
11+
# - name: test gpu is available
12+
# run: nvidia-smi
1313

14-
- name: build image
15-
run: make build
14+
# - name: build image
15+
# run: make build
1616

17-
- name: test-no-docker
18-
run: make test-no-docker
17+
# - name: test-no-docker
18+
# run: make test-no-docker
1919

20-
- name: test
21-
run: make test
20+
# - name: test
21+
# run: make test
2222

23-
- name: test-no-gpu
24-
run: make test-no-gpu
23+
# - name: test-no-gpu
24+
# run: make test-no-gpu
2525

2626
publish-to-pypi-and-github-release:
2727
if: "startsWith(github.ref, 'refs/tags/')"
2828
runs-on: ubuntu-latest
29-
needs: test
29+
# needs: test
3030
steps:
31-
- uses: actions/checkout@master
31+
- uses: actions/checkout@v4
3232
- name: Set up Python
33-
uses: actions/setup-python@v4
33+
uses: actions/setup-python@v5
3434
with:
35-
python-version: '3.11'
35+
python-version: '3.12'
3636

3737
- name: Install pypa/build
3838
run: python -m pip install --upgrade setuptools build twine

‎.gitlab-ci.yml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
stages:
2+
# - build
3+
- test
4+
5+
variables:
6+
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
7+
PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit"
8+
RUFF_CACHE_DIR: "$CI_PROJECT_DIR/.cache/ruff_cache"
9+
MYPY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/mypy_cache"
10+
11+
cache:
12+
key: $CI_PROJECT_NAME
13+
paths:
14+
- .cache/
15+
16+
17+
test:
18+
stage: test
19+
needs: []
20+
tags:
21+
- u60-docker-gpu
22+
image: tandav/pitch-detectors:12.4.1-cudnn-devel-ubuntu22.0-python3.12
23+
variables:
24+
PITCH_DETECTORS_SPICE_MODEL_PATH: /models/spice_model
25+
PITCH_DETECTORS_PENN_CHECKPOINT_PATH: /models/fcnf0++.pt
26+
script:
27+
- export $(grep -v '^#' $S3_ENV | xargs) && python scripts/download_models.py
28+
- pytest --cov pitch_detectors --cov-report term --cov-report xml --junitxml report.xml
29+
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
30+
artifacts:
31+
when: always
32+
expire_in: 1 week
33+
reports:
34+
coverage_report:
35+
coverage_format: cobertura
36+
path: coverage.xml
37+
junit: report.xml
38+
39+
lint:
40+
stage: test
41+
needs: []
42+
image: python:3.12@sha256:fce9bc7648ef917a5ab67176cf1c7eb41b110452e259736144bc22f32f3aa622
43+
variables:
44+
PIP_INDEX_URL: https://pypi.tandav.me/index/
45+
script:
46+
- pip install .[dev]
47+
- pre-commit run --all-files

‎.pre-commit-config.yaml

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
repos:
22
- repo: https://github.com/pre-commit/pre-commit-hooks
3-
rev: v4.3.0
3+
rev: v4.6.0
44
hooks:
55
- id: check-added-large-files
66
- id: check-yaml
@@ -18,47 +18,46 @@ repos:
1818
- id: detect-private-key
1919
- id: double-quote-string-fixer
2020
- id: name-tests-test
21-
- id: requirements-txt-fixer
2221

2322
- repo: https://github.com/asottile/add-trailing-comma
24-
rev: v2.3.0
23+
rev: v3.1.0
2524
hooks:
2625
- id: add-trailing-comma
2726

2827
- repo: https://github.com/asottile/pyupgrade
29-
rev: v3.1.0
28+
rev: v3.16.0
3029
hooks:
3130
- id: pyupgrade
3231

33-
- repo: https://github.com/pre-commit/mirrors-autopep8
34-
rev: v1.7.0
32+
- repo: https://github.com/hhatto/autopep8
33+
rev: v2.3.1
3534
hooks:
3635
- id: autopep8
3736

3837
- repo: https://github.com/PyCQA/autoflake
39-
rev: v1.7.6
38+
rev: v2.3.1
4039
hooks:
4140
- id: autoflake
4241

43-
- repo: https://github.com/charliermarsh/ruff-pre-commit
44-
rev: v0.0.254
42+
- repo: https://github.com/astral-sh/ruff-pre-commit
43+
rev: v0.5.0
4544
hooks:
4645
- id: ruff
4746
args: [--fix, --exit-non-zero-on-fix]
4847

4948
- repo: https://github.com/PyCQA/pylint
50-
rev: v2.17.0
49+
rev: v3.2.4
5150
hooks:
5251
- id: pylint
5352
additional_dependencies: ["pylint-per-file-ignores"]
5453

5554
- repo: https://github.com/pre-commit/mirrors-mypy
56-
rev: v0.982
55+
rev: v1.10.1
5756
hooks:
5857
- id: mypy
5958
additional_dependencies: [types-redis, types-tabulate, pydantic]
6059

61-
- repo: https://github.com/RobertCraigie/pyright-python
62-
rev: v1.1.302
63-
hooks:
64-
- id: pyright
60+
# - repo: https://github.com/RobertCraigie/pyright-python
61+
# rev: v1.1.369
62+
# hooks:
63+
# - id: pyright

‎Dockerfile

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎Makefile

Lines changed: 0 additions & 56 deletions
This file was deleted.

‎README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
[![pipeline status](https://gitlab.tandav.me/pitchtrack/pitch-detectors/badges/master/pipeline.svg)](https://gitlab.tandav.me/pitchtrack/pitch-detectors/-/commits/master)
2+
13
# pitch-detectors
24
collection of pitch (f0, fundamental frequency) detection algorithms with unified interface
35

@@ -50,3 +52,8 @@ plt.show()
5052
## additional features
5153
- [ ] robust (vote + median) ensemble algorithm using all models
5254
- [ ] json import/export
55+
56+
## notes:
57+
Tests are running in subprocess (using `scripts/run_algorithm.py`) to avoid pytorch cuda import caching.
58+
It's difficult to disable gpu after it has been initialized. (https://github.com/pytorch/pytorch/issues/9158)
59+
It is also difficult to set correct PATH and LD_LIBRARY_PATH without a subprocess.

‎Taskfile.yml

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
version: '3'
2+
dotenv: ['.env']
3+
vars:
4+
BASE_IMAGE: tandav/pitch-detectors-base:12.4.1-cudnn-devel-ubuntu22.0-python3.12
5+
IMAGE: tandav/pitch-detectors:12.4.1-cudnn-devel-ubuntu22.0-python3.12
6+
7+
tasks:
8+
build-base:
9+
cmd: docker build --tag {{.BASE_IMAGE}} --file docker/base.dockerfile .
10+
11+
push-base:
12+
cmd: docker push {{.BASE_IMAGE}}
13+
14+
build:
15+
cmd: docker build --build-arg="BASE_IMAGE={{.BASE_IMAGE}}" --tag {{.IMAGE}} --file docker/pitch-detectors.dockerfile .
16+
17+
push:
18+
cmd: docker push {{.IMAGE}}
19+
20+
test:
21+
deps: [build]
22+
cmd: >
23+
docker run --rm -t --gpus all
24+
-v /media/tandav/sg8tb1/downloads-archive/libmv-data/spice_model:/spice_model:ro
25+
-v /media/tandav/sg8tb1/downloads-archive/libmv-data/fcnf0++.pt:/fcnf0++.pt:ro
26+
{{.IMAGE}}
27+
pytest -v
28+
29+
test-no-docker:
30+
cmd: pytest -v
31+
32+
freeze:
33+
cmd: docker run --rm -t --gpus all {{.IMAGE}} /venv/bin/pip freeze > freeze.txt
34+
35+
bumpver:
36+
desc: 'Bump version. Pass --<part>. Usage example: task bumpver -- --minor'
37+
cmds:
38+
- bumpver update --no-fetch {{.CLI_ARGS}}
39+
40+
evaluation:
41+
deps: [build]
42+
cmd: >
43+
docker run --rm -t --gpus all
44+
-e PITCH_DETECTORS_GPU=true
45+
-e REDIS_URL={{.REDIS_URL}}
46+
-v /media/tandav/sg8tb1/downloads-archive/f0-datasets:/app/f0-datasets:ro
47+
{{.IMAGE}}
48+
python -m pitch_detectors.evaluation
49+
50+
table:
51+
cmd: python -m pitch_detectors.evaluation.table

‎docker/base.dockerfile

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04@sha256:0a1cb6e7bd047a1067efe14efdf0276352d5ca643dfd77963dab1a4f05a003a4
2+
3+
# https://github.com/NVIDIA/nvidia-docker/wiki/Usage
4+
# https://github.com/NVIDIA/nvidia-docker/issues/531
5+
ENV NVIDIA_DRIVER_CAPABILITIES compute,video,utility
6+
ENV DEBIAN_FRONTEND=noninteractive
7+
8+
ARG PYTHON_VERSION=3.12
9+
10+
RUN apt-get update && \
11+
apt-get install -y software-properties-common && \
12+
add-apt-repository -y ppa:deadsnakes/ppa && \
13+
apt-get install -y python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv libsndfile-dev libasound-dev portaudio19-dev
14+
15+
# this is only need for crepe @ git+https://github.com/tandav/crepe
16+
RUN apt-get install -y git
17+
18+
# https://pythonspeed.com/articles/activate-virtualenv-dockerfile/
19+
ENV VIRTUAL_ENV=/venv
20+
RUN python${PYTHON_VERSION} -m venv $VIRTUAL_ENV
21+
ENV PATH="$VIRTUAL_ENV/bin:$PATH"

‎docker/pitch-detectors.dockerfile

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
ARG BASE_IMAGE
2+
FROM ${BASE_IMAGE}
3+
4+
WORKDIR /app
5+
COPY pyproject.toml .
6+
ENV PIP_INDEX_URL=https://pypi.tandav.me/index/
7+
RUN --mount=type=cache,target=/root/.cache/pip \
8+
pip install --upgrade pip setuptools wheel && \
9+
pip install .[dev]
10+
11+
COPY pitch_detectors /app/pitch_detectors
12+
13+
RUN --mount=type=cache,target=/root/.cache/pip \
14+
pip install --no-deps .
15+
16+
COPY tests /app/tests
17+
COPY scripts/ /app/scripts
18+
COPY data /app/data

0 commit comments

Comments
 (0)