From 6ad3b60529f82371dcc939919810efdd6bc284c6 Mon Sep 17 00:00:00 2001 From: d33bs Date: Tue, 2 Jan 2024 14:01:26 -0700 Subject: [PATCH 1/4] build(docker): add Dockerfile and tests This change adds a Dockerfile for pycytominer along with associated checks and tests to ensure the image is prepared for use on distribution platforms like Docker Hub. --- .github/workflows/integration-test.yml | 19 ++++++++++++++++ .pre-commit-config.yaml | 4 ++++ Makefile | 5 +++++ build/docker/Dockerfile | 31 ++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 build/docker/Dockerfile diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 9c7f2987..a24667fc 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -105,3 +105,22 @@ jobs: # For every other version and/or OS, run pytest without coverage if: (matrix.os != 'ubuntu-latest') || (matrix.python-version != env.TARGET_PYTHON_VERSION ) run: poetry run pytest ${{ github.event.inputs.pytest_addopts }} + docker-image-test: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + # note: roughly follows Docker documentation on GitHub Actions usage + # found on the following https://github.com/docker/build-push-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Try to build docker image + run: | + docker build -f build/docker/Dockerfile -t pycytominer:latest . + - name: Run tests through docker image + run: | + docker run pycytominer:latest poetry run pytest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 210a5c30..466ea54b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,3 +17,7 @@ repos: rev: v1.6.26 hooks: - id: actionlint + - repo: https://github.com/hadolint/hadolint + rev: v2.12.1-beta + hooks: + - id: hadolint-docker diff --git a/Makefile b/Makefile index b97de95f..b184c7bd 100644 --- a/Makefile +++ b/Makefile @@ -18,3 +18,8 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# note: presumes the existence of docker daemon to receive docker commands +test_docker_build: + docker build -f build/docker/Dockerfile -t pycytominer:latest . && \ + docker run pycytominer:latest poetry run pytest diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile new file mode 100644 index 00000000..318da9a0 --- /dev/null +++ b/build/docker/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11 + +# set various metadata, loosely following biocontainers standards from: +# https://biocontainers-edu.readthedocs.io/en/latest/what_is_biocontainers.html +LABEL base_image="python:3.11" +LABEL software="pycytominer" +LABEL about.summary="Python package for processing image-based profiling data" +LABEL about.home="https://github.com/cytomining/pycytominer" +LABEL about.documentation="https://pycytominer.readthedocs.io/en/stable/" +LABEL about.license_file="https://github.com/cytomining/pycytominer/blob/main/LICENSE.md" +LABEL about.license="SPDX:BSD-3-Clause" + +# set the workdir to /app +WORKDIR /app + +# copy pyproject and poetry lockfile for stepped installation +COPY pyproject.toml poetry.lock ./ + +# install poetry and poetry dynamic versioning +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir poetry poetry-dynamic-versioning + +# copy the rest of the repository +COPY . . + +# install pycytominer from poetry with group deps and all extras +RUN poetry install --with dev,docs --all-extras -v --no-interaction + +# set an alias for running python through the poetry env of pycytominer +# hadolint ignore=DL3059 +RUN echo 'alias python="poetry run python"' >> ~/.bashrc From a05e541461521da3e6beeebe3300cf4fd409b8c9 Mon Sep 17 00:00:00 2001 From: d33bs Date: Wed, 31 Jan 2024 13:56:45 -0700 Subject: [PATCH 2/4] exclude docs from dockerfile deps install Co-Authored-By: Ken Brewer --- build/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 318da9a0..a71dfa75 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -24,7 +24,7 @@ RUN pip install --no-cache-dir poetry poetry-dynamic-versioning COPY . . # install pycytominer from poetry with group deps and all extras -RUN poetry install --with dev,docs --all-extras -v --no-interaction +RUN poetry install --with dev --all-extras -v --no-interaction # set an alias for running python through the poetry env of pycytominer # hadolint ignore=DL3059 From ef4adb55d3fe3a96aeb34ff705cfacea42eed565 Mon Sep 17 00:00:00 2001 From: d33bs Date: Fri, 8 Mar 2024 11:27:20 -0700 Subject: [PATCH 3/4] build(docker): refine docker image and build tests --- .github/workflows/integration-test.yml | 40 ++++++++++++++++++++++++-- build/docker/Dockerfile | 26 ++++++++++------- pyproject.toml | 1 + 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 1b8a39d4..ba3eb86a 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -146,16 +146,50 @@ jobs: retention-days: 90 docker-image-test: runs-on: ubuntu-latest + # set a dependency sequence to occur after build job completion + needs: build steps: + # checks out selected files for docker image build testing + - name: Checkout selected files + uses: actions/checkout@v4 + with: + sparse-checkout: | + build/docker/Dockerfile + tests + sparse-checkout-cone-mode: false + - name: Download sdist artifact + uses: actions/download-artifact@v4 + with: + name: sdist + # gather the sdist tar.gz name (which varies) + - name: Get sdist tar.gz + id: get_sdist_tar_gz + run: | + echo "filename=$(ls ./*.tar.gz)" >> "$GITHUB_OUTPUT" + # unzips the sdist + - name: Extract sdist + run: | + tar -xzvf "${{ steps.get_sdist_tar_gz.outputs.filename }}" # note: roughly follows Docker documentation on GitHub Actions usage # found on the following https://github.com/docker/build-push-action - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Try to build docker image + # attempts to build a test image for observing test behavior + # and modeling a production image build. Version label + # assumes pwd of, for example: pycytominer-1.0.1.post37.dev0+55690e4, + # where we attempt to use 1.0.1.post37.dev0+55690e4 as a version label. + - name: Build docker image for testing run: | - docker build -f build/docker/Dockerfile -t pycytominer:latest . + cd "$(echo "${{ steps.get_sdist_tar_gz.outputs.filename }}" | sed 's/\.tar\.gz$//')" && \ + cp -r ../tests . && \ + docker build -f ../build/docker/Dockerfile \ + -t pycytominer:testing \ + --label version="$(basename "$(pwd)" | sed 's/pycytominer-//')" \ + --target testing \ + . + # runs pytest for pycytominer within a docker container based on the image above - name: Run tests through docker image run: | - docker run pycytominer:latest poetry run pytest + docker run pycytominer:testing pytest diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index a71dfa75..5240ef14 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11 +FROM python:3.11 as base # set various metadata, loosely following biocontainers standards from: # https://biocontainers-edu.readthedocs.io/en/latest/what_is_biocontainers.html @@ -13,19 +13,25 @@ LABEL about.license="SPDX:BSD-3-Clause" # set the workdir to /app WORKDIR /app -# copy pyproject and poetry lockfile for stepped installation -COPY pyproject.toml poetry.lock ./ +# configures poetry to install dependencies without virtualenvs +# see: https://python-poetry.org/docs/configuration/#virtualenvscreate +ENV POETRY_VIRTUALENVS_CREATE=false # install poetry and poetry dynamic versioning # hadolint ignore=DL3013 -RUN pip install --no-cache-dir poetry poetry-dynamic-versioning +RUN pip install --no-cache-dir poetry -# copy the rest of the repository +# create stage for production build (no dev or non-essential deps) +FROM base as production + +# copy the pycytominer source into image COPY . . -# install pycytominer from poetry with group deps and all extras -RUN poetry install --with dev --all-extras -v --no-interaction +# install pycytominer using poetry root dependencies and all extras +RUN poetry install --only-root --all-extras -v --no-interaction -# set an alias for running python through the poetry env of pycytominer -# hadolint ignore=DL3059 -RUN echo 'alias python="poetry run python"' >> ~/.bashrc +# create stage for production build (no dev or non-essential deps) +FROM production as testing + +# install pycytominer from poetry with group deps and all extras for testing +RUN poetry install --with dev --all-extras -v --no-interaction diff --git a/pyproject.toml b/pyproject.toml index d39c9cc1..2f6b466a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ license = "BSD-3-Clause" readme = "README.md" homepage = "https://pycytominer.readthedocs.io/" repository = "https://github.com/cytomining/pycytominer" +include = [{ path = "poetry.lock", format = "sdist" }] [tool.poetry.dependencies] python = ">=3.8,<4.0" From bc093ef34045aa50860f032a349c2f77c4462f2b Mon Sep 17 00:00:00 2001 From: Dave Bunten Date: Sat, 16 Mar 2024 14:14:56 -0600 Subject: [PATCH 4/4] build(docker): refine docker build tests Co-Authored-By: Ken Brewer --- .github/workflows/integration-test.yml | 14 +++++++------- build/docker/Dockerfile | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ba3eb86a..71873240 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -148,6 +148,10 @@ jobs: runs-on: ubuntu-latest # set a dependency sequence to occur after build job completion needs: build + env: + version: ${{ needs.build.outputs.version }} + sdist_filename: pycytominer-${{ needs.build.outputs.version}}.tar.gz + sdist_extracted_name: pycytominer-${{ needs.build.outputs.version}} steps: # checks out selected files for docker image build testing - name: Checkout selected files @@ -162,14 +166,10 @@ jobs: with: name: sdist # gather the sdist tar.gz name (which varies) - - name: Get sdist tar.gz - id: get_sdist_tar_gz - run: | - echo "filename=$(ls ./*.tar.gz)" >> "$GITHUB_OUTPUT" # unzips the sdist - name: Extract sdist run: | - tar -xzvf "${{ steps.get_sdist_tar_gz.outputs.filename }}" + tar -xzvf "${{ env.sdist_filename}}" # note: roughly follows Docker documentation on GitHub Actions usage # found on the following https://github.com/docker/build-push-action - name: Set up QEMU @@ -182,11 +182,11 @@ jobs: # where we attempt to use 1.0.1.post37.dev0+55690e4 as a version label. - name: Build docker image for testing run: | - cd "$(echo "${{ steps.get_sdist_tar_gz.outputs.filename }}" | sed 's/\.tar\.gz$//')" && \ + cd "${{ env.sdist_extracted_name }}" && \ cp -r ../tests . && \ docker build -f ../build/docker/Dockerfile \ -t pycytominer:testing \ - --label version="$(basename "$(pwd)" | sed 's/pycytominer-//')" \ + --label version="${{ env.version }}" \ --target testing \ . # runs pytest for pycytominer within a docker container based on the image above diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 5240ef14..9948094a 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -28,7 +28,7 @@ FROM base as production COPY . . # install pycytominer using poetry root dependencies and all extras -RUN poetry install --only-root --all-extras -v --no-interaction +RUN poetry install --all-extras -v --no-interaction # create stage for production build (no dev or non-essential deps) FROM production as testing