Dev/update (#188)

* Change the name of the `Dockerfile` to `train.Dockerfile` and place it in the `dockerfiles` directory as it is no longer that special.

* Updated $PYTHONPATH to incorporate any pre-existing values.

* Added PYTHONPATH to NGC conda installation to check if it reduces the number of downloads.

* Undo adding PYTHON PATH during conda installation as there is a worry that conda will uninstall existing packages.

* Add code to remove pre-installed `pip` packages that should use the versions installed via `conda` instead.

* Update `ruff` version.

* Add `uninstalls` to `.dockerignore`.

* Add `pip` package removal functionality to the NGC image.

---------

Co-authored-by: joonhyung.lee <[email protected]>

Author: veritas9872

.dockerignore

@@ -9,3 +9,4 @@
 !**/*environment*.yaml
 !*conda-lock*.yaml
 !**/*conda-lock*.yaml
+!**/*uninstalls*.txt

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
 
   # Ruff should be executed before other formatters.
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.8
+    rev: v0.1.13
     hooks:
       - id: ruff
         args: [--exit-non-zero-on-fix]

docker-compose.yaml

@@ -80,7 +80,7 @@ services:
 
     build:
       context: . # Nearly all files are ignored due to `.dockerignore` settings.
-      target: ${TARGET_STAGE:-train} # Specify Dockerfile target build stage.
+      target: ${TARGET_STAGE:-train} # Specify the `train.Dockerfile` target build stage.
       args: # Common build-time environment variables.
         # Even if these variables are unnecessary during the build,
         # they can be ignored simply by not defining them in that stage.
@@ -114,7 +114,7 @@ services:
       service: base
     build: # Options for building. Used when `--build` is called in `docker compose`.
       # Set `TARGET_STAGE` to `train-builds` to get just the wheels in `/tmp/dist`.
-      dockerfile: Dockerfile
+      dockerfile: dockerfiles/train.Dockerfile
       args: # Equivalent to `--build-arg`.
         BUILD_MODE: ${BUILD_MODE:-exclude}
 #        BUILD_TEST: 0
@@ -155,7 +155,7 @@ services:
       service: base
     build:
       target: ${TARGET_STAGE:-build-base} # All builds begin at `build-base`.
-      dockerfile: Dockerfile
+      dockerfile: dockerfiles/train.Dockerfile
 
   ngc: # Service based on images from the NGC PyTorch image catalog.
     # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/running.html

dockerfiles/ngc.Dockerfile

@@ -22,6 +22,7 @@ RUN git clone --depth 1 ${ZSHS_URL} /opt/zsh/zsh-syntax-highlighting
 # Copy `apt` and `conda` requirements for ngc images.
 COPY --link ../reqs/ngc-apt.requirements.txt /tmp/apt/requirements.txt
 COPY --link ../reqs/ngc-environment.yaml /tmp/env/environment.yaml
+COPY --link ../reqs/ngc-pip.uninstalls.txt /tmp/pip/uninstalls.txt
 
 ########################################################################
 FROM ${BASE_IMAGE} AS install-conda
@@ -110,6 +111,10 @@ RUN --mount=type=bind,from=stash,source=/tmp/apt,target=/tmp/apt \
     xargs -r apt-get install -y --no-install-recommends && \
     rm -rf /var/lib/apt/lists/*
 
+# Remove pre-installed `pip` packages that should use the versions installed via `conda` instead.
+RUN --mount=type=bind,from=stash,source=/tmp/pip,target=/tmp/pip \
+    python -m pip uninstall -y -r /tmp/pip/uninstalls.txt
+
 ########################################################################
 FROM train-base AS train-adduser-include
 
@@ -208,7 +213,8 @@ ENV PATH=${PATH}:/opt/conda/bin
 # Configure `PYTHONPATH` to prioritize system packages over `conda` packages to
 # prevent conflict when `conda` installs different versions of the same package.
 ARG PROJECT_ROOT=/opt/project
-ENV PYTHONPATH=${PROJECT_ROOT}:/usr/local/lib/python3/dist-packages
+ENV PYTHONPATH=${PYTHONPATH:+${PYTHONPATH}:}${PROJECT_ROOT}
+ENV PYTHONPATH=${PYTHONPATH}:/usr/local/lib/python3/dist-packages
 ENV PYTHONPATH=${PYTHONPATH}:/opt/conda/lib/python3/site-packages
 
 WORKDIR ${PROJECT_ROOT}

dockerfiles/simple.Dockerfile

@@ -265,6 +265,6 @@ RUN {   echo "fpath+=${PURE_PATH}"; \
 
 ENV PATH=/opt/conda/bin:${PATH}
 ARG PROJECT_ROOT=/opt/project
-ENV PYTHONPATH=${PROJECT_ROOT}
+ENV PYTHONPATH=${PROJECT_ROOT}${PYTHONPATH:+:${PYTHONPATH}}
 WORKDIR ${PROJECT_ROOT}
 CMD ["/bin/zsh"]

Dockerfile dockerfiles/train.DockerfileDockerfile renamed to dockerfiles/train.Dockerfile

@@ -103,11 +103,11 @@ RUN --mount=type=bind,from=curl-conda,source=/tmp/conda,target=/tmp/conda \
 
 ########################################################################
 FROM install-conda AS build-base
-# `build-base` is the base stage for all heavy builds in the Dockerfile.
+# `build-base` is the base stage for all heavy builds in the train.Dockerfile.
 
 # Get build requirements. Set package versions manually if compatibility issues arise.
 ARG BUILD_REQS=/tmp/conda/build-requirements.txt
-COPY --link reqs/train-conda-build.requirements.txt ${BUILD_REQS}
+COPY --link ../reqs/train-conda-build.requirements.txt ${BUILD_REQS}
 
 # Conda packages are preferable to system packages because they
 # are much more likely to be the latest (and the greatest!) packages.
@@ -359,7 +359,7 @@ FROM ${BUILD_IMAGE} AS train-stash
 
 # This stage prevents direct contact between the `train` stage and external files.
 # Other files such as `.deb` package files may also be stashed here.
-COPY --link reqs/train-apt.requirements.txt /tmp/apt/requirements.txt
+COPY --link ../reqs/train-apt.requirements.txt /tmp/apt/requirements.txt
 
 ########################################################################
 FROM ${BUILD_IMAGE} AS train-builds-include
@@ -422,7 +422,7 @@ ARG conda=/opt/conda/bin/${CONDA_MANAGER}
 ARG PIP_CACHE_DIR=/root/.cache/pip
 ARG CONDA_PKGS_DIRS=/opt/conda/pkgs
 ARG CONDA_ENV_FILE=/tmp/train/environment.yaml
-COPY --link reqs/train-environment.yaml ${CONDA_ENV_FILE}
+COPY --link ../reqs/train-environment.yaml ${CONDA_ENV_FILE}
 RUN --mount=type=cache,target=${PIP_CACHE_DIR},sharing=locked \
     --mount=type=cache,target=${CONDA_PKGS_DIRS},sharing=locked \
     find /tmp/dist -name '*.whl' | sed 's/^/      - /' >> ${CONDA_ENV_FILE} && \
@@ -566,6 +566,6 @@ RUN {   echo "fpath+=${PURE_PATH}"; \
 ENV PATH=/opt/conda/bin:${PATH}
 # `PROJECT_ROOT` is where the project code will reside.
 ARG PROJECT_ROOT=/opt/project
-ENV PYTHONPATH=${PROJECT_ROOT}
+ENV PYTHONPATH=${PROJECT_ROOT}${PYTHONPATH:+:${PYTHONPATH}}
 WORKDIR ${PROJECT_ROOT}
 CMD ["/bin/zsh"]

reqs/ngc-pip.uninstalls.txt

@@ -0,0 +1,2 @@
+# Remove pre-installed `pip` packages that should use the versions installed via `conda` instead.
+dummy  # Placeholder package to prevent errors durring the build.

reqs/train-environment.yaml

@@ -1,4 +1,4 @@
-# Packages to be installed in `Dockerfile` in `train` mode.
+# Packages to be installed in `train.Dockerfile` in `train` mode.
 # PyTorch install variables are in the Docker Compose file.
 # PyTorch and related libraries are either compiled or installed via pip
 # to reduce dependency issues with conda and for greater flexibility.

tests/test_run.py

@@ -19,7 +19,7 @@
 The hit to performance from antivirus programs is nontrivial.
 
 Please note that a clean installation of PyTorch on the same image
-as provided in the `Dockerfile` will probably not give any speedup.
+as provided in the `train.Dockerfile` will probably not give any speedup.
 Use your environment as you were using it for a fair comparison.
 """
 import logging