From d61d7389545a5d3732787bdde4445aa399899d09 Mon Sep 17 00:00:00 2001 From: Maksym Sobolyev Date: Wed, 17 Jul 2024 16:08:16 -0700 Subject: [PATCH] Add simple pipeline to install dependencies and preload models. --- .github/workflows/main.yml | 131 +++++++++++++++++++++++++++++++++ docker/Dockerfile | 27 +++++++ docker/install_conda.sh | 15 ++++ docker/install_requirements.sh | 19 +++++ docker/preload_it_de.yaml | 7 ++ docker/preload_models.sh | 10 +++ 6 files changed, 209 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 docker/Dockerfile create mode 100755 docker/install_conda.sh create mode 100755 docker/install_requirements.sh create mode 100644 docker/preload_it_de.yaml create mode 100755 docker/preload_models.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..2388450 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,131 @@ +# This is a basic workflow to help you get started with Actions + +name: Build, Test & Publush + +# Controls when the action will run. +on: + # Triggers the workflow on all push or pull request events + push: + pull_request: + + release: + types: [created] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + + schedule: + - cron: "0 0 * * *" + +# added using https://github.com/step-security/secure-repo +permissions: + contents: read + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + build_and_test_python: + if: false + strategy: + matrix: + python-version: ['3.11', '3.12'] + os: ['24.10'] + + runs-on: ubuntu-latest + container: + image: ubuntu:${{ matrix.os }} + env: + PYTHON_CMD: "python${{ matrix.python-version }}" + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: Install git + run: | + apt-get update + apt-get install --no-install-recommends -y git lsb-release ca-certificates + + - uses: actions/checkout@v4 + + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: ${{ matrix.python-version }} + activate-environment: Infernos + + - name: Conda init + run: conda init + + - name: Conda list + run: | + . ~/miniconda3/etc/profile.d/conda.sh + conda activate Infernos + conda list + + - name: Install dependencies + run: | + apt-get install --no-install-recommends -y gcc g++ libc6-dev cmake pkg-config make + . ~/miniconda3/etc/profile.d/conda.sh + conda activate Infernos + conda install -y pip + ${PYTHON_CMD} -m pip install -U -r requirements.txt + + - name: Preload models + run: | + . ~/miniconda3/etc/profile.d/conda.sh + conda activate Infernos + rm -rf ~/.cache + mkdir ~/.cache + python Infernos.py -f docker/preload_it_de.yaml + + Docker: + name: Build&Push to DockerHub + if: (github.event_name == 'push' || github.event_name == 'pull_request') + runs-on: ubuntu-latest + env: + DOCKER_REPO: 'sippylabs/infernos' + BASE_IMAGE: 'ubuntu:24.10' + PYTHON_VER: '3.11' + CONDA_MAINENV': 'Infernos' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_REPO }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=latest,enable={{is_default_branch}} + type=sha + + - name: Build Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/Dockerfile + push: true + build-args: | + BASE_IMAGE=${{ env.BASE_IMAGE }} + PYTHON_VER=${{ env.PYTHON_VER }} + CONDA_MAINENV=${{ env.CONDA_MAINENV }} + tags: | + ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..0dd9ee3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,27 @@ +ARG BASE_IMAGE=ubuntu:24.10 +ARG PYTHON_VER +ARG CONDA_MAINENV +FROM $BASE_IMAGE AS build +LABEL maintainer="Maksym Sobolyev " + +USER root + +# Set Environment Variables +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHON_VER=${PYTHON_VER} +ENV CONDA_MAINENV=${CONDA_MAINENV} + +WORKDIR /docker + +# Build & install everything +RUN apt-get update +RUN apt-get install --no-install-recommends -y git lsb-release ca-certificates && apt-get clean +COPY docker/install_conda.sh /docker +RUN /docker/install_conda.sh +COPY docker/install_requirements.sh /docker +COPY requirements.txt /docker +RUN cd /docker && ./install_requirements.sh + +WORKDIR /Infernos +COPY . /Infernos +RUN /Infernos/docker/preload_models.sh diff --git a/docker/install_conda.sh b/docker/install_conda.sh new file mode 100755 index 0000000..8a2945b --- /dev/null +++ b/docker/install_conda.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e +set -x + +apt-get install --no-install-recommends -y curl gpg +curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > /tmp/conda.gpg +install -o root -g root -m 644 /tmp/conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg +rm /tmp/conda.gpg + +echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list + +apt update +apt-get install --no-install-recommends -y conda +apt-get clean diff --git a/docker/install_requirements.sh b/docker/install_requirements.sh new file mode 100755 index 0000000..09bfbcf --- /dev/null +++ b/docker/install_requirements.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -e +set -x + +PYTHON_CMD="python${PYTHON_VER}" +DEV_PKGS="gcc g++ libc6-dev cmake pkg-config make" + +apt-get install --no-install-recommends -y ${DEV_PKGS} +. /opt/conda/etc/profile.d/conda.sh +conda create -y --name "${CONDA_MAINENV}" python=${PYTHON_VER} +conda activate "${CONDA_MAINENV}" +conda install -y pip +${PYTHON_CMD} -m pip install -U -r requirements.txt +apt-get remove -y ${DEV_PKGS} +apt-get autoremove -y +apt-get clean +rm -r ~/.cache +mkdir ~/.cache diff --git a/docker/preload_it_de.yaml b/docker/preload_it_de.yaml new file mode 100644 index 0000000..e1dac51 --- /dev/null +++ b/docker/preload_it_de.yaml @@ -0,0 +1,7 @@ +apps: + live_translator: + profiles: + configuration1: + stt_langs: ['de', 'it'] + tts_langs: ['it', 'de'] + live_translator_precache: true diff --git a/docker/preload_models.sh b/docker/preload_models.sh new file mode 100755 index 0000000..40d33b0 --- /dev/null +++ b/docker/preload_models.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e +set -x + +PYTHON_CMD="python${PYTHON_VER}" + +. /opt/conda/etc/profile.d/conda.sh +conda activate "${CONDA_MAINENV}" +${PYTHON_CMD} /Infernos/Infernos.py -f /Infernos/docker/preload_it_de.yaml