Rui/use system deps (#80)

--------- Signed-off-by: rmdg88 <[email protected]> Signed-off-by: Michele Dolfi <[email protected]> Co-authored-by: Michele Dolfi <[email protected]>
DS4SD · Oct 9, 2024 · 8e93d57 · 8e93d57
1 parent 73ac4b6
commit 8e93d57
Show file tree

Hide file tree

Showing 22 changed files with 650 additions and 433 deletions.
diff --git a/.github/actions/setup-poetry/action.yml b/.github/actions/setup-poetry/action.yml
@@ -10,10 +10,17 @@ runs:
     - name: Install poetry
       run: pipx install poetry==1.8.3
       shell: bash
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
+      id: py
       with:
         python-version: ${{ inputs.python-version }}
+        update-environment: false
         cache: 'poetry'
+    - name: Setup poetry env with correct python
+      run: |
+        poetry env use ${{ steps.py.outputs.python-path }}
+        poetry run python --version
+      shell: bash
     - name: Install only dependencies and not the package itself
       run: poetry install --all-extras --no-root
       shell: bash
diff --git a/.github/scripts/build_rhel.sh b/.github/scripts/build_rhel.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -e  # trigger failure on error - do not remove!
+set -x  # display command on output
+
+# Build the Python package with Poetry
+poetry build -f sdist
+
+USE_SYSTEM_DEPS="ON"
+
+docker build --progress=plain \
+             --build-arg USE_SYSTEM_DEPS="$USE_SYSTEM_DEPS" \
+             -f - . <<EOF
+    # syntax=docker/dockerfile:1
+
+    FROM quay.io/centos/centos:stream9
+
+    RUN dnf config-manager --set-enabled crb
+    
+    RUN dnf copr -y enable cheimes/deepsearch-glm rhel-9-x86_64
+    
+    RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \
+        && dnf clean all
+
+    RUN dnf install -y --nodocs \
+            autoconf automake binutils cmake gcc gcc-c++ git glibc-devel glibc-headers glibc-static kernel-devel libtool libstdc++-devel make ninja-build pkgconfig zlib-devel \
+            python3.11 python3.11-pip python3.11-devel \
+            cxxopts-devel fasttext-devel fmt-devel json-schema-validator-devel pcre2-devel sentencepiece-devel utf8cpp-devel \
+        && dnf clean all
+
+    # RUN dnf install -y --nodocs loguru-devel
+
+    # TEMPORARY loguru install method
+    # https://koji.fedoraproject.org/koji/buildinfo?buildID=2563067
+    RUN curl -O https://kojipkgs.fedoraproject.org//packages/loguru/2.2.0%5E20230406git4adaa18/5.el9/x86_64/loguru-2.2.0%5E20230406git4adaa18-5.el9.x86_64.rpm
+    RUN dnf install -y loguru-2.2.0%5E20230406git4adaa18-5.el9.x86_64.rpm
+    RUN curl -O https://kojipkgs.fedoraproject.org//packages/loguru/2.2.0%5E20230406git4adaa18/5.el9/x86_64/loguru-devel-2.2.0%5E20230406git4adaa18-5.el9.x86_64.rpm
+    RUN dnf install -y loguru-devel-2.2.0%5E20230406git4adaa18-5.el9.x86_64.rpm
+
+    RUN mkdir /src
+
+    COPY ./dist/*.tar.gz /src/
+
+    ARG USE_SYSTEM_DEPS
+
+    RUN USE_SYSTEM_DEPS=\$USE_SYSTEM_DEPS pip3.11 install /src/deepsearch_glm*.tar.gz
+
+    COPY ./tests /src/tests
+    
+    RUN cd /src \
+        && pip3.11 install pytest \
+        && pytest ./tests/test_glm.py -v
+EOF
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -19,7 +19,10 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install with poetry
-        run: poetry install --all-extras
+        run: |
+          poetry install --all-extras
+          ls -l
+          ls -l deepsearch_glm
       - name: Run styling check
         run: poetry run pre-commit run --all-files
       - name: Testing

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,4 +21,5 @@ jobs:
     uses: ./.github/workflows/checks.yml
   build-wheels:
     uses: ./.github/workflows/wheels.yml
-
+  rhel-build:
+    uses: ./.github/workflows/rhel.yml
diff --git a/.github/workflows/rhel.yml b/.github/workflows/rhel.yml
@@ -0,0 +1,12 @@
+on:
+  workflow_call:
+
+jobs:
+  run-checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-poetry
+      - name: Run build in docker
+        run: ./.github/scripts/build_rhel.sh
+        shell: bash
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -51,11 +51,21 @@ jobs:
 
       - name: Setup Python
         uses: actions/setup-python@v5
+        id: py
         with:
           python-version: ${{ matrix.python-version }}
+          update-environment: false
 
       - name: Install Poetry
-        run: python -m pip install poetry==1.8.3
+        run: |
+          which python
+          python --version
+          which python3
+          python3 --version
+          echo "pythonpath: ${{ steps.py.outputs.python-path }}"
+          ${{ steps.py.outputs.python-path }} --version
+          pipx install poetry==1.8.3
+          poetry env use ${{ steps.py.outputs.python-path }}
 
       - name: Set up custom PATH and set py version to cpXYZ [windows]
         if: ${{matrix.os.platform_id == 'win_amd64'}}
@@ -97,9 +107,15 @@ jobs:
           CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0"
           ARCHFLAGS: -arch x86_64
           BUILD_THREADS: "4"
+          PYTORCH_MPS_HIGH_WATERMARK_RATIO: "0.0"
         run: |
-          echo "Building wheel ${CIBW_BUILD}"
+          PY_CACHE_TAG=$(poetry run python -c 'import sys;print(sys.implementation.cache_tag)')
+          echo "Building wheel ${CIBW_BUILD} ${{ env.CIBW_BUILD }}"
+          echo "Building cp: ${{ env.python_cp_version }}"
+          echo "Building cache_tag: ${PY_CACHE_TAG}"
+          echo "Building platform_id: ${{ matrix.os.platform_id }}"
           poetry run python --version
+          poetry run python --version | grep ${{ matrix.python-version }} 
           poetry install --no-root --only=build
           cat ./pyproject.toml
           poetry run python -m cibuildwheel --output-dir wheelhouse
@@ -115,8 +131,12 @@ jobs:
           for file in ./wheelhouse/*.whl; do
             echo "Inspecting $file"
             poetry run python -m zipfile --list "$file"
+            echo "Checking if .so is contained in the wheel"
+            poetry run python -m zipfile --list "$file" | grep \\.so
+            echo "Checking if the correct python version is contained in the wheel"
+            poetry run python -m zipfile --list "$file" | grep ${PY_CACHE_TAG}
           done
-          mkdir ./dist
+          mkdir -p ./dist
           cp wheelhouse/*.whl ./dist/
 
       # there is an error with the tagging of wheels for macosx-arm64
@@ -137,9 +157,16 @@ jobs:
           CIBW_ENVIRONMENT: "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.macos_version }}.0"
           ARCHFLAGS: -arch arm64
           BUILD_THREADS: "4"
+          PYTORCH_MPS_HIGH_WATERMARK_RATIO: "0.0"
+          CUDA_VISIBLE_DEVICES: "cpu"
         run: |
-          echo "Building wheel ${CIBW_BUILD}"
+          PY_CACHE_TAG=$(poetry run python -c 'import sys;print(sys.implementation.cache_tag)')
+          echo "Building wheel ${CIBW_BUILD} ${{ env.CIBW_BUILD }}"
+          echo "Building cp: ${{ env.python_cp_version }}"
+          echo "Building cache_tag: ${PY_CACHE_TAG}"
+          echo "Building platform_id: ${{ matrix.os.platform_id }}"
           poetry run python --version
+          poetry run python --version | grep ${{ matrix.python-version }} 
           poetry install --no-root --only=build
           cat ./pyproject.toml
           poetry run python -m cibuildwheel --output-dir wheelhouse
@@ -155,8 +182,12 @@ jobs:
           for file in ./wheelhouse/*.whl; do
             echo "Inspecting $file"
             poetry run python -m zipfile --list "$file"
+            echo "Checking if .so is contained in the wheel"
+            poetry run python -m zipfile --list "$file" | grep \\.so
+            echo "Checking if the correct python version is contained in the wheel"
+            poetry run python -m zipfile --list "$file" | grep ${PY_CACHE_TAG}
           done
-          mkdir ./dist
+          mkdir -p ./dist
           cp wheelhouse/*.whl ./dist/
 
       - name: Set up QEMU [linux]
@@ -165,31 +196,53 @@ jobs:
         with:
             platforms: all
 
+      - name: Build sdist
+        # build only on Linux to avoid too many duplicates of the sdist
+        if: matrix.os.name == 'ubuntu-latest' 
+        run: |
+          echo "Building wheel ${CIBW_BUILD}"
+          poetry build -f sdist
+
       - name: Build wheels [linux]
         if: matrix.os.name == 'ubuntu-latest'
         env:
-          # CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }}
+          ## CIBW_BUILD: ${{ env.python_cp_version }}-${{ matrix.os.platform_id }}
           CIBW_ARCHS: auto x86_64 aarch64
           CIBW_PLATFORM: linux
           CIBW_SKIP: "pp* *musllinux_* *_i686* *_s390* *pypy*"
           CIBW_PROJECT_REQUIRES_PYTHON: "~=${{ matrix.python-version }}.0"
           CIBW_BUILD_VERBOSITY: 3
           BUILD_THREADS: "8"
         run: |
-          echo "Building wheel ${CIBW_BUILD}"
+          PY_CACHE_TAG=$(poetry run python -c 'import sys;print(sys.implementation.cache_tag)')
+          echo "Building cp: ${{ env.python_cp_version }}"
+          echo "Building cache_tag: ${PY_CACHE_TAG}"
+          echo "Building platform_id: ${{ matrix.os.platform_id }}"
           poetry run python --version
+          poetry run python --version | grep ${{ matrix.python-version }} 
           poetry install --no-root --only=build
           cat ./pyproject.toml
           poetry run python -m cibuildwheel --output-dir ./wheelhouse
           ls -l ./wheelhouse
           for file in ./wheelhouse/*.whl; do
             echo "Inspecting $file"
             poetry run python -m zipfile --list "$file"
+            echo "Checking if .so is contained in the wheel"
+            poetry run python -m zipfile --list "$file" | grep \\.so
+            echo "Checking if the correct python version is contained in the wheel"
+            poetry run python -m zipfile --list "$file" | grep ${PY_CACHE_TAG}
           done
-          mkdir ./dist
+          mkdir -p ./dist
           cp wheelhouse/*.whl ./dist/
 
-      - name: Set up MSYS2
+      - name: Test install wheel [macos-arm64]
+        if: matrix.os.platform_id == 'macosx_arm64' && matrix.os.name == 'macos-14'
+        shell: bash
+        run: |
+          wheel_file=$(ls dist/*.whl | head -n 1)
+          poetry run pip install "$wheel_file"
+
+      - name: Set up MSYS2 [windows]
         if: ${{matrix.os.platform_id == 'win_amd64'}}
         uses: msys2/setup-msys2@v2
         with:
@@ -206,7 +259,7 @@ jobs:
           Copy-Item -Path "C:/mingw64/bin/libstdc++-6.dll" -Destination ".\deepsearch_glm"
           Copy-Item -Path "C:/mingw64/bin/libwinpthread-1.dll" -Destination ".\deepsearch_glm"
 
-      - name: Build wheels
+      - name: Build wheels [windows]
         if: ${{matrix.os.platform_id == 'win_amd64'}}
         env:
           CIBW_PLATFORM: windows
@@ -235,12 +288,27 @@ jobs:
               New-Item -Path .\dist -ItemType Directory
           }
           Copy-Item -Path .\wheelhouse\*.whl -Destination .\dist\
+    
+      - name: Test install wheel [windows]
+        if: ${{matrix.os.platform_id == 'win_amd64'}}
+        shell: pwsh
+        run: |
+          $wheel_file = Get-ChildItem dist\*.whl | Select-Object -First 1
+          Write-Host "Installing wheel: $wheel_file"
+          poetry run pip install "$wheel_file"
 
-      - name: publish wheels (dry run)
+      - name: Quick test wheel
+        if: ${{ matrix.os.platform_id == 'win_amd64' || (matrix.os.platform_id == 'macosx_arm64' && matrix.os.name == 'macos-14') }}
+        run: |
+          poetry install --no-interaction --no-root --only=test
+          poetry run python -c 'from deepsearch_glm import andromeda_glm'
+          poetry run pytest ./tests/test_glm.py -v
+          
+      - name: Publish wheels (dry run)
         run: |
-          poetry publish --dry-run --no-interaction -vvv
+          poetry publish --skip-existing --dry-run --no-interaction -vvv
 
-      - name: publish wheels (on publishing) [for releases only]
+      - name: Publish wheels (on publishing) [for releases only]
         if: ${{ startsWith(github.ref, 'refs/tags/') }}
         run: |
-          poetry publish --no-interaction -vvv
+          poetry publish --skip-existing --no-interaction -vvv