feat: support for Windows (#79)

--------- Signed-off-by: rmdg88 <[email protected]>
DS4SD · Oct 2, 2024 · 5af213a · 5af213a
1 parent 7e8530e
commit 5af213a
Show file tree

Hide file tree

Showing 25 changed files with 445 additions and 301 deletions.
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -41,6 +41,10 @@ jobs:
             macos_version: "14"
             platform_id: "macosx_arm64"
 
+          - name: "windows-latest"
+            platform: "windows"
+            platform_id: "win_amd64"
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -49,27 +53,35 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-
-      - name: Install poetry
-        run: pipx install poetry==1.8.3 --python $(which python3)
-        shell: bash
 
-      - name: Test poetry
-        run: |
-          poetry run python --version
+      - name: Install Poetry
+        run: python -m pip install poetry==1.8.3
 
-      - name: Setup pypi for poetry [for releases only]
-        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+      - name: Set up custom PATH and set py version to cpXYZ [windows]
+        if: ${{matrix.os.platform_id == 'win_amd64'}}
+        shell: pwsh
         run: |
-          poetry config keyring.enabled false
-          poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}
+          echo "C:\msys64\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "C:\msys64\mingw64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "C:\Users\runneradmin\.local\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "C:\nasm" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          $version = "${{ matrix.python-version }}"
+          $cp_version = "cp$($version -replace '\.', '')"
+          Add-Content -Path $env:GITHUB_ENV -Value "python_cp_version=$cp_version"
 
       - name: Convert python version to cpXYZ
+        if: ${{matrix.os.platform_id != 'win_amd64'}}
         run: |
           version=${{ matrix.python-version }}
           cp_version="cp${version//.}"
           echo "python_cp_version=$cp_version" >> $GITHUB_ENV
 
+      - name: Setup pypi for poetry [for releases only]
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
+        run: |
+          poetry config keyring.enabled false
+          poetry config pypi-token.pypi ${{ secrets.PYPI_TOKEN }}
+          
       - name: Build wheels [macos-x86_64]
         if: ${{matrix.os.platform_id == 'macosx_x86_64'}}
         env:
@@ -147,20 +159,6 @@ jobs:
           mkdir ./dist
           cp wheelhouse/*.whl ./dist/
 
-      - name: publish wheels (dry run) [macos]
-        if: matrix.os.platform == 'macos'
-        run: |
-          ls -l ./
-          ls -l ./dist
-          poetry publish --dry-run --no-interaction -vvv
-
-      - name: publish wheels (on publishing) [macos]
-        if: ${{ matrix.os.platform == 'macos' && startsWith(github.ref, 'refs/tags/') }}
-        run: |
-          ls -l ./
-          ls -l ./dist
-          poetry publish --no-interaction -vvv
-
       - name: Set up QEMU [linux]
         if: matrix.os.name == 'ubuntu-latest'
         uses: docker/setup-qemu-action@v3
@@ -191,17 +189,58 @@ jobs:
           mkdir ./dist
           cp wheelhouse/*.whl ./dist/
 
-      - name: publish wheels (dry run) [linux]
-        if: matrix.os.platform == 'linux'
+      - name: Set up MSYS2
+        if: ${{matrix.os.platform_id == 'win_amd64'}}
+        uses: msys2/setup-msys2@v2
+        with:
+          update: true
+          install: >
+            mingw-w64-x86_64-toolchain
+            mingw-w64-x86_64-gcc-libs
+
+      - name: Set up external-libs [windows]
+        if: ${{matrix.os.platform_id == 'win_amd64'}}
+        shell: pwsh
+        run: |
+          Copy-Item -Path "C:/mingw64/bin/libgcc_s_seh-1.dll" -Destination ".\deepsearch_glm"
+          Copy-Item -Path "C:/mingw64/bin/libstdc++-6.dll" -Destination ".\deepsearch_glm"
+          Copy-Item -Path "C:/mingw64/bin/libwinpthread-1.dll" -Destination ".\deepsearch_glm"
+
+      - name: Build wheels
+        if: ${{matrix.os.platform_id == 'win_amd64'}}
+        env:
+          CIBW_PLATFORM: windows
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_ARCHS: AMD64
+          CIBW_PROJECT_REQUIRES_PYTHON: "~=${{ matrix.python-version }}.0"
+          PKG_CONFIG_PATH: "C:/msys64/usr/lib/pkgconfig"
+          PKG_CONFIG_EXECUTABLE: "C:/mingw64/bin/pkg-config.exe"
+          CMAKE_PREFIX_PATH: "C:/msys64/mingw64;C:/mingw64"
+          CMAKE_LIBRARY_PATH: "C:/msys64/mingw64/lib;C:/mingw64/lib"
+          CMAKE_INCLUDE_PATH: "C:/msys64/mingw64/include;C:/mingw64/include"
+          CMAKE_GENERATOR: "MSYS Makefiles"
+          BUILD_THREADS: 1
+        shell: pwsh
+        run: |
+          Remove-Item -Recurse -Force "C:\Strawberry\"
+          poetry install --no-interaction --no-root --only=build
+          poetry run python -m cibuildwheel --output-dir .\wheelhouse
+          Get-ChildItem -Path .\wheelhouse -Filter *.whl | Format-List
+          Get-ChildItem -Path .\wheelhouse -Filter *.whl | ForEach-Object {
+              $file = $_.FullName
+              Write-Output "Inspecting $file"
+              poetry run python -m zipfile --list "$file"
+          }
+          if (-not (Test-Path -Path .\dist)) {
+              New-Item -Path .\dist -ItemType Directory
+          }
+          Copy-Item -Path .\wheelhouse\*.whl -Destination .\dist\
+
+      - name: publish wheels (dry run)
         run: |
-          ls -l ./
-          ls -l ./dist
           poetry publish --dry-run --no-interaction -vvv
 
-      - name: publish wheels (on publishing) [linux]
-        if: ${{ matrix.os.platform == 'linux' && startsWith(github.ref, 'refs/tags/') }}
+      - name: publish wheels (on publishing) [for releases only]
+        if: ${{ startsWith(github.ref, 'refs/tags/') }}
         run: |
-          ls -l ./
-          ls -l ./dist
           poetry publish --no-interaction -vvv
-
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -73,8 +73,17 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -O0 -DROOT_PATH='\"${TOPLEVEL_PREFIX_PATH}\"' ")
 #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -g3 -DROOT_PATH='\"${TOPLEVEL_PREFIX_PATH}\"' -arch arm64")
 
-set(CMAKE_LIB_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -O3 -DROOT_PATH='\"${TOPLEVEL_PREFIX_PATH}\"' ${ENV_ARCHFLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -O3 -DROOT_PATH='\"${TOPLEVEL_PREFIX_PATH}\"' ${ENV_ARCHFLAGS}")
+#set(CMAKE_LIB_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -O3 -DROOT_PATH='\"${TOPLEVEL_PREFIX_PATH}\"' ${ENV_ARCHFLAGS}")
+#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -O3 -DROOT_PATH='\"${TOPLEVEL_PREFIX_PATH}\"' ${ENV_ARCHFLAGS}")
+
+if (WIN32)
+    add_definitions(-DPCRE2_STATIC)
+    add_definitions(-DNOMINMAX)
+    add_definitions(-DROOT_PATH="\\\"${TOPLEVEL_PREFIX_PATH}\\\"")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -O3 ${ENV_ARCHFLAGS}")
+else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-sign-compare -O3 -DROOT_PATH='\"${TOPLEVEL_PREFIX_PATH}\"' ${ENV_ARCHFLAGS}")
+endif()
 
 message(STATUS "cxx-compiler: " ${CMAKE_CXX_COMPILER})
 message(STATUS "cxx-flags   : " ${CMAKE_CXX_FLAGS})
@@ -163,7 +172,7 @@ include(FetchContent)
 FetchContent_Declare(pybind11
   FETCHCONTENT_BASE_DIR extlib_pybind11 
   GIT_REPOSITORY https://github.com/pybind/pybind11.git
-  GIT_TAG        v2.10.0
+  GIT_TAG        v2.13.5
   GIT_SHALLOW    TRUE
 )
 FetchContent_MakeAvailable(pybind11)

diff --git a/cmake/extlib_fasttext.cmake b/cmake/extlib_fasttext.cmake
@@ -5,34 +5,25 @@ message(STATUS "entering in extlib_fasttext.cmake")
 include(ExternalProject)
 include(CMakeParseArguments)
 
-#set(FASTTEXT_URL https://github.com/facebookresearch/fastText.git)
 set(FASTTEXT_URL https://github.com/PeterStaar-IBM/fastText.git)
-#set(FASTTEXT_TAG v0.9.2)
+set(FASTTEXT_TAG 9d5b2a2b364f49ed2707ff3be48a0f1ba6d86022)
 
 set(CXX_FLAGS "${ENV_ARCHFLAGS} -O3")
 
 ExternalProject_Add(extlib_fasttext
     PREFIX extlib_fasttext
 
     GIT_REPOSITORY ${FASTTEXT_URL}
-    #GIT_TAG ${FASTTEXT_TAG}
+    GIT_TAG ${FASTTEXT_TAG}
 
     UPDATE_COMMAND ""
 
-    #CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH}
-    #CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare -g3 -Wno-dev"
-
     CMAKE_ARGS \\
-    -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\
-    -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS}
-
-    #-DCMAKE_CXX_FLAGS=${ENV_ARCHFLAGS}
-    #-DCMAKE_CXX_FLAGS=-O3
-
-    #CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} -DCMAKE_BUILD_TYPE=RELEASE #-DCMAKE_CXX_FLAGS=-g3
-    #CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -Wno-dev"
-
-    #BUILD_ALWAYS ON
+        -DCMAKE_INSTALL_PREFIX=${EXTERNALS_PREFIX_PATH} \\
+        -DCMAKE_CXX_FLAGS=${CMAKE_LIB_FLAGS} \\
+        -DCMAKE_INSTALL_LIBDIR=${EXTERNALS_PREFIX_PATH}/lib \\
+        -DCMAKE_INSTALL_BINDIR=${EXTERNALS_PREFIX_PATH}/bin \\
+        -DCMAKE_INSTALL_INCLUDEDIR=${EXTERNALS_PREFIX_PATH}/include
 
     INSTALL_DIR ${EXTERNALS_PREFIX_PATH}
 
@@ -41,6 +32,5 @@ ExternalProject_Add(extlib_fasttext
     LOG_BUILD ON)
 
 add_library(fasttext STATIC IMPORTED)
-#set_target_properties(fasttext PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libfasttext.a)
 set_target_properties(fasttext PROPERTIES IMPORTED_LOCATION ${EXTERNALS_PREFIX_PATH}/lib/libfasttext_pic.a)
 add_dependencies(fasttext extlib_fasttext)
diff --git a/cmake/extlib_loguru.cmake b/cmake/extlib_loguru.cmake
@@ -9,13 +9,13 @@ set(LOGURU_INCLUDE_DIR ${EXTERNALS_PREFIX_PATH}/include/loguru)
 execute_process(COMMAND mkdir -p ${LOGURU_INCLUDE_DIR})
 
 set(LOGURU_URL https://github.com/emilk/loguru)
-#set(LOGURU_TAG )
+set(LOGURU_TAG v2.1.0)
 
 ExternalProject_Add(extlib_loguru
     PREFIX extlib_loguru
 
     GIT_REPOSITORY ${LOGURU_URL}
-    #GIT_TAG ${LOGURU_TAG}
+    GIT_TAG ${LOGURU_TAG}
 
     UPDATE_COMMAND ""
     CONFIGURE_COMMAND ""

diff --git a/cmake/extlib_pcre2.cmake b/cmake/extlib_pcre2.cmake
@@ -6,7 +6,7 @@ include(ExternalProject)
 include(CMakeParseArguments)
 
 set(PCRE2_URL https://github.com/PCRE2Project/pcre2.git)
-#set(PCRE2_TAG pcre2-10.40)
+#set(PCRE2_TAG pcre2-10.44)
 
 ExternalProject_Add(extlib_pcre2
     PREFIX extlib_pcre2
@@ -24,6 +24,7 @@ ExternalProject_Add(extlib_pcre2
     -DBUILD_STATIC_LIBS=ON \\
     -DPCRE2_STATIC_PIC=ON \\
     -DPCRE2_SHOW_REPORT=OFF
+    #-DPCRE2_BUILD_PCRE2GREP=OFF
 
     #-DCMAKE_CXX_FLAGS="-O3 ${ENV_ARCHFLAGS}" \\
 

diff --git a/cmake/extlib_pybind11.cmake b/cmake/extlib_pybind11.cmake
@@ -6,7 +6,7 @@ include(ExternalProject)
 include(CMakeParseArguments)
 
 set(PYBIND11_URL https://github.com/pybind/pybind11.git)
-set(PYBIND11_TAG v2.10.0)
+set(PYBIND11_TAG v2.13.5)
 
 ExternalProject_Add(extlib_pybind11
     PREFIX extlib_pybind11

diff --git a/cmake/extlib_sentencepiece.cmake b/cmake/extlib_sentencepiece.cmake
@@ -6,7 +6,7 @@ include(ExternalProject)
 include(CMakeParseArguments)
 
 set(SENTENCEPIECE_URL https://github.com/google/sentencepiece.git)
-set(SENTENCEPIECE_TAG v0.1.99)
+set(SENTENCEPIECE_TAG v0.2.0)
 
 message(STATUS "extlib_sentencepiece cxx-flags: " ${CMAKE_CXX_FLAGS})
 

diff --git a/cmake/os_opts.cmake b/cmake/os_opts.cmake
@@ -4,7 +4,9 @@ message(STATUS "entering in 'os_opts.cmake'")
 
 if(WIN32)
    message(STATUS "compiling on windows")
-
+
+   set(LIB_LINK fmt json loguru cxxopts pcre2 sentencepiece sentencepiece_train fasttext utf8 z ws2_32 user32)
+
 elseif(APPLE)
    message(STATUS "compiling on mac-osx")
 
@@ -34,5 +36,3 @@ elseif(UNIX)
    list(APPEND LIB_LINK dl m pthread rt resolv)
 
 endif()
-
-
diff --git a/deepsearch_glm/utils/load_pretrained_models.py b/deepsearch_glm/utils/load_pretrained_models.py
@@ -5,6 +5,7 @@
 import json
 import os
 import subprocess
+from urllib.parse import urljoin
 
 
 def get_resources_dir():
@@ -85,45 +86,41 @@ def load_pretrained_nlp_models(force: bool = False, verbose: bool = False):
     """Function to load pretrained NLP models"""
 
     resources_dir = get_resources_dir()
-
     with open(f"{resources_dir}/models.json", "r", encoding="utf-8") as fr:
         models = json.load(fr)
 
     cos_url = models["object-store"]
     cos_prfx = models["nlp"]["prefix"]
-    cos_path = os.path.join(cos_url, cos_prfx)
 
     cmds = {}
     for name, files in models["nlp"]["trained-models"].items():
-        source = os.path.join(cos_path, files[0])
+        source = urljoin(cos_url, f"{cos_prfx}/{files[0]}")
         target = os.path.join(resources_dir, files[1])
 
         cmd = ["curl", source, "-o", target, "-s"]
         cmds[name] = cmd
 
-    models = []
+    downloaded_models = []
 
     for name, cmd in cmds.items():
         model_weights = cmd[3]
 
-        if force or (not os.path.exists(model_weights)):
+        if force or not os.path.exists(model_weights):
             if verbose:
-                # print(f"downloading {os.path.basename(model_weights)} ... ", end="")
-                print(f"downloading {name} ... ", end="")
+                print(f"Downloading {name} ... ", end="")
 
-            message = subprocess.run(cmd, check=True)
+            subprocess.run(cmd, check=True)
 
             if verbose:
                 print("done!")
-            models.append(name)
+            downloaded_models.append(name)
 
         elif os.path.exists(model_weights):
             if verbose:
-                # print(f" -> already downloaded {os.path.basename(cmd[3])}")
-                print(f" -> already downloaded {name}")
-            models.append(name)
+                print(f"Already downloaded {name}")
+            downloaded_models.append(name)
 
         else:
-            print(f" -> missing {name}")
+            print(f" -> Missing {name}")
 
-    return models
+    return downloaded_models