Do not depend on pkg not specified in setup.py (#214)

* Added a test agains missing dependencies

* Make sure distributed is installed automatically

* Make the joblib dependency optional

* Make the sklearn dependency optional

Author: nils-braun

.github/workflows/test.yml

@@ -170,3 +170,44 @@ jobs:
           pytest tests
         env:
           DASK_SQL_TEST_SCHEDULER: tcp://127.0.0.1:8786
+  test_import:
+    name: "Test importing with bare requirements"
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache local Maven repository
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-v1-11-${{ hashFiles('**/pom.xml') }}
+      - name: Cache downloaded conda packages
+        uses: actions/cache@v2
+        with:
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-conda-v2-11-${{ hashFiles('conda.txt') }}
+      - name: Set up Python
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: 3.8
+          mamba-version: "*"
+          channels: conda-forge,defaults
+          channel-priority: true
+          use-only-tar-bz2: true
+      - name: Download the pre-build jar
+        uses: actions/download-artifact@v1
+        with:
+          name: jar
+          path: dask_sql/jar/
+      - name: Install dependencies and nothing else
+        shell: bash -l {0}
+        run: |
+          pip install -e .
+
+          which python
+          pip list
+          mamba list
+      - name: Try to import dask-sql
+        shell: bash -l {0}
+        run: |
+          python -c "import dask_sql; print('ok')"

dask_sql/physical/rel/custom/export_model.py

@@ -1,9 +1,6 @@
 import logging
 import pickle
 
-import joblib
-import sklearn
-
 from dask_sql.physical.rel.base import BaseRelPlugin
 from dask_sql.utils import convert_sql_kwargs
 
@@ -62,16 +59,21 @@ def convert(
             with open(location, "wb") as pkl_file:
                 pickle.dump(model, pkl_file, **kwargs)
         elif format == "joblib":
-            joblib.dump(model, location, **kwargs)
+            import joblib
 
+            joblib.dump(model, location, **kwargs)
         elif format == "mlflow":
             try:
                 import mlflow
             except ImportError:  # pragma: no cover
                 raise ImportError(
                     f"For export in the mlflow format, you need to have mlflow installed"
                 )
-            if isinstance(model, sklearn.base.BaseEstimator):
+            try:
+                import sklearn
+            except ImportError:  # pragma: no cover
+                sklearn = None
+            if sklearn is not None and isinstance(model, sklearn.base.BaseEstimator):
                 mlflow.sklearn.save_model(model, location, **kwargs)
             else:
                 raise NotImplementedError(

setup.py

@@ -74,7 +74,7 @@ def run(self):
     python_requires=">=3.6",
     setup_requires=["setuptools_scm"] + sphinx_requirements,
     install_requires=[
-        "dask[dataframe]>=2.19.0,!=2021.3.0",  # dask 2021.3.0 makes
+        "dask[dataframe,distributed]>=2.19.0,!=2021.3.0",  # dask 2021.3.0 makes
         # dask-ml fail (see https://github.com/dask/dask-ml/issues/803)
         "pandas>=1.0.0",  # below 1.0, there were no nullable ext. types
         "jpype1>=1.0.2",