spark3 off release

Author: Ben-Epstein

requirements.txt

@@ -1,7 +1,7 @@
-py4j==0.10.7.0
+py4j
 mlflow==1.8.0
 pyyaml
-mleap==0.15.0
+mleap==0.16.0
 graphviz
 requests
 gorilla==0.3.0
@@ -10,7 +10,7 @@ pyspark-dist-explore==0.1.8
 numpy
 pandas
 scipy
-pyspark>=2.4.0,<=2.4.5
-h2o-pysparkling-2.4==3.28.1.2-1
+pyspark>=3.0.1
+h2o-pysparkling-3.0==3.32.0.4-1
 IPython
 cloudpickle==1.6.0

requirements3.txt requirements2.txtrequirements3.txt renamed to requirements2.txt

@@ -1,7 +1,7 @@
-py4j
+py4j==0.10.7.0
 mlflow==1.8.0
 pyyaml
-mleap==0.16.0
+mleap==0.15.0
 graphviz
 requests
 gorilla==0.3.0
@@ -10,7 +10,7 @@ pyspark-dist-explore==0.1.8
 numpy
 pandas
 scipy
-pyspark>=3.0.1
-h2o-pysparkling-3.0==3.32.0.4-1
+pyspark>=2.4.0,<=2.4.5
+h2o-pysparkling-2.4==3.28.1.2-1
 IPython
 cloudpickle==1.6.0

splicemachine/mlflow_support/mlflow_support.py

@@ -63,8 +63,6 @@
 import requests
 import sklearn
 import yaml
-# from h2o.model.model_base import ModelBase as H2OModel TODO: For Spark3
-from h2o.estimators.estimator_base import ModelBase as H2OModel
 from pandas.core.frame import DataFrame as PandasDF
 from pyspark.ml.base import Model as SparkModel
 from pyspark.sql import DataFrame as SparkDF
@@ -78,6 +76,11 @@
 from splicemachine import SpliceMachineException
 from splicemachine.spark.context import PySpliceContext
 
+try: # PySpark/H2O 3.X
+    from h2o.model.model_base import ModelBase as H2OModel
+except: # PySpark/H2O 2.X
+    from h2o.estimators.estimator_base import ModelBase as H2OModel
+
 # For recording notebook history
 try:
     from IPython import get_ipython