Dbaas 4217 (DBAAS-4377) (#78)

* DBAAS-4217: Updated Pysplice logging to use mlmodel

* DBAAS-4217: Reverted Spark2.2 Changes

* more testing

* DBAAS-4217: more testing 2

* more testinfg 3

* fixed bytearray bug

* DBAAS-4217: some new 🐹

* DBAAS-4217: some new  stuff

* DBAAS-4217: ben is that goat

* DBAAS-4217: ben is the goat

* DBAAS-4217: some new stuff

* DBAAS-4217: Removed mlruns

* fixed logging bugs

* Added DB Deploy and cleaned up and added k8s

* fixed df bool check

* added k8s deploy to targets

* added reference table

* fixed api endpoint

* fixed delay on rt log watching

* fixed api endpoint

* better algorithm for log streaming

* testing log watching from ip

* fixed watch logs

* fixed some new stuff

* auto login

* logs

* pickle

Co-authored-by: Amrit Baveja <[email protected]>

docs/conf.py

@@ -6,13 +6,14 @@
 
 # -- Path setup --------------------------------------------------------------
 
+import inspect
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
 import sys
-import inspect
+
 sys.path.insert(0, os.path.abspath('..'))
 
 

requirements.txt

@@ -1,6 +1,7 @@
-py4j==0.10.7
+py4j==0.10.7.0
 pytest==5.1.3
-mlflow==1.6.0
+mlflow==1.8.0
+pyyaml==5.3.1
 mleap==0.15.0
 graphviz==0.13
 requests
@@ -15,3 +16,4 @@ pyspark
 h2o-pysparkling-2.4==3.28.1.2-1
 sphinx-tabs
 IPython
+cloudpickle==1.6.0

setup.py

@@ -14,7 +14,7 @@
 limitations under the License.
 """
 
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 REQUIREMENTS_FILE = "requirements.txt"
 

splicemachine/mlflow_support/constants.py

@@ -1,48 +1,4 @@
-from enum import Enum
-
-# When storing models for in-DB deployment, we save the model with a name.
-class DBLibraries():
-    MLeap = 'mleap'
-    H2OMOJO = 'h2omojo'
-    SKLearn = 'sklearn'
-    Keras = 'keras'
-    SUPPORTED_LIBRARIES = [MLeap, H2OMOJO, SKLearn, Keras]
-
-class H2OModelType(Enum): # Based off https://github.com/h2oai/h2o-3/blob/master/h2o-genmodel/src/main/java/hex/ModelCategory.java
-    REGRESSION = 0 # Models that return a single Double value (Regression, HGLMRegression)
-    SINGULAR = 1 # Models that return a single Int value (Clustering)
-    CLASSIFICATION = 2 # Models that only return N classes with values associated (Binomial, Multinomial, Ordinal)
-    KEY_VALUE = 3 # Models whose output labels are known (AutoEncoder, TargetEncoder, DimReduction, WordEmbedding, AnomalyDetection)
-
-
-class SparkModelType(Enum):
-    """
-    Model types for MLeap Deployment to DB
-    """
-    CLASSIFICATION = 0
-    REGRESSION = 1
-    CLUSTERING_WITH_PROB = 2
-    CLUSTERING_WO_PROB = 3
-
-class SklearnModelType(Enum):
-    """
-    Model Types for SKLearn models
-    Sklearn isn't as well defined in their model categories, so we are going to classify them by their return values
-    """
-    REGRESSION = 0
-    POINT_PREDICTION_CLF = 1
-    KEY_VALUE = 2
-
-class KerasModelType(Enum):
-    """
-    Model Types for SKLearn models
-    Sklearn isn't as well defined in their model categories, so we are going to classify them by their return values
-    """
-    REGRESSION = 0
-    KEY_VALUE = 1
-
-
-class FileExtensions():
+class FileExtensions:
     """
     Class containing names for
     valid File Extensions
@@ -63,6 +19,7 @@ def get_valid() -> tuple:
             FileExtensions.spark, FileExtensions.keras, FileExtensions.h2o, FileExtensions.sklearn
         )
 
+
 class ModelStatuses():
     """
     Class containing names