Support better native support for SKlearn

- Remove SKlearnCompat wrapper
  - The make_sklearn_compat and .to_lale() methods are
    marked as deprecated and will be removed soon.
    They are now identity functions
- get_params is now supported for Operators (at least for deep=False)
- sklearn clone is now supported for Operators
- set_params is now supported for TrainableOperators and BasePipelines
- with_params, an immutable variant of set_params is now supported for all Operators
  and the recommended alternative

- Implementation of Operator:
  - _impl and _impl_class_ have changed a bit.  Make sure that you use
    the accessor functions to access them and instantiate them as needed.
  - _hyperparams may now store values that were not explicitly set by the user, with
    _frozen_hyperparams marking which keys were explicitly set.  Use the
    various hyperparams() accessor methods to get the desired set of values.
  - Since one of our tests (TestAutoAIOutputConsumption) currently uses an old pickle,
    code was temporarily added to migrate this pickle to the new format.

Author: shinnar

examples/docs_guide_for_sklearn_users.ipynb

@@ -1262,7 +1262,7 @@
    ],
    "source": [
     "from lale.pretty_print import ipython_display\n",
-    "ipython_display(Tree.get_defaults())"
+    "ipython_display(dict(Tree.get_defaults()))"
    ]
   },
   {

lale/grammar.py

@@ -8,11 +8,11 @@
     Operator,
     OperatorChoice,
     PlannedOperator,
+    clone_op,
     make_choice,
     make_pipeline,
     make_pipeline_graph,
 )
-from lale.sklearn_compat import clone_op
 
 
 class NonTerminal(Operator):
@@ -24,6 +24,25 @@ def get_params(self, deep: bool = True) -> Dict[str, Any]:
         out["name"] = self._name
         return out
 
+    def _with_params(self, try_mutate: bool, **impl_params) -> Operator:
+        """
+        This method updates the parameters of the operator.  NonTerminals do not support
+        in-place mutation
+        """
+        known_keys = set(["name"])
+        if impl_params:
+            new_keys = set(impl_params.keys())
+            if not new_keys.issubset(known_keys):
+                unknowns = {k: v for k, v in impl_params.items() if k not in known_keys}
+                raise ValueError(
+                    f"NonTerminal._with_params called with unknown parameters: {unknowns}"
+                )
+            else:
+                assert "name" in impl_params
+                return NonTerminal(impl_params["name"])
+        else:
+            return self
+
     def __init__(self, name):
         self._name = name
 
@@ -56,8 +75,19 @@ def get_params(self, deep: bool = True) -> Dict[str, Any]:
         out = {}
         out["variables"] = self.variables
         # todo: support deep=True
+        # just like a higher order operator does
         return out
 
+    def _with_params(self, try_mutate: bool, **impl_params) -> Operator:
+        """
+        This method updates the parameters of the operator.
+        If try_mutate is set, it will attempt to update the operator in place
+        this may not always be possible
+        """
+        # TODO implement support
+        # from this point of view, Grammar is just a higher order operator
+        raise NotImplementedError("setting Grammar parameters is not yet supported")
+
     def __init__(self, variables: Dict[str, Operator] = {}):
         self._variables = variables
 

lale/helpers.py

@@ -21,7 +21,18 @@
 import sys
 import time
 import traceback
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+)
 
 import h5py
 import jsonschema
@@ -434,8 +445,9 @@ def create_instance_from_hyperopt_search_space(lale_object, hyperparams):
 
     if isinstance(lale_object, PlannedIndividualOp):
         new_hyperparams: Dict[str, Any] = dict_without(hyperparams, "name")
-        if lale_object._hyperparams is not None:
-            obj_hyperparams = dict(lale_object._hyperparams)
+        hps = lale_object.hyperparams()
+        if hps is not None:
+            obj_hyperparams = dict(hps)
         else:
             obj_hyperparams = {}
 
@@ -548,7 +560,7 @@ def get_equivalent_lale_op(sklearn_obj, fitted):
             lale_op = class_
         else:
             lale_op = lale.operators.TrainedIndividualOp(
-                class_._name, class_._impl, class_._schemas, None
+                class_._name, class_._impl, class_._schemas, None, _lale_trained=True
             )
 
         try:
@@ -767,3 +779,140 @@ def add_missing_values(orig_X, missing_rate=0.1, seed=None):
                 i_missing_sample += 1
                 missing_X.iloc[i_sample, i_feature] = np.nan
     return missing_X
+
+
+# helpers for manipulating (extended) sklearn style paths.
+# documentation of the path format is part of the operators module docstring
+
+
+def partition_sklearn_params(
+    d: Dict[str, Any]
+) -> Tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]:
+    sub_parts: Dict[str, Dict[str, Any]] = {}
+    main_parts: Dict[str, Any] = {}
+
+    for k, v in d.items():
+        ks = k.split("__", 1)
+        if len(ks) == 1:
+            assert k not in main_parts
+            main_parts[k] = v
+        else:
+            assert len(ks) == 2
+            bucket: Dict[str, Any] = {}
+            group: str = ks[0]
+            param: str = ks[1]
+            if group in sub_parts:
+                bucket = sub_parts[group]
+            else:
+                sub_parts[group] = bucket
+            assert param not in bucket
+            bucket[param] = v
+    return (main_parts, sub_parts)
+
+
+def partition_sklearn_choice_params(d: Dict[str, Any]) -> Tuple[int, Dict[str, Any]]:
+    discriminant_value: int = -1
+    choice_parts: Dict[str, Any] = {}
+
+    for k, v in d.items():
+        if k == discriminant_name:
+            assert discriminant_value == -1
+            discriminant_value = int(v)
+        else:
+            k_rest = unnest_choice(k)
+            choice_parts[k_rest] = v
+    assert discriminant_value != -1
+    return (discriminant_value, choice_parts)
+
+
+DUMMY_SEARCH_SPACE_GRID_PARAM_NAME: str = "$"
+discriminant_name: str = "?"
+choice_prefix: str = "?"
+structure_type_name: str = "#"
+structure_type_list: str = "list"
+structure_type_tuple: str = "tuple"
+structure_type_dict: str = "dict"
+
+
+def get_name_and_index(name: str) -> Tuple[str, int]:
+    """ given a name of the form "name@i", returns (name, i)
+        if given a name of the form "name", returns (name, 0)
+    """
+    splits = name.split("@", 1)
+    if len(splits) == 1:
+        return splits[0], 0
+    else:
+        return splits[0], int(splits[1])
+
+
+def make_degen_indexed_name(name, index):
+    return f"{name}@{index}"
+
+
+def make_indexed_name(name, index):
+    if index == 0:
+        return name
+    else:
+        return f"{name}@{index}"
+
+
+def make_array_index_name(index, is_tuple: bool = False):
+    sep = "##" if is_tuple else "#"
+    return f"{sep}{str(index)}"
+
+
+def is_numeric_structure(structure_type: str):
+
+    if structure_type == "list" or structure_type == "tuple":
+        return True
+    elif structure_type == "dict":
+        return False
+    else:
+        assert False, f"Unknown structure type {structure_type} found"
+
+
+V = TypeVar("V")
+
+
+def nest_HPparam(name: str, key: str):
+    if key == DUMMY_SEARCH_SPACE_GRID_PARAM_NAME:
+        # we can get rid of the dummy now, since we have a name for it
+        return name
+    return name + "__" + key
+
+
+def nest_HPparams(name: str, grid: Mapping[str, V]) -> Dict[str, V]:
+    return {(nest_HPparam(name, k)): v for k, v in grid.items()}
+
+
+def nest_all_HPparams(
+    name: str, grids: Iterable[Mapping[str, V]]
+) -> List[Dict[str, V]]:
+    """ Given the name of an operator in a pipeline, this transforms every key(parameter name) in the grids
+        to use the operator name as a prefix (separated by __).  This is the convention in scikit-learn pipelines.
+    """
+    return [nest_HPparams(name, grid) for grid in grids]
+
+
+def nest_choice_HPparam(key: str):
+    return choice_prefix + key
+
+
+def nest_choice_HPparams(grid: Mapping[str, V]) -> Dict[str, V]:
+    return {(nest_choice_HPparam(k)): v for k, v in grid.items()}
+
+
+def nest_choice_all_HPparams(grids: Iterable[Mapping[str, V]]) -> List[Dict[str, V]]:
+    """ this transforms every key(parameter name) in the grids
+        to be nested under a choice, using a ? as a prefix (separated by __).  This is the convention in scikit-learn pipelines.
+    """
+    return [nest_choice_HPparams(grid) for grid in grids]
+
+
+def unnest_choice(k: str) -> str:
+    assert k.startswith(choice_prefix)
+    return k[len(choice_prefix) :]
+
+
+def unnest_HPparams(k: str) -> List[str]:
+    return k.split("__")

lale/json_operator.py

@@ -391,12 +391,12 @@ def _op_to_json_rec(
         if isinstance(op, lale.operators.TrainableIndividualOp):
             if hasattr(op._impl, "viz_label"):
                 jsn["viz_label"] = op._impl.viz_label()
-            if op.hyperparams() is None:
+            if op.reduced_hyperparams() is None:
                 jsn["hyperparams"] = None
             else:
                 steps: Dict[str, JSON_TYPE] = {}
                 jsn["hyperparams"] = _hps_to_json_rec(
-                    op.hyperparams(), cls2label, gensym, steps
+                    op.reduced_hyperparams(), cls2label, gensym, steps
                 )
                 if len(steps) > 0:
                     jsn["steps"] = steps

lale/lib/imblearn/base_resampler.py

@@ -12,12 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from lale.sklearn_compat import make_sklearn_compat_opt
-
 
 class BaseResamplerImpl:
     def __init__(self, operator=None, resampler=None):
-        self.operator = make_sklearn_compat_opt(operator)
+        self.operator = operator
         self.resampler = resampler
 
     def fit(self, X, y=None):

lale/lib/lale/grid_search_cv.py

@@ -71,8 +71,14 @@ def fit(self, X, y):
 
         observed_op = op
         obs = self._hyperparams["observer"]
-        if obs is not None:
-            observed_op = Observing(op=op, observer=obs)
+        # We always create an observer.
+        # Otherwise, we can have a problem with PlannedOperators
+        # (that are not trainable):
+        # GridSearchCV checks if a fit method is present before
+        # configuring the operator, and our planned operators
+        # don't have a fit method
+        # Observing always has a fit method, and so solves this problem.
+        observed_op = Observing(op=op, observer=obs)
 
         hp_grid = self._hyperparams["hp_grid"]
         data_schema = lale.helpers.fold_schema(
@@ -86,14 +92,24 @@ def fit(self, X, y):
                 pgo=self._hyperparams["pgo"],
                 data_schema=data_schema,
             )
+        else:
+            # if hp_grid is specified manually, we need to add a level of nesting
+            # since we are wrapping it in an observer
+            if isinstance(hp_grid, list):
+                hp_grid = lale.helpers.nest_all_HPparams("op", hp_grid)
+            else:
+                assert isinstance(hp_grid, dict)
+                hp_grid = lale.helpers.nest_HPparams("op", hp_grid)
+
         if not hp_grid and isinstance(op, lale.operators.IndividualOp):
             hp_grid = [
-                lale.search.lale_grid_search_cv.get_defaults_as_param_grid(observed_op)
+                lale.search.lale_grid_search_cv.get_defaults_as_param_grid(observed_op)  # type: ignore
             ]
         be: lale.operators.TrainableOperator
         if hp_grid:
             if obs is not None:
-                observed_op._impl.startObserving(
+                impl = observed_op._impl  # type: ignore
+                impl.startObserving(
                     "optimize",
                     hp_grid=hp_grid,
                     op=op,
@@ -103,27 +119,30 @@ def fit(self, X, y):
                 )
             try:
                 self.grid = lale.search.lale_grid_search_cv.get_lale_gridsearchcv_op(
-                    lale.sklearn_compat.make_sklearn_compat(observed_op),
+                    observed_op,
                     hp_grid,
                     cv=self._hyperparams["cv"],
                     verbose=self._hyperparams["verbose"],
                     scoring=self._hyperparams["scoring"],
                     n_jobs=self._hyperparams["n_jobs"],
                 )
                 self.grid.fit(X, y)
-                be = self.grid.best_estimator_.to_lale()
+                be = self.grid.best_estimator_
             except BaseException as e:
                 if obs is not None:
-                    assert isinstance(observed_op._impl, ObservingImpl)
-                    observed_op._impl.failObserving("optimize", e)
+                    impl = observed_op._impl  # type: ignore
+                    assert isinstance(impl, ObservingImpl)
+                    impl.failObserving("optimize", e)
                 raise
 
-            if obs is not None:
-                impl = getattr(be, "_impl")
-                if impl is not None:
-                    assert isinstance(impl, ObservingImpl)
-                    be = impl.getOp()
-                    observed_op._impl.endObserving("optimize", best=be)
+            impl = getattr(be, "_impl", None)
+            if impl is not None:
+                assert isinstance(impl, ObservingImpl)
+                be = impl.getOp()
+                if obs is not None:
+                    obs_impl = observed_op._impl  # type: ignore
+
+                    obs_impl.endObserving("optimize", best=be)
         else:
             assert isinstance(op, lale.operators.TrainableOperator)
             be = op
@@ -142,7 +161,8 @@ def get_pipeline(self, pipeline_name=None, astype="lale"):
         if result is None or astype == "lale":
             return result
         assert astype == "sklearn", astype
-        return lale.sklearn_compat.make_sklearn_compat(result)
+        # TODO: should this try and return an actual sklearn pipeline?
+        return result
 
 
 _hyperparams_schema = {

lale/lib/lale/smac.py

@@ -217,7 +217,8 @@ def get_pipeline(self, pipeline_name=None, astype="lale"):
         if result is None or astype == "lale":
             return result
         assert astype == "sklearn", astype
-        return lale.sklearn_compat.make_sklearn_compat(result)
+        # TODO: should this try and return an actual sklearn pipeline?
+        return result
 
 
 _hyperparams_schema = {