Use coerce in datetime transform (#1162)

* use coerce in datetime transform

* update release notes

* remove unused fixture

* update release notes

* add warning message for pandas

* update warning message

Author: thehomebrewnerd

docs/source/release_notes.rst

@@ -7,6 +7,7 @@ Future Release
     * Enhancements
     * Fixes
         * Fixed an issue when inferring the format of datetime strings with day of week or meridiem placeholders (:pr:`1158`)
+        * Implements change in ``Datetime.transform`` to prevent initialization failure in some cases (:pr:`1162`)
     * Changes
     * Documentation Changes
     * Testing Changes

woodwork/exceptions.py

@@ -45,6 +45,10 @@ def __init__(self, series, new_dtype, logical_type):
         super().__init__(message)
 
 
+class TypeConversionWarning(UserWarning):
+    pass
+
+
 class ParametersIgnoredWarning(UserWarning):
     pass
 

woodwork/logical_types.py

@@ -1,8 +1,10 @@
+import warnings
+
 import pandas as pd
 import pandas.api.types as pdtypes
 
 from woodwork.accessor_utils import _is_dask_series, _is_koalas_series
-from woodwork.exceptions import TypeConversionError
+from woodwork.exceptions import TypeConversionError, TypeConversionWarning
 from woodwork.type_sys.utils import _get_specified_ltype_params
 from woodwork.utils import (
     _infer_datetime_format,
@@ -200,19 +202,24 @@ def transform(self, series):
         new_dtype = self._get_valid_dtype(type(series))
         if new_dtype != str(series.dtype):
             self.datetime_format = self.datetime_format or _infer_datetime_format(series)
-            try:
-                if _is_dask_series(series):
-                    name = series.name
-                    series = dd.to_datetime(series, format=self.datetime_format)
-                    series.name = name
-                elif _is_koalas_series(series):
-                    series = ks.Series(ks.to_datetime(series.to_numpy(),
-                                                      format=self.datetime_format),
-                                       name=series.name)
-                else:
+            if _is_dask_series(series):
+                name = series.name
+                series = dd.to_datetime(series, format=self.datetime_format, errors="coerce")
+                series.name = name
+            elif _is_koalas_series(series):
+                series = ks.Series(ks.to_datetime(series.to_numpy(),
+                                                  format=self.datetime_format,
+                                                  errors="coerce"),
+                                   name=series.name)
+            else:
+                try:
                     series = pd.to_datetime(series, format=self.datetime_format)
-            except (TypeError, ValueError):
-                raise TypeConversionError(series, new_dtype, type(self))
+                except (TypeError, ValueError):
+                    warnings.warn(f"Some rows in series '{series.name}' are incompatible with datetime format "
+                                  f"'{self.datetime_format}' and have been replaced with null values. You may be "
+                                  "able to fix this by using an instantiated Datetime logical type with a different format "
+                                  "string specified for this column during Woodwork initialization.", TypeConversionWarning)
+                    series = pd.to_datetime(series, format=self.datetime_format, errors="coerce")
         return super().transform(series)
 
 

woodwork/tests/logical_types/test_logical_types.py

@@ -3,8 +3,8 @@
 import pandas as pd
 import pytest
 
-from woodwork.accessor_utils import _is_dask_series, _is_koalas_series
-from woodwork.exceptions import TypeConversionError
+from woodwork.accessor_utils import _is_koalas_series
+from woodwork.exceptions import TypeConversionWarning
 from woodwork.logical_types import (
     Boolean,
     Categorical,
@@ -141,15 +141,32 @@ def test_datetime_transform(datetimes):
         assert datetime.datetime_format is not None
 
 
-def test_datetime_conversion_error(sample_series):
-    if _is_dask_series(sample_series):
-        pytest.xfail('Dask does not show error until compute is made.')
-
-    dtype = str(sample_series.dtype)
-    match = f'Error converting datatype for sample_series from type {dtype} to type datetime64[ns]. '
-    match += 'Please confirm the underlying data is consistent with logical type Datetime.'
-    with pytest.raises(TypeConversionError, match=re.escape(match)):
-        Datetime().transform(sample_series)
+def test_datetime_inference_ambiguous_format():
+    datetime = Datetime()
+    dates = pd.Series(["01/01/2017"] * 2 + ["13/12/2017"], name="dates")
+    warning = "Some rows in series 'dates' are incompatible with datetime format " \
+              "'%m/%d/%Y' and have been replaced with null values. You may be able " \
+              "to fix this by using an instantiated Datetime logical type with a different " \
+              "format string specified for this column during Woodwork initialization."
+    with pytest.warns(TypeConversionWarning, match=warning):
+        transformed = datetime.transform(dates)
+    assert str(transformed.dtype) == "datetime64[ns]"
+    assert transformed[2] is pd.NaT
+    assert datetime.datetime_format == "%m/%d/%Y"
+
+
+def test_datetime_coerce_user_format():
+    datetime = Datetime(datetime_format="%m/%d/%Y")
+    dates = pd.Series(["01/01/2017"] * 2 + ["13/12/2017"], name="dates")
+    warning = "Some rows in series 'dates' are incompatible with datetime format " \
+              "'%m/%d/%Y' and have been replaced with null values. You may be able " \
+              "to fix this by using an instantiated Datetime logical type with a different " \
+              "format string specified for this column during Woodwork initialization."
+    with pytest.warns(TypeConversionWarning, match=warning):
+        transformed = datetime.transform(dates)
+    assert str(transformed.dtype) == "datetime64[ns]"
+    assert transformed[2] is pd.NaT
+    assert datetime.datetime_format == "%m/%d/%Y"
 
 
 def test_ordinal_transform(sample_series):