Complete tests for data mgt enusc.

Author: MarenBermudezBoeckle

tests/data_management/test_data_mgt_enusc.py

@@ -3,10 +3,22 @@
 
 from project_mbb.config import BLD
 from project_mbb.data_management.clean_enusc import (
+    _fill_missing,
     _filter_enusc,
+    _map_categories,
     _rename_enusc,
+    _set_data_types,
+)
+from project_mbb.parameters import (
+    categories,
+    floats,
+    integers,
+    relevant_var,
+    rename_mapping,
+    security_levels_mapping,
+    ses_mapping,
+    strings,
 )
-from project_mbb.parameters import relevant_var, rename_mapping
 
 
 @pytest.fixture
@@ -17,21 +29,13 @@ def enusc_raw():
 
 
 @pytest.fixture
-def data_test():
-    data = {
-        "variable_name": ["rph_id", "idhogar", "enc_idr", "enc_region", "enc_rpc"],
-        "label": [
-            "Identificador de persona",
-            "Identificador de hogar",
-            "Folio de la vivienda",
-            "Región",
-            "Región, provincia, comuna",
-        ],
-    }
-    return pd.DataFrame(data)
-
-
-def test_check_if_filter_correct_columns(enusc_raw):
+def enusc_raw_short():
+    enusc_raw_path = BLD / "data" / "ENUSC_raw.csv"
+    enusc_raw_short = pd.read_csv(enusc_raw_path, nrows=3)
+    return enusc_raw_short
+
+
+def test_filter_correct_columns(enusc_raw):
     enusc_filtered = _filter_enusc(enusc_raw, relevant_var)
 
     actual_columns = set(enusc_filtered.columns)
@@ -58,3 +62,160 @@ def test_rename_enusc(enusc_raw):
     assert expected_columns.issubset(
         actual_columns
     ), f"Missing columns: {missing_columns}, Extra: {extra_columns}"
+
+
+@pytest.mark.parametrize(
+    ("column", "expected_value"),
+    [
+        ("socioecon_status", "Low SES"),
+        ("insec_driving", "Secure"),
+        ("trust_pdi", "Quite a bit of trust"),
+        ("crime_info_source_nation", "News programs on television"),
+    ],
+)
+def test_value_presence(enusc_raw_short, column, expected_value):
+    enusc_filtered = _filter_enusc(enusc_raw_short, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+    enusc_category = _map_categories(enusc_renamed)
+
+    assert (
+        expected_value in enusc_category[column].values
+    ), f" {expected_value} not found in {column}"
+
+
+@pytest.mark.parametrize(
+    "column",
+    ["socioecon_status", "insec_driving", "trust_pdi", "crime_info_source_nation"],
+)
+def test_column_is_categorical(enusc_raw_short, column):
+    enusc_filtered = _filter_enusc(enusc_raw_short, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+    enusc_category = _map_categories(enusc_renamed)
+
+    assert isinstance(
+        enusc_category[column].dtype, pd.CategoricalDtype
+    ), f"{column} is not categorical"
+
+
+@pytest.mark.parametrize(
+    ("column", "values_dict"),
+    [
+        ("socioecon_status", ses_mapping),
+        ("insec_driving", security_levels_mapping),
+    ],
+)
+def test_value_presence_continued(enusc_raw_short, column, values_dict):
+    errors = []
+    enusc_filtered = _filter_enusc(enusc_raw_short, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+    enusc_category = _map_categories(enusc_renamed)
+
+    for value in enusc_category[column]:
+        if pd.isna(value):  # Skip missing values
+            continue
+        if value not in values_dict.values():
+            errors.append(f"Unexpected value {value} in column '{column}'")
+
+    if errors:
+        pytest.fail("\n".join(errors))
+
+
+@pytest.mark.xfail
+def test_fail_age_not_categorical(enusc_raw_short):
+    enusc_filtered = _filter_enusc(enusc_raw_short, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+    enusc_category = _map_categories(enusc_renamed)
+    assert isinstance(enusc_category["age"].dtype, pd.CategoricalDtype)
+
+
+def test_amount_of_missing(enusc_raw_short):
+    enusc_filtered = _filter_enusc(enusc_raw_short, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+    enusc_category = _map_categories(enusc_renamed)
+    enusc_filled = _fill_missing(enusc_category)
+
+    errors = []
+    missing_value = 99
+
+    for column in enusc_raw_short.columns:
+        if column in rename_mapping:
+            filled_column = rename_mapping[column]
+            if filled_column in enusc_filled.columns:
+                missing = enusc_raw_short[column].isna().sum()
+                defined_missing = (enusc_raw_short[column] == missing_value).sum()
+                difference = enusc_filled[filled_column].isna().sum()
+
+                if difference != missing + defined_missing:
+                    errors.append(
+                        f"Column '{column}' (mapped to '{filled_column}') failed: "
+                        f"expected missing value count of {missing + defined_missing},"
+                        f"but got {difference}"
+                    )
+
+    if errors:
+        pytest.fail("\n".join(errors))
+
+
+@pytest.mark.parametrize(
+    ("column", "expected_dtype"),
+    [(value, pd.Float64Dtype()) for value in floats]
+    + [(var, pd.Int8Dtype()) for var in integers]
+    + [(val, pd.CategoricalDtype()) for val in categories]
+    + [(ent, "object") for ent in strings],
+)
+def test_set_data_types(enusc_raw_short, column, expected_dtype):
+    errors = []
+    enusc_filtered = _filter_enusc(enusc_raw_short, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+    enusc_category = _map_categories(enusc_renamed)
+    enusc_filled = _fill_missing(enusc_category)
+    enusc_dtypes = _set_data_types(enusc_filled)
+
+    if isinstance(expected_dtype, pd.CategoricalDtype):
+        if not isinstance(enusc_dtypes[column].dtype, pd.CategoricalDtype):
+            errors.append(
+                f"Column '{column}' failed: expected type {expected_dtype},"
+                f"but got {enusc_dtypes[column].dtype}"
+            )
+    elif enusc_dtypes[column].dtype != expected_dtype:
+        errors.append(
+            f"Column '{column}' failed: expected type {expected_dtype},"
+            f"but got {enusc_dtypes[column].dtype}"
+        )
+
+    if errors:
+        pytest.fail("\n".join(errors))
+
+
+@pytest.mark.xfail
+@pytest.mark.parametrize(
+    ("column", "wrong_expected_dtype"),
+    [(value, pd.CategoricalDtype()) for value in floats]
+    + [(var, pd.Float64Dtype()) for var in integers]
+    + [(val, pd.Float64Dtype()) for val in categories]
+    + [(ent, "thing") for ent in strings],
+)
+def test_fail_set_data_types_for_check_testing(
+    enusc_raw_short, column, wrong_expected_dtype
+):
+    errors = []
+    enusc_filtered = _filter_enusc(enusc_raw_short, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+    enusc_category = _map_categories(enusc_renamed)
+    enusc_filled = _fill_missing(enusc_category)
+    enusc_dtypes = _set_data_types(enusc_filled)
+
+    if isinstance(wrong_expected_dtype, pd.CategoricalDtype):
+        if not isinstance(enusc_dtypes[column].dtype, pd.CategoricalDtype):
+            errors.append(
+                f"Column '{column}' failed: expected type {wrong_expected_dtype},"
+                f" but got {enusc_dtypes[column].dtype}"
+            )
+    elif enusc_dtypes[column].dtype != wrong_expected_dtype:
+        errors.append(
+            f"Column '{column}' failed: expected type {wrong_expected_dtype},"
+            f"but got {enusc_dtypes[column].dtype}"
+        )
+
+    if errors:
+        pytest.fail("\n".join(errors))