Start tests enusc dtmgt and small logistical changes.

Author: MarenBermudezBoeckle

src/project_mbb/data_management/clean_enusc.py

@@ -1,38 +1,37 @@
 import pandas as pd
 
-pd.options.mode.copy_on_write = True
-pd.options.future.infer_string = True
-pd.options.plotting.backend = "plotly"
-
 from project_mbb.parameters import (
     categories,
     floats,
     integers,
     map_category,
-    relevant_var,
     rename_mapping,
     strings,
 )
 
+pd.options.mode.copy_on_write = True
+pd.options.future.infer_string = True
+pd.options.plotting.backend = "plotly"
+
 
 def clean_enusc(raw_enusc):
     enusc_filtered = _filter_enusc(raw_enusc)
-    enusc_renamed = _rename_enusc(enusc_filtered)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
     enusc_category = _map_categories(enusc_renamed)
     # here check for no objects present
     enusc_filled = _fill_missings(enusc_category)
     enusc_dtypes = _set_data_types_of_numbers(enusc_filled)
     return enusc_dtypes
 
 
-def _filter_enusc(raw_enusc):
+def _filter_enusc(raw_enusc, relevant_var):
     enusc_filtered = raw_enusc[relevant_var]
     return enusc_filtered
 
 
-def _rename_enusc(enusc_lower):
-    enusc_lower.columns = enusc_lower.columns.str.lower()
-    enusc_renamed = enusc_lower.rename(columns=rename_mapping)
+def _rename_enusc(enusc_filtered, rename_mapping):
+    enusc_filtered.columns = enusc_filtered.columns.str.lower()
+    enusc_renamed = enusc_filtered.rename(columns=rename_mapping)
     return enusc_renamed
 
 

src/project_mbb/data_management/clean_labels.py

@@ -1,16 +1,16 @@
 import pandas as pd
 
+from project_mbb.parameters import relevant_var, rename_mapping
+
 pd.options.mode.copy_on_write = True
 pd.options.future.infer_string = True
 pd.options.plotting.backend = "plotly"
 
-from project_mbb.parameters import relevant_var, rename_mapping
-
 
-def create_labels_var_relation(raw_enusc, labels_raw):
-    labels_var = _concat_labels_var(raw_enusc, labels_raw)
-    labels_var_filt = _filter_labels_var(labels_var)
-    labels_renamed = _rename_variables_eng(labels_var_filt)
+def create_labels_var_relation(raw_enusc, column_labels):
+    labels_var = _concat_labels_var(raw_enusc, column_labels)
+    labels_var_filt = _filter_labels_var(labels_var, relevant_var)
+    labels_renamed = _rename_variables_eng(labels_var_filt, rename_mapping)
     return labels_renamed
 
 
@@ -24,12 +24,12 @@ def _concat_labels_var(raw_enusc, column_labels):
     return labels_var
 
 
-def _filter_labels_var(var_labels):
+def _filter_labels_var(var_labels, relevant_var):
     var_labels_filt = var_labels[var_labels["variable_name"].isin(relevant_var)]
     return var_labels_filt
 
 
-def _rename_variables_eng(labels_var_filt):
+def _rename_variables_eng(labels_var_filt, rename_mapping):
     labels_var_filt["variable_name"] = labels_var_filt["variable_name"].str.lower()
     labels_renamed = labels_var_filt.replace({"variable_name": rename_mapping})
     return labels_renamed

src/project_mbb/data_management/task_data_management.py

@@ -41,9 +41,9 @@ def task_clean_labels(
     labels_raw=BLD / "data" / "column_labels.csv",
     produces=BLD / "data" / "variable_labels_clean.pkl",
 ):
-    raw = pd.read_csv(raw)
+    raw_enusc = pd.read_csv(raw)
     labels_raw = pd.read_csv(labels_raw)
-    labels_var = create_labels_var_relation(raw, labels_raw)
+    labels_var = create_labels_var_relation(raw_enusc, labels_raw)
     labels_var.to_pickle(produces)
 
 

src/project_mbb/parameters.py

@@ -123,7 +123,6 @@
     "hur_contacto_mp": "contact_prosecutor_post_report",
     "hur_motiv_no_den": "reason_not_reporting_theft",
     # residency history
-    "antig_sector": "residency_duration_sector",
     # crime perception indicators
     "pad": "crime_perception_national",
     "padc": "crime_perception_commune",
@@ -467,7 +466,6 @@
 }
 
 
-
 floats = {
     "fact_pers_com",
     "fact_pers_reg",
@@ -511,10 +509,11 @@
     "household_theft_victim",
     "theft_reported",
     "education_level",
-    "reason_not_reporting_theft"
+    "reason_not_reporting_theft",
 }
 
 
-strings = {"person_id",
-"household_id",
-}
+strings = {
+    "person_id",
+    "household_id",
+}

tests/data_management/test_data_mgt_enusc.py

@@ -0,0 +1,60 @@
+import pandas as pd
+import pytest
+
+from project_mbb.config import BLD
+from project_mbb.data_management.clean_enusc import (
+    _filter_enusc,
+    _rename_enusc,
+)
+from project_mbb.parameters import relevant_var, rename_mapping
+
+
+@pytest.fixture
+def enusc_raw():
+    enusc_raw_path = BLD / "data" / "ENUSC_raw.csv"
+    enusc_raw = pd.read_csv(enusc_raw_path)
+    return enusc_raw
+
+
+@pytest.fixture
+def data_test():
+    data = {
+        "variable_name": ["rph_id", "idhogar", "enc_idr", "enc_region", "enc_rpc"],
+        "label": [
+            "Identificador de persona",
+            "Identificador de hogar",
+            "Folio de la vivienda",
+            "Región",
+            "Región, provincia, comuna",
+        ],
+    }
+    return pd.DataFrame(data)
+
+
+def test_check_if_filter_correct_columns(enusc_raw):
+    enusc_filtered = _filter_enusc(enusc_raw, relevant_var)
+
+    actual_columns = set(enusc_filtered.columns)
+    expected_columns = set(relevant_var)
+
+    missing_columns = expected_columns - actual_columns
+    extra_columns = actual_columns - expected_columns
+
+    assert (
+        expected_columns == actual_columns
+    ), f"Mismatch in columns. Missing: {missing_columns}, Extra: {extra_columns}"
+
+
+def test_rename_enusc(enusc_raw):
+    enusc_filtered = _filter_enusc(enusc_raw, relevant_var)
+    enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
+
+    actual_columns = set(enusc_renamed.columns)
+    expected_columns = set(rename_mapping.values())
+
+    missing_columns = expected_columns - actual_columns
+    extra_columns = actual_columns - expected_columns
+
+    assert expected_columns.issubset(
+        actual_columns
+    ), f"Missing columns: {missing_columns}, Extra: {extra_columns}"

tests/data_management/test_data_mgt_labels.py

@@ -47,7 +47,7 @@ def test_concat_labels_var(enusc_raw, labels_raw):
     assert "label" in data_test.columns, "Column 'label' not found in DataFrame"
 
     # indexes are original ones: s.t if we want to include other variables into
-    # the filtering then we can continue using this test
+    # the filtering we can continue using this test
     assert all(
         [
             data_test["variable_name"].iloc[0] == "rph_ID"