Add error handling to data mgt part.

Author: MarenBermudezBoeckle

.DS_Store

@@ -95,6 +95,7 @@ extend-ignore = [
     "S301",  # pickle module is unsafe
     "ARG001", # Unused function MB
     "ERA001", # commented MB
+    "TRY003", # Messages outside exception MB
 ]
 
 [tool.ruff.lint.per-file-ignores]

pyproject.toml

@@ -19,18 +19,19 @@ def clean_enusc(raw_enusc):
     enusc_filtered = _filter_enusc(raw_enusc, relevant_var)
     enusc_renamed = _rename_enusc(enusc_filtered, rename_mapping)
     enusc_mapped = _map_categories(enusc_renamed)
-    # here check for no objects present
     enusc_filled = _fill_missing(enusc_mapped)
     enusc_dtypes = _set_data_types_not_mapped_var(enusc_filled)
     return enusc_dtypes
 
 
 def _filter_enusc(raw_enusc, relevant_var):
+    _fail_if_not_list(relevant_var)
     enusc_filtered = raw_enusc[relevant_var]
     return enusc_filtered
 
 
 def _rename_enusc(enusc_filtered, rename_mapping):
+    _fail_if_not_equal_length(enusc_filtered, rename_mapping)
     enusc_renamed = enusc_filtered.copy()
     enusc_renamed.columns = enusc_renamed.columns.str.lower()
     enusc_renamed = enusc_renamed.rename(columns=rename_mapping)
@@ -52,7 +53,12 @@ def _map_categories(enusc_renamed):
 
 
 def _fill_missing(enusc_mapped):
+    _fail_if_not_dataframe(enusc_mapped)
+    _fail_if_missing_columns(enusc_mapped, categories, "categories")
+    _fail_if_missing_columns(enusc_mapped, map_category, "map_category")
+
     replacements = {99: pd.NA, 77: "Other", 88: "Don't know", 85: "Doesn't apply"}
+
     enusc_filling = enusc_mapped.copy()
     for column in categories:
         enusc_filling[column] = enusc_filling[column].astype(pd.Int8Dtype())
@@ -61,11 +67,7 @@ def _fill_missing(enusc_mapped):
         )
 
         if enusc_mapped[column].dtype != "object":
-            enusc_filling[column] = (
-                enusc_filling[column]
-                # .astype(pd.Int8Dtype()) esto se puede borrar
-                .astype(pd.CategoricalDtype())
-            )
+            enusc_filling[column] = enusc_filling[column].astype(pd.CategoricalDtype())
             enusc_filling[column] = enusc_filling[column].cat.rename_categories(
                 lambda x: replacements.get(x, x)
             )
@@ -86,6 +88,11 @@ def _fill_missing(enusc_mapped):
 
 
 def _set_data_types_not_mapped_var(enusc_filled):
+    _fail_if_columns_not_found(enusc_filled, floats)
+    _fail_if_columns_not_found(enusc_filled, integers)
+    _fail_if_columns_not_found(enusc_filled, categories)
+    _fail_if_columns_not_found(enusc_filled, strings)
+
     enusc_dtypes = enusc_filled.copy()
     for value in floats:
         enusc_dtypes[value] = enusc_dtypes[value].astype(pd.Float64Dtype())
@@ -96,3 +103,61 @@ def _set_data_types_not_mapped_var(enusc_filled):
     for ent in strings:
         enusc_dtypes[ent] = enusc_dtypes[ent].astype(str)
     return enusc_dtypes
+
+
+# Error Handling
+
+
+def _fail_if_not_list(relevant_var):
+    """Raise TypeError if relevant_var for filtering is not a list."""
+    if not isinstance(relevant_var, list):
+        error_msg = f"Expected a list, but got {type(relevant_var).__name__}"
+        raise TypeError(error_msg)
+
+
+class ShapeError(Exception):
+    """Custom exception for errors in _rename_enusc."""
+
+
+def _fail_if_not_equal_length(enusc_filtered, rename_mapping):
+    """Raise ShapeError if data and renaming dictionary have unequal width.
+
+    The number of columns in `enusc_filtered` must match the length of
+    `rename_mapping`.
+    """
+    if enusc_filtered.shape[1] != len(rename_mapping):
+        error_msg = (
+            f"Lists have unequal widths: "
+            f"{enusc_filtered.shape[1]} vs {len(rename_mapping)}"
+        )
+        raise ShapeError(error_msg)
+
+
+class MissingError(Exception):
+    """Custom exception for missing."""
+
+
+def _fail_if_not_dataframe(enusc_mapped):
+    """Raise an error if enusc_mapped is not a Pandas DataFrame."""
+    if not isinstance(enusc_mapped, pd.DataFrame):
+        error_msg = f"Expected a DataFrame, but got {type(enusc_mapped).__name__}"
+        raise TypeError(error_msg)
+
+
+def _fail_if_missing_columns(enusc_mapped, column_list, list_name):
+    """Raise an error if any column in column_list is missing from enusc_mapped."""
+    missing_columns = [col for col in column_list if col not in enusc_mapped.columns]
+    if missing_columns:
+        error_msg = (
+            f"The following columns from '{list_name}'"
+            f"are missing in the DataFrame: {missing_columns}"
+        )
+        raise MissingError(error_msg)
+
+
+def _fail_if_columns_not_found(enusc_filled, column_list):
+    """Raise an error if a column in the list is not found in the DataFrame."""
+    missing_columns = [col for col in column_list if col not in enusc_filled.columns]
+    if missing_columns:
+        error_msg = f"Columns not found in DataFrame: {', '.join(missing_columns)}"
+        raise MissingError(error_msg)

src/.DS_Store

@@ -15,6 +15,8 @@ def create_labels_var_relation(raw_enusc, column_labels):
 
 
 def _concat_labels_var(raw_enusc, column_labels):
+    _fail_if_mismatched_length(raw_enusc, column_labels)
+
     column_names = raw_enusc.columns.tolist()
     column_names = pd.DataFrame(column_names)
     column_names = column_names.iloc[1:]
@@ -31,5 +33,55 @@ def _filter_labels_var(var_labels, relevant_var):
 
 def _rename_variables_eng(labels_var_filt, rename_mapping):
     labels_var_filt["variable_name"] = labels_var_filt["variable_name"].str.lower()
+
+    _fail_if_mismatched_values(labels_var_filt, relevant_var)
+
     labels_renamed = labels_var_filt.replace({"variable_name": rename_mapping})
     return labels_renamed
+
+
+# Error Handling
+
+
+class ShapeError(Exception):
+    """Custom exception for errors in _rename_enusc."""
+
+
+def _fail_if_mismatched_length(raw_enusc, column_labels):
+    """Ensure 'raw_enusc' columns match 'column_labels' length.
+
+    Raises an error if not fulfilled.
+    """
+    expected_length = len(column_labels)
+
+    if len(raw_enusc.columns) - 1 != expected_length:
+        error_msg = (
+            f"Column length mismatch: {len(raw_enusc.columns)} columns in 'raw_enusc', "
+        )
+        f"but {expected_length} labels in 'column_labels'."
+        raise ShapeError(error_msg)
+
+
+class MissingError(Exception):
+    """Custom exception for missing."""
+
+
+def _fail_if_mismatched_values(labels_var_filt, relevant_var):
+    """Raise an error if 'variable_name' and 'relevant_var' are not equal.
+
+    Also isplays the mismatches.
+    """
+    var_labels_values = set(labels_var_filt["variable_name"].values)
+    relevant_var_lower = {x.lower() for x in relevant_var}
+
+    missing_in_var_labels = relevant_var_lower - var_labels_values
+    missing_in_relevant_var = var_labels_values - relevant_var_lower
+
+    mismatched_vars = list(missing_in_var_labels | missing_in_relevant_var)
+
+    if mismatched_vars:
+        error_msg = (
+            "The 'variable_name' values do not match 'relevant_var' values.\n",
+        )
+        f"Missing or mismatched variables: {', '.join(mismatched_vars)}"
+        raise MissingError(error_msg)

src/project_mbb/.DS_Store

@@ -44,5 +44,4 @@ def plot_crime_perception(enusc_clean):
             template="plotly_white",
         )
 
-        # Show figure
         fig.write_image(i)

src/project_mbb/data/.DS_Store

@@ -83,6 +83,14 @@
     "rph_nivel": "education_level",
     "rph_ciuo": "occupation_classification",
     "rph_nse": "socioecon_status",
+    "fact_pers_com": "personal_facts_commununal",
+    "fact_pers_reg": "personal_facts_regional",
+    "fact_pers_regional_102": "personal_facts_regional_102",
+    "fact_hog_com": "household_facts_commununal",
+    "fact_hog_reg": "household_facts_regional",
+    "fact_hog_regional_102": "household_facts_regional_102",
+    "varstrat": "stratum_variables",
+    "conglomerado": "conglomerate",
     # perception of crime increase
     "p_aumento_pais": "crime_increase_perception_nation",
     "p_aumento_com": "crime_increase_perception_commune",
@@ -467,15 +475,15 @@
 
 
 floats = {
-    "fact_pers_com",
-    "fact_pers_reg",
-    "fact_pers_regional_102",
-    "fact_hog_com",
-    "fact_hog_reg",
-    "fact_hog_regional_102",
-    "varstrat",
-    "conglomerado",
     "housing_folio",
+    "personal_facts_commununal",
+    "personal_facts_regional",
+    "personal_facts_regional_102",
+    "household_facts_commununal",
+    "household_facts_regional",
+    "household_facts_regional_102",
+    "stratum_variables",
+    "conglomerate",
 }
 
 integers = {

src/project_mbb/data_management/clean_enusc.py

@@ -24,14 +24,15 @@
 @pytest.fixture
 def enusc_raw():
     enusc_raw_path = BLD / "data" / "ENUSC_raw.csv"
-    enusc_raw = pd.read_csv(enusc_raw_path)
+    enusc_raw = pd.read_csv(enusc_raw_path, engine="pyarrow")
     return enusc_raw
 
 
 @pytest.fixture
 def enusc_raw_short():
     enusc_raw_path = BLD / "data" / "ENUSC_raw.csv"
-    enusc_raw_short = pd.read_csv(enusc_raw_path, nrows=3)
+    enusc_raw = pd.read_csv(enusc_raw_path, engine="pyarrow")
+    enusc_raw_short = enusc_raw.head(3)
     return enusc_raw_short