Add error handling to analysis part.

Author: MarenBermudezBoeckle

pyproject.toml

@@ -96,6 +96,7 @@ extend-ignore = [
     "ARG001", # Unused function MB
     "ERA001", # commented MB
     "TRY003", # Messages outside exception MB
+    "D415", # First line should end with a period, question mark, or exclamation MB
 ]
 
 [tool.ruff.lint.per-file-ignores]

src/project_mbb/analysis/desc_analysis.py

@@ -1,5 +1,7 @@
 import pandas as pd
 
+from project_mbb.parameters import commune_mapping
+
 pd.options.mode.copy_on_write = True
 pd.options.future.infer_string = True
 pd.options.plotting.backend = "plotly"
@@ -12,6 +14,8 @@ def calculate_perception_general(enusc_clean):
         "crime_increase_perception_neighborhood",
     ]
 
+    _fail_if_missing_columns(enusc_clean, perception_columns)
+
     perception_results = {}
 
     for col in perception_columns:
@@ -26,6 +30,8 @@ def calculate_perception_general(enusc_clean):
 
 
 def calculate_perception_by_commune(enusc_clean):
+    _fail_if_no_total_communes(enusc_clean, commune_mapping)
+
     perception_columns = [
         "crime_increase_perception_nation",
         "crime_increase_perception_commune",
@@ -64,6 +70,8 @@ def calculate_perception_by_commune(enusc_clean):
 
 
 def calculate_perception_by_ses(enusc_clean):
+    _fail_if_ses_not_categorical(enusc_clean)
+
     perception_columns = [
         "crime_increase_perception_nation",
         "crime_increase_perception_commune",
@@ -90,13 +98,44 @@ def calculate_perception_by_ses(enusc_clean):
     perception_results_df = pd.concat(perception_results, axis=0)
 
     perception_results_df = perception_results_df.pivot_table(
-        index=["socioecon_status", "perception"], 
-        columns="response", 
+        index=["socioecon_status", "perception"],
+        columns="response",
         values="percentage",
-        observed=False
+        observed=False,
     )
 
     perception_results_df = perception_results_df.reset_index()
 
     return perception_results_df
 
+
+# Error Handling
+
+
+def _fail_if_missing_columns(enusc_clean, required_columns):
+    """Raise a KeyError if any required columns are missing in 'enusc_clean'."""
+    missing_cols = [col for col in required_columns if col not in enusc_clean.columns]
+    if missing_cols:
+        error_msg = f"Missing expected columns: {', '.join(missing_cols)}"
+        raise KeyError(error_msg)
+
+
+def _fail_if_no_total_communes(enusc_clean, commune_mapping):
+    """Raise a ValueError if the number of unique communes is not 106."""
+    enusc_communes = set(enusc_clean["commune"].unique())
+    communes_total = set(commune_mapping.values())
+
+    difference = communes_total - enusc_communes
+    if difference:
+        error_msg = f"Missing communes: {difference}"
+        raise ValueError(error_msg)
+
+
+def _fail_if_ses_not_categorical(enusc_clean, ses_column="socioecon_status"):
+    """Raise a TypeError if 'socioecon_status' is not categorical."""
+    if not pd.api.types.is_categorical_dtype(enusc_clean[ses_column]):
+        error_msg = (
+            f"Column '{ses_column}' must be of type 'category',"
+            f"but got {enusc_clean[ses_column].dtype}."
+        )
+        raise TypeError(error_msg)

src/project_mbb/analysis/model.py

@@ -2,6 +2,8 @@
 import statsmodels.api as sm
 import statsmodels.formula.api as smf
 
+from project_mbb.parameters import info_sources_mapping, perception_change_mapping
+
 
 def regression_perception_info(enusc_clean):
     enusc_model_pre = _set_category_values(enusc_clean)
@@ -12,17 +14,23 @@ def regression_perception_info(enusc_clean):
 
 
 def _set_category_values(enusc_clean):
-    enusc_model = pd.DataFrame()
-    enusc_model["crime_increase_perception_commune"] = enusc_clean[
+    _fail_if_invalid_categories_perception(enusc_clean, perception_change_mapping)
+    _fail_if_invalid_categories_source(enusc_clean, info_sources_mapping)
+
+    enusc_model_pre = pd.DataFrame()
+    enusc_model_pre["crime_increase_perception_commune"] = enusc_clean[
         "crime_increase_perception_commune"
     ].cat.codes
-    enusc_model["crime_info_source_commune"] = enusc_clean[
+    enusc_model_pre["crime_info_source_commune"] = enusc_clean[
         "crime_info_source_commune"
     ].cat.codes
-    return enusc_model
+    return enusc_model_pre
 
 
 def _set_binary_for_info_source(enusc_model):
+    _fail_if_invalid_category_values(enusc_model, "crime_increase_perception_commune")
+    _fail_if_invalid_category_values(enusc_model, "crime_info_source_commune")
+
     tech_based_codes = {2, 5, 6}
 
     enusc_model["tech_based"] = enusc_model["crime_info_source_commune"].apply(
@@ -32,6 +40,8 @@ def _set_binary_for_info_source(enusc_model):
 
 
 def _drop_missing(enusc_model):
+    _fail_if_invalid_tech_based_values(enusc_model)
+
     enusc_model_clean = enusc_model[
         (enusc_model["crime_increase_perception_commune"] != -1)
         & (enusc_model["crime_info_source_commune"] != -1)
@@ -40,9 +50,136 @@ def _drop_missing(enusc_model):
 
 
 def _run_logistic_regression(enusc_model_clean):
+    _fail_if_missing_values_after_drop(enusc_model_clean)
+
     x = enusc_model_clean[["tech_based"]]
     x = sm.add_constant(x)
     model = smf.mnlogit(
         "crime_increase_perception_commune ~ tech_based", data=enusc_model_clean
     ).fit()
     return model
+
+
+# Error Handling
+
+
+def _fail_if_invalid_categories_perception(enusc_clean, perception_change_mapping):
+    """Raises ValueError if the categories in 'crime_increase_perception_commune'
+
+    are missing.
+    """
+    ignored_cat = 99
+    perception_categories = set(
+        enusc_clean["crime_increase_perception_commune"].cat.categories
+    )
+    expected_categories = set(perception_change_mapping.values())
+
+    perception_categories = {
+        cat
+        for cat in perception_categories
+        if cat != ignored_cat and pd.isna(cat) is False
+    }
+    expected_categories = {
+        cat
+        for cat in expected_categories
+        if cat != ignored_cat and pd.isna(cat) is False
+    }
+
+    missing_categories = expected_categories - perception_categories
+    extra_categories = perception_categories - expected_categories
+
+    if missing_categories or extra_categories:
+        error_msg = (
+            f"Invalid categories for 'crime_increase_perception_commune':\n"
+            f"Missing categories: {missing_categories}\n"
+            f"Extra categories: {extra_categories}"
+        )
+        raise ValueError(error_msg)
+
+
+def _fail_if_invalid_categories_source(enusc_clean, info_source_mapping):
+    """Raises ValueError if the categories in 'crime_info_source_commune'
+
+    are missing.
+    """
+    ignored_cat = 99
+
+    source_categories = set(enusc_clean["crime_info_source_commune"].cat.categories)
+    expected_categories = set(info_sources_mapping.values())
+
+    source_categories = {
+        cat for cat in source_categories if cat != ignored_cat and pd.isna(cat) is False
+    }
+    expected_categories = {
+        cat
+        for cat in expected_categories
+        if cat != ignored_cat and pd.isna(cat) is False
+    }
+
+    missing_categories = expected_categories - source_categories
+    extra_categories = source_categories - expected_categories
+
+    if missing_categories or extra_categories:
+        error_msg = (
+            f"Invalid categories for 'crime_info_source_commune':\n"
+            f"Missing categories: {missing_categories}\n"
+            f"Extra categories: {extra_categories}"
+        )
+        raise ValueError(error_msg)
+
+
+def _fail_if_invalid_category_values(enusc_model, column_name):
+    """Raises an error if any value in the specified column is
+
+    outside the valid range.
+    """
+    if column_name == "crime_increase_perception_commune":
+        valid_range = set(range(5)) | {-1}  # Expecting 0 to 4
+    elif column_name == "crime_info_source_commune":
+        valid_range = set(range(11)) | {-1}  # Expecting 0 to 9
+
+    column_values = enusc_model[column_name]
+    invalid_values = column_values[~column_values.isin(valid_range)].unique()
+
+    if len(invalid_values) > 0:
+        error_msg = (
+            f"Invalid category values in '{column_name}': {invalid_values}."
+            f"Expected values within {valid_range}."
+        )
+        raise ValueError(error_msg)
+
+
+def _fail_if_invalid_tech_based_values(enusc_model):
+    """Raises an error if 'tech_based' column contains values other than -1, 1 or 0."""
+    valid_values = {0, 1, -1}
+    invalid_values = enusc_model[~enusc_model["tech_based"].isin(valid_values)][
+        "tech_based"
+    ].unique()
+
+    if len(invalid_values) > 0:
+        error_msg = (
+            f"Invalid values in 'tech_based' column: {invalid_values}. "
+            " Expected values are 0 and 1."
+        )
+        raise ValueError(error_msg)
+
+
+def _fail_if_missing_values_after_drop(enusc_model_clean):
+    """Raises an error if there are still missing values (i.e., -1 or NaN)
+
+    in the specified columns.
+    """
+    missing_values = enusc_model_clean[
+        (enusc_model_clean["crime_increase_perception_commune"] == -1)
+        | (enusc_model_clean["crime_info_source_commune"] == -1)
+        | enusc_model_clean["crime_increase_perception_commune"].isna()
+        | enusc_model_clean["crime_info_source_commune"].isna()
+    ]
+
+    if not missing_values.empty:
+        error_msg = (
+            f"Missing values after dropping: {missing_values.shape[0]}"
+            "rows contain missing values in 'crime_increase_perception_commune'"
+            " or 'crime_info_source_commune'."
+        )
+        raise ValueError(error_msg)

work_maren/tests_maren.ipynb

@@ -127392,37 +127392,36 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 168,
+   "execution_count": 179,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "<bound method Series.unique of 0         2\n",
-       "1        -1\n",
-       "2        -1\n",
-       "3        -1\n",
-       "4        -1\n",
-       "         ..\n",
-       "146289   -1\n",
-       "146290    2\n",
-       "146291    2\n",
-       "146292   -1\n",
-       "146293   -1\n",
-       "Length: 146294, dtype: int8>"
+       "Index(['Don't know',\n",
+       "       'Information from other non-family people, friends, neighbors, or acquaintances',\n",
+       "       'Information gathered from social media, Facebook, Twitter, TikTok, and similar',\n",
+       "       'Information provided by family members',\n",
+       "       'National newspapers in print and/or electronic form',\n",
+       "       'News programs on television', 'Other',\n",
+       "       'Other tv programs not news: morning shows, documentaries, or special programs',\n",
+       "       'Personal experience',\n",
+       "       'Regional or local newspapers in print and/or electronic form',\n",
+       "       'Through the radio'],\n",
+       "      dtype='string')"
       ]
      },
-     "execution_count": 168,
+     "execution_count": 179,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
     "# Show the categories and their corresponding codes\n",
-    "categories = enusc_clean[\"crime_increase_perception_nation\"].cat.categories\n",
-    "codes =  enusc_clean[\"crime_increase_perception_nation\"].cat.codes\n",
+    "categories = enusc_clean[\"crime_info_source_commune\"].cat.categories\n",
+    "codes =  enusc_clean[\"crime_info_source_commune\"].cat.codes\n",
     "categories\n",
-    "codes\n",
+    "\n",
     "\n",
     "\n",
     "#0 corresponds to 'Decreased'\n",