Create rename and filter enusc function and start mapping in ntb.

Author: MarenBermudezBoeckle

.DS_Store

@@ -15,94 +15,3 @@
 
 DOCUMENTS = ROOT.joinpath("documents").resolve()
 
-REL_VAR = [
-    "0" "enc_rpc",
-    "rph_ID",
-    "idhogar",
-    "enc_idr",
-    "enc_region",
-    "rph_nivel",
-    "rph_ciuo",
-    "rph_nse",
-    "rph_parentesco",
-    "rph_edad",
-    "rph_sexo",
-    "rph_idgen",
-    "Kish",
-    "Hogar_Kish",
-    "ANTIG_SECTOR",
-    "Fact_Pers_Com",
-    "Fact_Pers_Reg",
-    "Fact_Pers_Regional_102",
-    "Fact_Hog_Com",
-    "Fact_Hog_Reg",
-    "Fact_Hog_Regional_102",
-    "VarStrat",
-    "Conglomerado",
-    "P_AUMENTO_PAIS",
-    "P_AUMENTO_COM",
-    "P_AUMENTO_BARRIO",
-    "P_FUENTE_INFO_PAIS_1",
-    "P_FUENTE_INFO_COM_1",
-    "P_INSEG_LUGARES_1",
-    "P_INSEG_LUGARES_2",
-    "P_INSEG_OSCURO_1",
-    "P_INSEG_DIA_1",
-    "P_INSEG_OSCURO_2",
-    "P_INSEG_DIA_2",
-    "P_DESORDENES_3",
-    "P_INCIVILIDADES_6",
-    "P_EXPOS_DELITO",
-    "P_DELITO_PRONOSTICO__1",
-    "P_DELITO_PRONOSTICO__2",
-    "P_DELITO_PRONOSTICO__3",
-    "P_DELITO_PRONOSTICO__5",
-    "P_DELITO_PRONOSTICO__6",
-    "P_DELITO_PRONOSTICO__77",
-    "P_DELITO_PRONOSTICO__88",
-    "P_DELITO_PRONOSTICO__99",
-    "P_MOD_ACTIVIDADES_1",
-    "P_MOD_ACTIVIDADES_3",
-    "P_MOD_ACTIVIDADES_8",
-    "P_MOD_ACTIVIDADES_7",
-    "P_MOD_ACTIVIDADES_11",
-    "EV_CONOCE_CCH",
-    "EV_CONFIA_CCH",
-    "EV_CONOCE_PDI",
-    "EV_CONFIA_PDI",
-    "EV_CONOCE_FMP",
-    "EV_CONFIA_FMP",
-    "HUR_QUIEN__0",
-    "HUR_QUIEN__1",
-    "HUR_QUIEN__2",
-    "HUR_QUIEN__3",
-    "HUR_QUIEN__4",
-    "HUR_QUIEN__5",
-    "HUR_QUIEN__6",
-    "HUR_QUIEN__7",
-    "HUR_MES",
-    "HUR_DONDE_ESP",
-    "HUR_DONDE_COMUNA",
-    "HUR_DONDE_REGION",
-    "HUR_DENUNCIA",
-    "HUR_DENUNCIA_COMO",
-    "HUR_SATISF_DEN",
-    "HUR_ACCION_POLICIAL",
-    "HUR_CONTACTO_MP",
-    "HUR_MOTIV_NO_DEN",
-    "SCREEN_ROB_HUR",
-    "HUR_DENUNCIAS",
-    "PAD",
-    "PADC",
-    "PADB",
-    "PCOS",
-    "VH_DV",
-    "VP_DV",
-    "DEN_VHDV",
-    "COSC_DV",
-    "DEN_ROBOS",
-    "VH_ROBOS",
-    "RDV",
-    "RDV_INTENTO",
-    "HUR",
-]

src/project_mbb/.DS_Store

@@ -1,6 +1,8 @@
 import pandas as pd
 
-from project_mbb.config import REL_VAR
+
+from project_mbb.parameters import rename_mapping, region_mapping, relevant_var
+
 
 pd.options.mode.copy_on_write = True
 pd.options.future.infer_string = True
@@ -27,20 +29,33 @@ def _concat_labels_var(raw_enusc, labels_raw):
 
 
 def _filter_labels_var(var_labels):
-    var_labels_filt = var_labels[var_labels["variable_name"].isin(REL_VAR)]
+    var_labels_filt = var_labels[var_labels["variable_name"].isin(relevant_var)]
     return var_labels_filt
 
 
-# def _no_ready_clean_enusc(
-#   raw=BLD / "data" / "ENUSC_raw.csv",
-#  labels_raw=BLD / "data" / "variable_labels.pkl",
-# produces=products_filter,
-# ):
-#   enusc_filtered = _filter_relevant_variables(raw)
-#  enusc_filtered.to_pickle(products_filter["enusc_filt"])
 
 
-# def _no_ready_filter_relevant_variables(raw):
-#  enusc = pd.read_csv(raw)
-# enusc_filtered = enusc[REL_VAR]
-# return enusc_filtered
+
+def clean_enusc(raw_enusc):
+    enusc_filtered = _filter_enusc(raw_enusc)
+    enusc_category = _map_category_region(enusc_filtered)
+    #enusc_renamed = _rename_enusc(enusc_filtered)
+    return enusc_category
+
+def _filter_enusc(raw_enusc):
+    enusc = pd.read_csv(raw_enusc)
+    enusc_filtered = enusc[relevant_var]
+    return enusc_filtered
+
+
+def _rename_enusc(enusc_lower):
+    enusc_lower.columns = enusc_lower.columns.str.lower()
+    enusc_renamed = enusc_lower.rename(columns=rename_mapping)
+    return enusc_renamed
+
+def _map_category_region(enusc_filtered):
+
+    return enusc_filtered
+
+
+

src/project_mbb/config.py

@@ -4,7 +4,9 @@
 import pyreadstat
 
 from project_mbb.config import BLD, DATA
-from project_mbb.data_management.filter_clean import create_labels_var_relation
+from project_mbb.data_management.filter_clean import create_labels_var_relation, clean_enusc
+
+
 
 pd.options.mode.copy_on_write = True
 pd.options.future.infer_string = True
@@ -22,7 +24,7 @@
     "labels_filt": BLD / "data" / "variable_labels_filtered.pkl",
 }
 
-
+"""
 def task_create_data_meta(
     raw_data=DATA / "base-usuario-20-enusc-2023.sav", produces=products
 ):
@@ -40,6 +42,7 @@ def task_create_data_meta(
         # Convert column_labels to DataFrame directly
         column_labels_df = pd.DataFrame(meta.column_labels)
         column_labels_df.to_csv(file, index=False)
+"""
 
 
 def task_clean_labels(
@@ -50,17 +53,10 @@ def task_clean_labels(
     labels_var = create_labels_var_relation(raw, labels_raw)
     labels_var.to_pickle(produces)
 
+def task_clean_enusc(raw=BLD / "data" / "ENUSC_raw.csv",
+    labels_raw=BLD / "data" / "column_labels.csv",
+    produces=BLD / "data" / "enusc_test.pkl",):
+    data_test = clean_enusc(raw)
+    data_test.to_pickle(produces)
 
-# def no_ready_task_clean_data(raw=BLD / "data" / "ENUSC_raw.csv",
-#   labels_raw=BLD / "data" / "variable_labels.pkl",
-#  produces=products_filter):
 
-# def no_ready_filter_variables_and_labels(
-#   raw=BLD / "data" / "ENUSC_raw.csv",
-#  labels_raw=BLD / "data" / "variable_labels.pkl",
-# produces=products_filter,
-# ):
-# enusc_filtered = filter_relevant_variables(raw)
-# enusc_filtered.to_pickle(products_filter["enusc_filt"])
-# var_labels = filter_labels_var(labels_raw)
-# var_labels.to_pickle(products_filter["labels_filt"])