Unify Filter and Label Functions in Pytask Format.

Author: MarenBermudezBoeckle

.DS_Store

@@ -15,7 +15,6 @@
 
 DOCUMENTS = ROOT.joinpath("documents").resolve()
 
-
 REL_VAR = [
     "enc_rpc",
     "rph_ID",
@@ -93,9 +92,6 @@
     "HUR_MOTIV_NO_DEN",
     "SCREEN_ROB_HUR",
     "HUR_DENUNCIAS",
-]
-
-AUX_VAR = [
     "PAD",
     "PADC",
     "PADB",

src/.DS_Store

@@ -0,0 +1,19 @@
+import pandas as pd
+
+from project_mbb.config import REL_VAR
+
+pd.options.mode.copy_on_write = True
+pd.options.future.infer_string = True
+pd.options.plotting.backend = "plotly"
+
+
+def filter_relevant_variables(raw):
+    enusc = pd.read_pickle(raw)
+    enusc_filtered = enusc[REL_VAR]
+    return enusc_filtered
+
+
+def filter_labels_var(labels_raw):
+    var_labels = pd.read_pickle(labels_raw)
+    var_labels = var_labels[var_labels["variable_name"].isin(REL_VAR)]
+    return var_labels

src/project_mbb/.DS_Store

@@ -5,6 +5,10 @@
 import pyreadstat
 
 from project_mbb.config import BLD, DATA
+from project_mbb.data_management.filter_clean import (
+    filter_labels_var,
+    filter_relevant_variables,
+)
 
 pd.options.mode.copy_on_write = True
 pd.options.future.infer_string = True
@@ -14,11 +18,16 @@
 products = {
     "column_labels": BLD / "data" / "column_labels.pkl",
     "value_labels": BLD / "data" / "value_labels.pkl",
-    "ENUSC_raw": BLD / "data" / "ENUSC_raw.pkl",
+    "enucs_raw": BLD / "data" / "ENUSC_raw.pkl",
+}
+
+products_filter = {
+    "enusc_filt": BLD / "data" / "enusc_filtered.pkl",
+    "labels_filt": BLD / "data" / "variable_labels_filtered.pkl",
 }
 
 
-def task_create_data_info(
+def task_create_data_meta(
     sav_data=DATA / "base-usuario-20-enusc-2023.sav", produces=products
 ):
     output_dir = BLD / "data"
@@ -36,3 +45,28 @@ def task_create_data_info(
     with Path.open(products["column_labels"], "wb") as file:
         column_labels_df = pd.DataFrame(meta.column_labels)
         pickle.dump(obj=column_labels_df, file=file)
+
+
+def task_create_labels_var_relation(
+    raw_enusc=BLD / "data" / "ENUSC_raw.pkl",
+    column_labels=BLD / "column_labels.pkl",
+    produces=BLD / "data" / "variable_labels.pkl",
+):
+    raw = pd.read_pickle(raw_enusc)
+    column_names = raw.columns.tolist()
+    column_names = pd.DataFrame(column_names)
+    column_labels = pd.read_pickle(column_labels)
+    labels_var = pd.concat([column_names, column_labels], axis=1)
+    labels_var.columns = ["variable_name", "label"]
+    labels_var.to_pickle(produces)
+
+
+def task_filter_variables_and_labels(
+    raw=BLD / "data" / "ENUSC_raw.pkl",
+    labels_raw=BLD / "data" / "variable_labels.pkl",
+    produces=products_filter,
+):
+    enusc_filtered = filter_relevant_variables(raw)
+    enusc_filtered.to_pickle(products_filter["enusc_filt"])
+    var_labels = filter_labels_var(labels_raw)
+    var_labels.to_pickle(products_filter["labels_filt"])