Update of var-label relation functions for pytask.

Author: MarenBermudezBoeckle

.DS_Store

@@ -93,7 +93,8 @@ extend-ignore = [
     "RET504", # Don't force to calculate upon return
     "S101",  # Use of `assert` detected.
     "S301",  # pickle module is unsafe
-    "ARG001", # Unused function
+    "ARG001", # Unused function MB
+    "ERA001", # commented MB
 ]
 
 [tool.ruff.lint.per-file-ignores]

pyproject.toml

@@ -16,7 +16,7 @@
 DOCUMENTS = ROOT.joinpath("documents").resolve()
 
 REL_VAR = [
-    "enc_rpc",
+    "0" "enc_rpc",
     "rph_ID",
     "idhogar",
     "enc_idr",

src/.DS_Store

@@ -7,13 +7,40 @@
 pd.options.plotting.backend = "plotly"
 
 
-def filter_relevant_variables(raw):
-    enusc = pd.read_csv(raw)
-    enusc_filtered = enusc[REL_VAR]
-    return enusc_filtered
+def create_labels_var_relation(raw_enusc, labels_raw):
+    labels_var = _concat_labels_var(raw_enusc, labels_raw)
+    labels_var_filt = _filter_labels_var(labels_var)
+    return labels_var_filt
 
 
-def filter_labels_var(labels_raw):
-    var_labels = pd.read_pickle(labels_raw)
-    var_labels = var_labels[var_labels["variable_name"].isin(REL_VAR)]
-    return var_labels
+def _concat_labels_var(raw_enusc, labels_raw):
+    raw = pd.read_csv(raw_enusc)
+    column_names = raw.columns.tolist()
+    column_names = pd.DataFrame(column_names)
+    column_names = column_names.iloc[1:]
+    column_labels = pd.read_csv(labels_raw)
+    column_labels.index = column_names.index
+    labels_var = pd.concat([column_names, column_labels], axis=1)
+    labels_var.columns = ["variable_name", "label"]
+    # labels_var = pd.DataFrame(labels_var)
+    return labels_var
+
+
+def _filter_labels_var(var_labels):
+    var_labels_filt = var_labels[var_labels["variable_name"].isin(REL_VAR)]
+    return var_labels_filt
+
+
+# def _no_ready_clean_enusc(
+#   raw=BLD / "data" / "ENUSC_raw.csv",
+#  labels_raw=BLD / "data" / "variable_labels.pkl",
+# produces=products_filter,
+# ):
+#   enusc_filtered = _filter_relevant_variables(raw)
+#  enusc_filtered.to_pickle(products_filter["enusc_filt"])
+
+
+# def _no_ready_filter_relevant_variables(raw):
+#  enusc = pd.read_csv(raw)
+# enusc_filtered = enusc[REL_VAR]
+# return enusc_filtered

src/project_mbb/.DS_Store

@@ -4,10 +4,7 @@
 import pyreadstat
 
 from project_mbb.config import BLD, DATA
-from project_mbb.data_management.filter_clean import (
-    filter_labels_var,
-    filter_relevant_variables,
-)
+from project_mbb.data_management.filter_clean import create_labels_var_relation
 
 pd.options.mode.copy_on_write = True
 pd.options.future.infer_string = True
@@ -27,14 +24,9 @@
 
 
 def task_create_data_meta(
-    sav_data=DATA / "base-usuario-20-enusc-2023.sav", produces=products
+    raw_data=DATA / "base-usuario-20-enusc-2023.sav", produces=products
 ):
-    output_dir = BLD / "data"
-
-    if not output_dir.exists():
-        output_dir.mkdir()
-
-    df, meta = pyreadstat.read_sav(sav_data)
+    df, meta = pyreadstat.read_sav(raw_data)
     df_pd = pd.DataFrame(df)
     df_pd.to_csv(products["enusc_raw"])
 
@@ -50,26 +42,25 @@ def task_create_data_meta(
         column_labels_df.to_csv(file, index=False)
 
 
-def task_create_labels_var_relation(
-    raw_enusc=BLD / "data" / "ENUSC_raw.csv",
-    column_labels=BLD / "data" / "column_labels.csv",
+def task_clean_labels(
+    raw=BLD / "data" / "ENUSC_raw.csv",
+    labels_raw=BLD / "data" / "column_labels.csv",
     produces=BLD / "data" / "variable_labels.pkl",
 ):
-    raw = pd.read_csv(raw_enusc)
-    column_names = raw.columns.tolist()
-    column_names = pd.DataFrame(column_names)
-    column_labels = pd.read_csv(column_labels)
-    labels_var = pd.concat([column_names, column_labels], axis=1)
-    labels_var.columns = ["variable_name", "label"]
+    labels_var = create_labels_var_relation(raw, labels_raw)
     labels_var.to_pickle(produces)
 
 
-def task_filter_variables_and_labels(
-    raw=BLD / "data" / "ENUSC_raw.csv",
-    labels_raw=BLD / "data" / "variable_labels.pkl",
-    produces=products_filter,
-):
-    enusc_filtered = filter_relevant_variables(raw)
-    enusc_filtered.to_pickle(products_filter["enusc_filt"])
-    var_labels = filter_labels_var(labels_raw)
-    var_labels.to_pickle(products_filter["labels_filt"])
+# def no_ready_task_clean_data(raw=BLD / "data" / "ENUSC_raw.csv",
+#   labels_raw=BLD / "data" / "variable_labels.pkl",
+#  produces=products_filter):
+
+# def no_ready_filter_variables_and_labels(
+#   raw=BLD / "data" / "ENUSC_raw.csv",
+#  labels_raw=BLD / "data" / "variable_labels.pkl",
+# produces=products_filter,
+# ):
+# enusc_filtered = filter_relevant_variables(raw)
+# enusc_filtered.to_pickle(products_filter["enusc_filt"])
+# var_labels = filter_labels_var(labels_raw)
+# var_labels.to_pickle(products_filter["labels_filt"])

src/project_mbb/config.py

@@ -0,0 +1 @@
+- test the function for merge label and variable to see if there is a label for each person and check some label var relations to see if they are correct in different parts of the code.