Change Pytask Functions to CSV due to Space Issues.

Author: MarenBermudezBoeckle

src/project_mbb/data_management/filter_clean.py

@@ -8,7 +8,7 @@
 
 
 def filter_relevant_variables(raw):
-    enusc = pd.read_pickle(raw)
+    enusc = pd.read_csv(raw)
     enusc_filtered = enusc[REL_VAR]
     return enusc_filtered
 

src/project_mbb/data_management/task_data_management.py

@@ -1,4 +1,3 @@
-import pickle
 from pathlib import Path
 
 import pandas as pd
@@ -16,9 +15,9 @@
 
 
 products = {
-    "column_labels": BLD / "data" / "column_labels.pkl",
-    "value_labels": BLD / "data" / "value_labels.pkl",
-    "enucs_raw": BLD / "data" / "ENUSC_raw.pkl",
+    "column_labels": BLD / "data" / "column_labels.csv",
+    "value_labels": BLD / "data" / "value_labels.csv",
+    "enusc_raw": BLD / "data" / "ENUSC_raw.csv",
 }
 
 products_filter = {
@@ -37,32 +36,36 @@ def task_create_data_meta(
 
     df, meta = pyreadstat.read_sav(sav_data)
     df_pd = pd.DataFrame(df)
-    df_pd.to_pickle(products["ENUSC_raw"])
+    df_pd.to_csv(products["enusc_raw"])
 
-    with Path.open(products["value_labels"], "wb") as file:
-        pickle.dump(obj=meta.value_labels, file=file)
+    with Path.open(products["value_labels"], "w") as file:
+        # Convert value_labels (dictionary) into a DataFrame for saving as CSV
+        value_labels_df = pd.DataFrame(meta.value_labels)
+        value_labels_df.to_csv(file, index=False)
 
-    with Path.open(products["column_labels"], "wb") as file:
+    # Save column_labels as a CSV
+    with Path.open(products["column_labels"], "w") as file:
+        # Convert column_labels to DataFrame directly
         column_labels_df = pd.DataFrame(meta.column_labels)
-        pickle.dump(obj=column_labels_df, file=file)
+        column_labels_df.to_csv(file, index=False)
 
 
 def task_create_labels_var_relation(
-    raw_enusc=BLD / "data" / "ENUSC_raw.pkl",
-    column_labels=BLD / "column_labels.pkl",
+    raw_enusc=BLD / "data" / "ENUSC_raw.csv",
+    column_labels=BLD / "data" / "column_labels.csv",
     produces=BLD / "data" / "variable_labels.pkl",
 ):
-    raw = pd.read_pickle(raw_enusc)
+    raw = pd.read_csv(raw_enusc)
     column_names = raw.columns.tolist()
     column_names = pd.DataFrame(column_names)
-    column_labels = pd.read_pickle(column_labels)
+    column_labels = pd.read_csv(column_labels)
     labels_var = pd.concat([column_names, column_labels], axis=1)
     labels_var.columns = ["variable_name", "label"]
     labels_var.to_pickle(produces)
 
 
 def task_filter_variables_and_labels(
-    raw=BLD / "data" / "ENUSC_raw.pkl",
+    raw=BLD / "data" / "ENUSC_raw.csv",
     labels_raw=BLD / "data" / "variable_labels.pkl",
     produces=products_filter,
 ):