Modified Data-Processing to write release into filenames

Author: finnhacks42

Data-Processing.ipynb

@@ -39,6 +39,7 @@
     "import pickle\n",
     "from clean import *\n",
     "from reed import regex_select\n",
+    "from hilda_config import release_suffix\n",
     "\n",
     "pd.options.display.max_columns=100\n",
     "pd.options.display.max_colwidth=200\n",
@@ -80,7 +81,7 @@
     "correlation_threshold = 0.90\n",
     "redundant_threshold=0.9\n",
     "test = False\n",
-    "release = \"general\" # \"restricted\""
+    "release = \"restricted\" # \"restricted\""
    ]
   },
   {
@@ -191,16 +192,11 @@
     "\n",
     "\n",
     "# read the combined file for the starting wave\n",
-    "release_suffix = {\n",
-    "    \"general\": \"c\",\n",
-    "    \"restricted\": \"u\"\n",
-    "}\n",
-    "\n",
-    "sfx = config.release_suffix[release]\n",
+    "sfx = release_suffix[release]\n",
     "df1, meta1 = pyreadstat.read_sav(f'data/part1/Combined {s}190{sfx}.sav')\n",
     "n0 = len(df1)\n",
     "print(f\"Number of people in initial wave {n0}\")\n",
-    "with open('data/metadata.pkl','wb') as f:\n",
+    "with open(f'data/metadata_{release}.pkl','wb') as f:\n",
     "    pickle.dump(meta1,f)\n",
     "    \n",
     "df1 = filter_participants(df1,min_start_age, already_studying_cols)"
@@ -687,7 +683,7 @@
     "\n",
     "if not test:\n",
     "    X, columns_dropped, r_vals = filter_raw_data(df1.copy(), missing_threshold=missing_threshold,correlation_threshold=correlation_threshold)\n",
-    "    write_data(X, treatment_outcomes, \"all_vars.csv\")"
+    "    write_data(X, treatment_outcomes, f\"all_vars_{release}.csv\")"
    ]
   },
   {
@@ -834,7 +830,7 @@
     "    print(\"\\n\")\n",
     "    return f_selected\n",
     "\n",
-    "def write_selected_featureset(X, columns_dropped, features, tag):\n",
+    "def write_selected_featureset(X, columns_dropped, features, tag, release):\n",
     "    selection = ['xwaveid']\n",
     "    missing = []\n",
     "    for f in features:\n",
@@ -844,7 +840,7 @@
     "            reason = columns_dropped.get(f,\"unkown\")\n",
     "            print(f\"Column {f} not present in X, reason:{reason}\")\n",
     "    \n",
-    "    write_data(X[selection], treatment_outcomes, f\"all_lasso_selected_{tag}.csv\")"
+    "    write_data(X[selection], treatment_outcomes, f\"all_lasso_selected_{tag}_{release}.csv\")"
    ]
   },
   {
@@ -862,10 +858,10 @@
     "\n",
     "    X, columns_dropped, r_vals = filter_raw_data(df1.copy(), missing_threshold=0.99,correlation_threshold=1)\n",
     "\n",
-    "    write_selected_featureset(X, columns_dropped, f10, 10)\n",
-    "    write_selected_featureset(X, columns_dropped, f20, 20)\n",
-    "    write_selected_featureset(X, columns_dropped, f50, 50)\n",
-    "    write_selected_featureset(X, columns_dropped, f100, 100)\n",
+    "    write_selected_featureset(X, columns_dropped, f10, 10, release)\n",
+    "    write_selected_featureset(X, columns_dropped, f20, 20, release)\n",
+    "    write_selected_featureset(X, columns_dropped, f50, 50, release)\n",
+    "    write_selected_featureset(X, columns_dropped, f100, 100, release)\n",
     "    for f in f20:\n",
     "        print(f, meta1.column_names_to_labels.get(f))"
    ]
@@ -900,7 +896,7 @@
     "# X_su = X[selected_cols]\n",
     "# print(\"number of features selected:\",len(selected_cols))\n",
     "# print(\"effective rank:\",effective_rank(Xs[selected]))\n",
-    "# write_data(X_su.reset_index(), treatment_outcomes, \"all_unsupervised_selected.csv\")"
+    "# write_data(X_su.reset_index(), treatment_outcomes, f\"all_unsupervised_selected_{release}.csv\")"
    ]
   }
  ],