Skip to content

Commit 8181718

Browse files
anujanegianujanegi
committed
interim params
1 parent bfb221a commit 8181718

File tree

2 files changed

+109
-97
lines changed

2 files changed

+109
-97
lines changed

‎bsi_zoo/run_benchmark.ipynb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{"cells": [{"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["from joblib import Memory\n", "from pathlib import Path\n", "import numpy as np\n", "import pandas as pd\n", "import time"]}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["from bsi_zoo.benchmark import Benchmark\n", "from bsi_zoo.estimators import (\n", " iterative_L1,\n", " iterative_L2,\n", " iterative_L1_typeII,\n", " iterative_L2_typeII,\n", " gamma_map,\n", " iterative_sqrt,\n", " fake_solver,\n", " eloreta,\n", ")\n", "from bsi_zoo.metrics import euclidean_distance, mse, emd, f1, reconstructed_noise\n", "from bsi_zoo.config import get_leadfield_path"]}, {"cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": ["n_jobs = 30\n", "nruns = 10\n", "spatial_cv = [False, True]\n", "subjects = [\"CC120166\", \"CC120313\", \"CC120264\", \"CC120313\", \"CC120309\"]\n", "metrics = [\n", " euclidean_distance,\n", " mse,\n", " emd,\n", " f1,\n", " reconstructed_noise,\n", "] # list of metric functions here\n", "nnzs = [1, 2, 3, 5]\n", "alpha_SNR = [0.99, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1, 0.01]\n", "# estimator_alphas = [\n", "# 0.01,\n", "# 0.01544452,\n", "# 0.02385332,\n", "# 0.03684031,\n", "# 0.0568981,\n", "# 0.08787639,\n", "# 0.13572088,\n", "# 0.2096144,\n", "# ] # logspaced\n", "estimator_alphas = np.logspace(0, -2, 20)[1:]\n", "memory = Memory(\".\")"]}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4"}}, "nbformat": 4, "nbformat_minor": 2}
1+
{"cells":[{"cell_type":"code","execution_count":75,"metadata":{},"outputs":[],"source":["import os\n","import numpy as np\n","import pandas as pd\n","\n","path = \"/home/anuja/tj/bdsg/BSI-Zoo/bsi_zoo/data/free4\"\n","files = os.listdir(path)\n","benchmark_files_free = [i for i in files if 'spatialCV' not in i]\n","benchmark_files_free_with_spatial_cv = [i for i in files if 'spatialCV' in i]\n","\n","\n","eloreta_path = \"/home/anuja/tj/bdsg/BSI-Zoo/bsi_zoo/data/eloreta\"\n","files = os.listdir(eloreta_path)\n","benchmark_files_eloreta = [i for i in files if 'benchmark' not in i]\n","\n","dfs_free = [pd.read_pickle(f'{path}/{file}') for file in benchmark_files_free]\n","df_results_free = pd.concat(dfs_free)\n","df_results_free['alpha'] = 1-df_results_free['alpha'].astype(float)\n","\n","\n","\n","dfs_free_with_spatial_cv = [pd.read_pickle(f'{path}/{file}') for file in benchmark_files_free_with_spatial_cv]\n","eloreta_dfs = [pd.read_pickle(f'{eloreta_path}/{file}') for file in benchmark_files_eloreta]\n","df_results_free_with_spatial_cv = pd.concat(dfs_free_with_spatial_cv)\n","df_results_eloreta = pd.concat(eloreta_dfs)\n","\n","# join with eloreta\n","df_results_free_with_spatial_cv = pd.concat([df_results_eloreta, df_results_free_with_spatial_cv])\n","df_results_free_with_spatial_cv['alpha'] = 1-df_results_free_with_spatial_cv['alpha'].astype(float)"]},{"cell_type":"code","execution_count":92,"metadata":{},"outputs":[],"source":["# save df_results_free as .mat file\n","import scipy.io\n","Output = {}\n","\n","scipy.io.savemat('free_spatialCV.mat', df_results_free_with_spatial_cv.to_dict('list'))"]},{"cell_type":"code","execution_count":95,"metadata":{},"outputs":[{"data":{"text/plain":["dict_keys(['__header__', '__version__', '__globals__', 'estimator', 'euclidean_distance', 'mse', 'emd', 'f1', 'reconstructed_noise', 'alpha', 'cov_type', 'n_sensors', 'n_sources', 'n_times', 'nnz', 'orientation_type', 'path_to_leadfield', 'extra_params', 'estimator__alpha', 'estimator__alpha_cv', 'error'])"]},"execution_count":95,"metadata":{},"output_type":"execute_result"}],"source":["# load .mat file /home/anuja/tj/bdsg/BSI-Zoo/bsi_zoo/data/sent to stefan/fixed.mat\n","import scipy.io\n","\n","d = scipy.io.loadmat('/home/anuja/tj/bdsg/BSI-Zoo/bsi_zoo/data/sent to stefan/free_spatialCV.mat')\n","d.keys()"]},{"cell_type":"code","execution_count":94,"metadata":{},"outputs":[{"data":{"text/plain":["dict_keys(['__header__', '__version__', '__globals__', 'Unnamed: 0', 'estimator', 'euclidean_distance', 'mse', 'emd', 'f1', 'reconstructed_noise', 'alpha', 'cov_type', 'n_times', 'nnz', 'orientation_type', 'path_to_leadfield', 'extra_params', 'estimator__alpha', 'estimator__alpha_cv', 'error'])"]},"execution_count":94,"metadata":{},"output_type":"execute_result"}],"source":["d = scipy.io.loadmat('/home/anuja/tj/bdsg/BSI-Zoo/bsi_zoo/data/sent to stefan/fixed_spatialCV.mat')\n","d.keys()"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.12"}},"nbformat":4,"nbformat_minor":2}

‎bsi_zoo/run_benchmark.py

Lines changed: 108 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
from bsi_zoo.metrics import euclidean_distance, mse, emd, f1, reconstructed_noise
1919
from bsi_zoo.config import get_leadfield_path
2020

21-
n_jobs = 30
22-
nruns = 10
23-
spatial_cv = [False, True]
24-
subjects = ["CC120264", "CC120313", "CC120309"]
25-
# "CC120166", "CC120313",
21+
n_jobs = 12
22+
nruns = 1
23+
spatial_cv = [True]
24+
# True]
25+
subjects = ["CC120313", "CC120309", "CC120166", "CC120264"]
2626
metrics = [
2727
euclidean_distance,
2828
mse,
@@ -42,18 +42,96 @@
4242
# 0.13572088,
4343
# 0.2096144,
4444
# ] # logspaced
45-
estimator_alphas = np.logspace(0, -2, 20)[1:]
45+
estimator_alphas_I = np.logspace(0, -2, 20)[1:]
46+
estimator_alphas_II = [0.0001, 0.001, 0.01, 0.1, 1, 10, 100, 1000, 10000, 100000]
4647
memory = Memory(".")
4748

4849
for do_spatial_cv in spatial_cv:
4950
for subject in subjects:
50-
"""Fixed orientation parameters for the benchmark"""
51+
52+
# """Fixed orientation parameters for the benchmark"""
5153

52-
orientation_type = "fixed"
54+
# orientation_type = "fixed"
55+
# data_args_I = {
56+
# # "n_sensors": [50],
57+
# "n_times": [10],
58+
# # "n_sources": [200],
59+
# "nnz": nnzs,
60+
# "cov_type": ["diag"],
61+
# "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
62+
# "orientation_type": [orientation_type],
63+
# "alpha": alpha_SNR, # this is actually SNR
64+
# }
65+
66+
# data_args_II = {
67+
# # "n_sensors": [50],
68+
# "n_times": [10],
69+
# # "n_sources": [200],
70+
# "nnz": nnzs,
71+
# "cov_type": ["full"],
72+
# "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
73+
# "orientation_type": [orientation_type],
74+
# "alpha": alpha_SNR, # this is actually SNR
75+
# }
76+
77+
# estimators = [
78+
# # (fake_solver, data_args_I, {"alpha": estimator_alphas_I}, {}),
79+
# (eloreta, data_args_I, {"alpha": estimator_alphas_II}, {}),
80+
# # (iterative_L1, data_args_I, {"alpha": estimator_alphas_I}, {}),
81+
# # (iterative_L2, data_args_I, {"alpha": estimator_alphas_I}, {}),
82+
# # (iterative_sqrt, data_args_I, {"alpha": estimator_alphas_I}, {}),
83+
# # (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas_I}, {}),
84+
# # (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas_I}, {}),
85+
# # (gamma_map, data_args_II, {"alpha": estimator_alphas_I}, {"update_mode": 1}),
86+
# (gamma_map, data_args_II, {"alpha": estimator_alphas_II}, {"update_mode": 2}),
87+
# # (gamma_map, data_args_II, {"alpha": estimator_alphas_I}, {"update_mode": 3}),
88+
# ]
89+
90+
# df_results = []
91+
# for estimator, data_args, estimator_args, estimator_extra_params in estimators:
92+
# benchmark = Benchmark(
93+
# estimator,
94+
# subject,
95+
# metrics,
96+
# data_args,
97+
# estimator_args,
98+
# random_state=42,
99+
# memory=memory,
100+
# n_jobs=n_jobs,
101+
# do_spatial_cv=do_spatial_cv,
102+
# estimator_extra_params=estimator_extra_params,
103+
# )
104+
# results = benchmark.run(nruns=nruns)
105+
# df_results.append(results)
106+
# # save results
107+
# data_path = Path("bsi_zoo/data/updated_alpha_grid")
108+
# data_path.mkdir(exist_ok=True)
109+
# if do_spatial_cv:
110+
# FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
111+
# else:
112+
# FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
113+
# results.to_pickle(data_path / FILE_NAME)
114+
115+
116+
# df_results = pd.concat(df_results, axis=0)
117+
118+
# data_path = Path("bsi_zoo/data/ramen")
119+
# data_path.mkdir(exist_ok=True)
120+
# if do_spatial_cv:
121+
# FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
122+
# else:
123+
# FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
124+
# df_results.to_pickle(data_path / FILE_NAME)
125+
126+
# print(df_results)
127+
128+
""" Free orientation parameters for the benchmark """
129+
130+
orientation_type = "free"
53131
data_args_I = {
54-
# "n_sensors": [50],
132+
"n_sensors": [50],
55133
"n_times": [10],
56-
# "n_sources": [200],
134+
"n_sources": [200],
57135
"nnz": nnzs,
58136
"cov_type": ["diag"],
59137
"path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
@@ -62,9 +140,9 @@
62140
}
63141

64142
data_args_II = {
65-
# "n_sensors": [50],
143+
"n_sensors": [50],
66144
"n_times": [10],
67-
# "n_sources": [200],
145+
"n_sources": [200],
68146
"nnz": nnzs,
69147
"cov_type": ["full"],
70148
"path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
@@ -73,16 +151,16 @@
73151
}
74152

75153
estimators = [
76-
(fake_solver, data_args_I, {"alpha": estimator_alphas}, {}),
77-
# (eloreta, data_args_I, {"alpha": estimator_alphas}, {}),
78-
# (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}),
79-
# (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}),
80-
# (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}),
81-
# (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
82-
# (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
83-
# (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}),
84-
# (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}),
85-
# (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}),
154+
# (fake_solver, data_args_I, {"alpha": estimator_alphas_I}, {}),
155+
(eloreta, data_args_I, {"alpha": estimator_alphas_II}, {}),
156+
# (iterative_L1, data_args_I, {"alpha": estimator_alphas_I}, {}),
157+
# (iterative_L2, data_args_I, {"alpha": estimator_alphas_I}, {}),
158+
# (iterative_sqrt, data_args_I, {"alpha": estimator_alphas_I}, {}),
159+
# (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas_I}, {}),
160+
# (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas_I}, {}),
161+
# (gamma_map, data_args_II, {"alpha": estimator_alphas_I}, {"update_mode": 1}),
162+
# (gamma_map, data_args_II, {"alpha": estimator_alphas_II}, {"update_mode": 2}),
163+
# (gamma_map, data_args_II, {"alpha": estimator_alphas_I}, {"update_mode": 3}),
86164
]
87165

88166
df_results = []
@@ -102,14 +180,19 @@
102180
results = benchmark.run(nruns=nruns)
103181
df_results.append(results)
104182
# save results
105-
data_path = Path("bsi_zoo/data")
183+
data_path = Path("bsi_zoo/data/eloreta")
106184
data_path.mkdir(exist_ok=True)
107-
FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
185+
186+
if do_spatial_cv:
187+
FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
188+
else:
189+
FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
108190
results.to_pickle(data_path / FILE_NAME)
191+
print(results)
109192

110193
df_results = pd.concat(df_results, axis=0)
111194

112-
data_path = Path("bsi_zoo/data")
195+
data_path = Path("bsi_zoo/data/eloreta")
113196
data_path.mkdir(exist_ok=True)
114197
if do_spatial_cv:
115198
FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
@@ -119,74 +202,3 @@
119202

120203
print(df_results)
121204

122-
# """ Free orientation parameters for the benchmark """
123-
124-
# orientation_type = "free"
125-
# data_args_I = {
126-
# "n_sensors": [50],
127-
# "n_times": [10],
128-
# "n_sources": [200],
129-
# "nnz": nnzs,
130-
# "cov_type": ["diag"],
131-
# "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
132-
# "orientation_type": [orientation_type],
133-
# "alpha": alpha_SNR, # this is actually SNR
134-
# }
135-
136-
# data_args_II = {
137-
# "n_sensors": [50],
138-
# "n_times": [10],
139-
# "n_sources": [200],
140-
# "nnz": nnzs,
141-
# "cov_type": ["full"],
142-
# "path_to_leadfield": [get_leadfield_path(subject, type=orientation_type)],
143-
# "orientation_type": [orientation_type],
144-
# "alpha": alpha_SNR, # this is actually SNR
145-
# }
146-
147-
# estimators = [
148-
# (fake_solver, data_args_I, {"alpha": estimator_alphas}, {}),
149-
# (eloreta, data_args_I, {"alpha": estimator_alphas}, {}),
150-
# (iterative_L1, data_args_I, {"alpha": estimator_alphas}, {}),
151-
# (iterative_L2, data_args_I, {"alpha": estimator_alphas}, {}),
152-
# (iterative_sqrt, data_args_I, {"alpha": estimator_alphas}, {}),
153-
# (iterative_L1_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
154-
# (iterative_L2_typeII, data_args_II, {"alpha": estimator_alphas}, {}),
155-
# # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 1}),
156-
# (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 2}),
157-
# # (gamma_map, data_args_II, {"alpha": estimator_alphas}, {"update_mode": 3}),
158-
# ]
159-
160-
# df_results = []
161-
# for estimator, data_args, estimator_args, estimator_extra_params in estimators:
162-
# benchmark = Benchmark(
163-
# estimator,
164-
# subject,
165-
# metrics,
166-
# data_args,
167-
# estimator_args,
168-
# random_state=42,
169-
# memory=memory,
170-
# n_jobs=n_jobs,
171-
# do_spatial_cv=do_spatial_cv,
172-
# estimator_extra_params=estimator_extra_params,
173-
# )
174-
# results = benchmark.run(nruns=nruns)
175-
# df_results.append(results)
176-
# # save results
177-
# data_path = Path("bsi_zoo/data")
178-
# data_path.mkdir(exist_ok=True)
179-
# FILE_NAME = f"{estimator}_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
180-
# results.to_pickle(data_path / FILE_NAME)
181-
182-
# df_results = pd.concat(df_results, axis=0)
183-
184-
# data_path = Path("bsi_zoo/data")
185-
# data_path.mkdir(exist_ok=True)
186-
# if do_spatial_cv:
187-
# FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_spatialCV_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
188-
# else:
189-
# FILE_NAME = f"benchmark_data_{subject}_{data_args['orientation_type'][0]}_{time.strftime('%b-%d-%Y_%H%M', time.localtime())}.pkl"
190-
# df_results.to_pickle(data_path / FILE_NAME)
191-
192-
# print(df_results)

0 commit comments

Comments
 (0)