-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy patheval_all.py
161 lines (146 loc) · 7.39 KB
/
eval_all.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
# Copyright (c) 2010-2024, InterDigital
# All rights reserved.
# See LICENSE under the root folder.
import argparse
import logging
import os
import yaml
from our_utils.parallel_process import parallel_process, Popen
from glob import glob
import json
import pandas as pd
import numpy as np
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S")
logger = logging.getLogger(__name__)
def read_json(file):
with open(file, 'r') as f:
return json.load(f)
def get_dict_results(paths):
x = []
y = []
for _, report in enumerate(paths):
x.append(read_json(report)["color_bits_per_input_point"])
y.append(read_json(report)["y_psnr"])
order = np.argsort(np.array(x))
return np.array(x)[order], np.array(y)[order]
def run_experiment(output_dir, model_name, arch_type,
N_levels, M, enh_channels,attention_channels,
num_scales,scale_min,scale_max,
color_space, squeeze_type, model_dir, pc_name,
pcerror_path, pcerror_cfg_path, input_pc, input_normals, no_stream_redirection=False):
os.makedirs(output_dir, exist_ok=True)
additional_params = []
if input_normals is not None:
additional_params += ["--input_normals", input_normals]
if no_stream_redirection:
f = None
additional_params += ["--no_stream_redirection"]
else:
f = open(os.path.join(output_dir, "experiment.log"), "w")
command = ["python", "eval_model.py",
"--output_dir", output_dir,
"--model_name", model_name,
"--arch_type", arch_type,
"--color_space", color_space,
"--squeeze_type", squeeze_type,
"--model_dir", model_dir,
# "--N_levels", str(N_levels),
"--M", str(M),
"--enh_channels", str(enh_channels),
"--attention_channels", str(attention_channels),
"--num_scales", str(num_scales),
"--scale_min", str(scale_min),
"--scale_max", str(scale_max),
"--pc_name", pc_name,
"--input_pc", input_pc,
"--pcerror_path", pcerror_path,
"--pcerror_cfg_path", pcerror_cfg_path,
]
command.append("--N_levels")
if isinstance(N_levels,int):
command.append(str(N_levels))
else:
for channels in N_levels:
command.append(str(channels))
return Popen(command + additional_params, stdout=f, stderr=f)
if __name__ == "__main__":
parser = argparse.ArgumentParser(prog="eval_all.py", description="Run experiments.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--config", help="Experiments file path.", default="config_files/eval_config.yaml")
parser.add_argument("--num_parallel", help="Number of parallel jobs. Adjust according to CPU memory.", default=2, type=int)
parser.add_argument("--no_stream_redirection", help="Disable stdout and stderr redirection.", default=False, action="store_true")
args = parser.parse_args()
with open(args.config, "r") as f:
experiments = yaml.load(f.read(), Loader=yaml.FullLoader)
keys = ["MODEL_PATH","MPEG_TMC13_DIR", "PCERROR", "MPEG_DATASET_DIR", "EXPERIMENT_DIR", "pcerror_mpeg_mode",
"model_configs"]
MODEL_PATH, MPEG_TMC13_DIR, PCERROR, MPEG_DATASET_DIR, EXPERIMENT_DIR, pcerror_mpeg_mode, model_configs = [experiments[x] for x in keys]
logger.info("Starting our method\"s experiments")
params = []
for experiment in experiments["data"]:
pc_name, cfg_name, input_pc, input_norm = [experiment[x] for x in ["pc_name", "cfg_name", "input_pc", "input_norm"]]
opt_output_dir = os.path.join(EXPERIMENT_DIR, pc_name)
for model_config in model_configs:
model_id = model_config["id"]
lambdas = model_config["lambda"]
color_space = model_config["color_space"]
arch_type = model_config["arch_type"]
N_levels = model_config["N_levels"]
M = model_config["num_filters_M"]
enh_channels = model_config["enh_channels"]
attention_channels = model_config["attention_channels"]
num_scales = model_config["num_scales"]
scale_min = model_config["scale_min"]
scale_max = model_config["scale_max"]
squeeze_type = model_config["squeeze_type"]
for i, lmbda in enumerate(lambdas):
if lmbda == "max":
lmbda=0
lmbda_str = f"{lmbda:.2e}"
checkpoint_id = model_config.get("checkpoint_id", model_id)
model_dir = os.path.join(MODEL_PATH, checkpoint_id, lmbda_str)
current_output_dir = os.path.join(opt_output_dir, model_id, lmbda_str)
pcerror_cfg_path = f"{MPEG_TMC13_DIR}/cfg/{pcerror_mpeg_mode}/{cfg_name}/r06/pcerror.cfg"
input_pc_full = os.path.join(MPEG_DATASET_DIR, input_pc)
if input_norm is not None:
input_norm_full = os.path.join(MPEG_DATASET_DIR, input_norm)
else:
input_norm_full = None
if not os.path.exists(os.path.join(model_dir, "done")):
logger.warning(f"Model training is not finished: skipping {model_dir} for {pc_name}")
else:
if not os.path.exists(os.path.join(current_output_dir, f"report.json")):
params.append((current_output_dir,
model_id,
arch_type,
N_levels[i],
M[i],
enh_channels[i],
attention_channels[i],
num_scales[i],
scale_min[i],
scale_max[i],
color_space[i],
squeeze_type[i],
model_dir,
pc_name,
PCERROR,
pcerror_cfg_path,
input_pc_full,
input_norm_full,
args.no_stream_redirection))
parallel_process(run_experiment, params, args.num_parallel)
for experiment in experiments["data"]:
pc_name, cfg_name, input_pc, input_norm = [experiment[x] for x in ["pc_name", "cfg_name", "input_pc", "input_norm"]]
opt_output_dir = os.path.join(EXPERIMENT_DIR, pc_name)
for model in model_config:
model_id = model_config["id"]
current_output_dir = os.path.join(opt_output_dir, model_id)
report_paths = glob(os.path.join(current_output_dir,"*/report.json"))
bpp, psnr = get_dict_results(report_paths)
df = pd.DataFrame({'bpp': bpp, 'y-psnr': psnr})
df.to_csv(os.path.join(current_output_dir,"results.csv"))
logger.info("Done")