Skip to content

Commit a279a2a

Browse files
XiaozhiChenXiaozhiChen
committed
eval for all classes and offline eval
1 parent 43a8417 commit a279a2a

File tree

3 files changed

+1074
-143
lines changed

3 files changed

+1074
-143
lines changed

Diff for: ‎evaluate_object_3d.cpp

+83-143
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,27 @@ const int32_t N_TESTIMAGES = 7518;
3636
// easy, moderate and hard evaluation level
3737
enum DIFFICULTY{EASY=0, MODERATE=1, HARD=2};
3838

39+
// evaluation metrics: image, ground or 3D
40+
enum METRIC{IMAGE=0, GROUND=1, BOX3D=2};
41+
3942
// evaluation parameter
4043
const int32_t MIN_HEIGHT[3] = {40, 25, 25}; // minimum height for evaluated groundtruth/detections
4144
const int32_t MAX_OCCLUSION[3] = {0, 1, 2}; // maximum occlusion level of the groundtruth used for evaluation
4245
const double MAX_TRUNCATION[3] = {0.15, 0.3, 0.5}; // maximum truncation level of the groundtruth used for evaluation
4346

4447
// evaluated object classes
4548
enum CLASSES{CAR=0, PEDESTRIAN=1, CYCLIST=2};
49+
const int NUM_CLASS = 3;
4650

4751
// parameters varying per class
4852
vector<string> CLASS_NAMES;
49-
double MIN_OVERLAP[3] = {0.7, 0.5, 0.5}; // the minimum overlap required for 2D evaluation on the image plane and the ground plane
53+
// the minimum overlap required for 2D evaluation on the image/ground plane and 3D evaluation
54+
const double MIN_OVERLAP[3][3] = {{0.7, 0.5, 0.5}, {0.5, 0.25, 0.25}, {0.5, 0.25, 0.25}};
5055

5156
// no. of recall steps that should be evaluated (discretized)
5257
const double N_SAMPLE_PTS = 41;
5358

59+
5460
// initialize class names
5561
void initGlobals () {
5662
CLASS_NAMES.push_back("car");
@@ -123,8 +129,8 @@ FUNCTIONS TO LOAD DETECTION AND GROUND TRUTH DATA ONCE, SAVE RESULTS
123129
vector<int32_t> indices;
124130

125131
vector<tDetection> loadDetections(string file_name, bool &compute_aos,
126-
bool &eval_car_image, bool &eval_pedestrian_image, bool &eval_cyclist_image,
127-
bool &eval_car_ground, bool &eval_car_3d, bool &success) {
132+
vector<bool> &eval_image, vector<bool> &eval_ground,
133+
vector<bool> &eval_3d, bool &success) {
128134

129135
// holds all detections (ignored detections are indicated by an index vector
130136
vector<tDetection> detections;
@@ -147,22 +153,21 @@ vector<tDetection> loadDetections(string file_name, bool &compute_aos,
147153
detections.push_back(d);
148154

149155
// orientation=-10 is invalid, AOS is not evaluated if at least one orientation is invalid
150-
if(d.box.alpha==-10)
156+
if(d.box.alpha == -10)
151157
compute_aos = false;
152158

153159
// a class is only evaluated if it is detected at least once
154-
if(!eval_car_image && !strcasecmp(d.box.type.c_str(), "car")) {
155-
eval_car_image = true;
156-
if (!eval_car_ground && d.t1 != -1 && d.t3 != -1) {
157-
eval_car_ground = true;
158-
if (!eval_car_3d && d.t2 != -1)
159-
eval_car_3d = true;
160+
for (int c = 0; c < NUM_CLASS; c++) {
161+
if (!strcasecmp(d.box.type.c_str(), CLASS_NAMES[c].c_str())) {
162+
if (!eval_image[c] && d.box.x1 >= 0)
163+
eval_image[c] = true;
164+
if (!eval_ground[c] && d.t1 != -1000)
165+
eval_ground[c] = true;
166+
if (!eval_3d[c] && d.t2 != -1000)
167+
eval_3d[c] = true;
168+
break;
160169
}
161170
}
162-
if(!eval_pedestrian_image && !strcasecmp(d.box.type.c_str(), "pedestrian"))
163-
eval_pedestrian_image = true;
164-
if(!eval_cyclist_image && !strcasecmp(d.box.type.c_str(), "cyclist"))
165-
eval_cyclist_image = true;
166171
}
167172
}
168173
fclose(fp);
@@ -295,11 +300,6 @@ inline double groundBoxOverlap(tDetection d, tGroundtruth g, int32_t criterion =
295300
intersection(gp, dp, in);
296301
union_(gp, dp, un);
297302

298-
double dt1 = d.t1 - g.t1;
299-
double dt2 = d.t2 - g.t2 + g.h/2;
300-
double dt3 = d.t3 - g.t3;
301-
double dt_sqr = dt1 * dt1 + dt2 * dt2 + dt3 * dt3;
302-
303303
double inter_area = in.empty() ? 0 : area(in.front());
304304
double union_area = area(un.front());
305305
double o;
@@ -453,7 +453,7 @@ tPrData computeStatistics(CLASSES current_class, const vector<tGroundtruth> &gt,
453453
const vector<tDetection> &det, const vector<tGroundtruth> &dc,
454454
const vector<int32_t> &ignored_gt, const vector<int32_t> &ignored_det,
455455
bool compute_fp, double (*boxoverlap)(tDetection, tGroundtruth, int32_t),
456-
bool compute_aos=false, double thresh=0, bool debug=false){
456+
METRIC metric, bool compute_aos=false, double thresh=0, bool debug=false){
457457

458458
tPrData stat = tPrData();
459459
const double NO_DETECTION = -10000000;
@@ -482,7 +482,6 @@ tPrData computeStatistics(CLASSES current_class, const vector<tGroundtruth> &gt,
482482
int32_t det_idx = -1;
483483
double valid_detection = NO_DETECTION;
484484
double max_overlap = 0;
485-
double max_overlap_anyway = 0;
486485

487486
// search for a possible detection
488487
bool assigned_ignored_det = false;
@@ -498,23 +497,22 @@ tPrData computeStatistics(CLASSES current_class, const vector<tGroundtruth> &gt,
498497

499498
// find the maximum score for the candidates and get idx of respective detection
500499
double overlap = boxoverlap(det[j], gt[i], -1);
501-
if (max_overlap_anyway < overlap) max_overlap_anyway = overlap;
502500

503501
// for computing recall thresholds, the candidate with highest score is considered
504-
if(!compute_fp && overlap>MIN_OVERLAP[current_class] && det[j].thresh>valid_detection){
502+
if(!compute_fp && overlap>MIN_OVERLAP[metric][current_class] && det[j].thresh>valid_detection){
505503
det_idx = j;
506504
valid_detection = det[j].thresh;
507505
}
508506

509507
// for computing pr curve values, the candidate with the greatest overlap is considered
510508
// if the greatest overlap is an ignored detection (min_height), the overlapping detection is used
511-
else if(compute_fp && overlap>MIN_OVERLAP[current_class] && (overlap>max_overlap || assigned_ignored_det) && ignored_det[j]==0){
509+
else if(compute_fp && overlap>MIN_OVERLAP[metric][current_class] && (overlap>max_overlap || assigned_ignored_det) && ignored_det[j]==0){
512510
max_overlap = overlap;
513511
det_idx = j;
514512
valid_detection = 1;
515513
assigned_ignored_det = false;
516514
}
517-
else if(compute_fp && overlap>MIN_OVERLAP[current_class] && valid_detection==NO_DETECTION && ignored_det[j]==1){
515+
else if(compute_fp && overlap>MIN_OVERLAP[metric][current_class] && valid_detection==NO_DETECTION && ignored_det[j]==1){
518516
det_idx = j;
519517
valid_detection = 1;
520518
assigned_ignored_det = true;
@@ -576,7 +574,7 @@ tPrData computeStatistics(CLASSES current_class, const vector<tGroundtruth> &gt,
576574

577575
// compute overlap and assign to stuff area, if overlap exceeds class specific value
578576
double overlap = boxoverlap(det[j], dc[i], 0);
579-
if(overlap>MIN_OVERLAP[current_class]){
577+
if(overlap>MIN_OVERLAP[metric][current_class]){
580578
assigned_detection[j] = true;
581579
nstuff++;
582580
}
@@ -619,7 +617,8 @@ bool eval_class (FILE *fp_det, FILE *fp_ori, CLASSES current_class,
619617
const vector< vector<tGroundtruth> > &groundtruth,
620618
const vector< vector<tDetection> > &detections, bool compute_aos,
621619
double (*boxoverlap)(tDetection, tGroundtruth, int32_t),
622-
vector<double> &precision, vector<double> &aos, DIFFICULTY difficulty) {
620+
vector<double> &precision, vector<double> &aos,
621+
DIFFICULTY difficulty, METRIC metric) {
623622
assert(groundtruth.size() == detections.size());
624623

625624
// init
@@ -643,7 +642,7 @@ bool eval_class (FILE *fp_det, FILE *fp_ori, CLASSES current_class,
643642

644643
// compute statistics to get recall values
645644
tPrData pr_tmp = tPrData();
646-
pr_tmp = computeStatistics(current_class, groundtruth[i], detections[i], dc, i_gt, i_det, false, boxoverlap);
645+
pr_tmp = computeStatistics(current_class, groundtruth[i], detections[i], dc, i_gt, i_det, false, boxoverlap, metric);
647646

648647
// add detection scores to vector over all images
649648
for(int32_t j=0; j<pr_tmp.v.size(); j++)
@@ -662,7 +661,8 @@ bool eval_class (FILE *fp_det, FILE *fp_ori, CLASSES current_class,
662661
for(int32_t t=0; t<thresholds.size(); t++){
663662
tPrData tmp = tPrData();
664663
tmp = computeStatistics(current_class, groundtruth[i], detections[i], dontcare[i],
665-
ignored_gt[i], ignored_det[i], true, boxoverlap, compute_aos, thresholds[t], t==38);
664+
ignored_gt[i], ignored_det[i], true, boxoverlap, metric,
665+
compute_aos, thresholds[t], t==38);
666666

667667
// add no. of TP, FP, FN, AOS for current frame to total evaluation for current threshold
668668
pr[t].tp += tmp.tp;
@@ -762,23 +762,6 @@ void saveAndPlotPlots(string dir_name,string file_name,string obj_type,vector<do
762762
system(command);
763763
}
764764

765-
vector<int32_t> getEvalIndices(const string& result_dir) {
766-
767-
DIR* dir;
768-
dirent* entity;
769-
dir = opendir(result_dir.c_str());
770-
if (dir) {
771-
while (entity = readdir(dir)) {
772-
string path(entity->d_name);
773-
int32_t len = path.size();
774-
if (len < 10) continue;
775-
int32_t index = atoi(path.substr(len - 10, 10).c_str());
776-
indices.push_back(index);
777-
}
778-
}
779-
return indices;
780-
}
781-
782765
bool eval(string result_sha,Mail* mail){
783766

784767
// set some global parameters
@@ -798,8 +781,10 @@ bool eval(string result_sha,Mail* mail){
798781

799782
// holds wether orientation similarity shall be computed (might be set to false while loading detections)
800783
// and which labels where provided by this submission
801-
bool compute_aos=true, eval_car_image=false, eval_pedestrian_image=false,
802-
eval_cyclist_image=false, eval_car_ground=false, eval_car_3d=false;
784+
bool compute_aos=true;
785+
vector<bool> eval_image(NUM_CLASS, false);
786+
vector<bool> eval_ground(NUM_CLASS, false);
787+
vector<bool> eval_3d(NUM_CLASS, false);
803788

804789
// for all images read groundtruth and detections
805790
mail->msg("Loading detections...");
@@ -813,8 +798,7 @@ bool eval(string result_sha,Mail* mail){
813798
bool gt_success,det_success;
814799
vector<tGroundtruth> gt = loadGroundtruth(gt_dir + "/" + file_name,gt_success);
815800
vector<tDetection> det = loadDetections(result_dir + "/data/" + file_name,
816-
compute_aos, eval_car_image, eval_pedestrian_image, eval_cyclist_image,
817-
eval_car_ground, eval_car_3d, det_success);
801+
compute_aos, eval_image, eval_ground, eval_3d, det_success);
818802
groundtruth.push_back(gt);
819803
detections.push_back(det);
820804

@@ -833,107 +817,63 @@ bool eval(string result_sha,Mail* mail){
833817
// holds pointers for result files
834818
FILE *fp_det=0, *fp_ori=0;
835819

836-
// eval cars
837-
if(eval_car_image){
838-
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[CAR] + "_detection.txt").c_str(),"w");
839-
if(compute_aos)
840-
fp_ori = fopen((result_dir + "/stats_" + CLASS_NAMES[CAR] + "_orientation.txt").c_str(),"w");
841-
vector<double> precision[3], aos[3];
842-
if( !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, imageBoxOverlap, precision[0],aos[0],EASY)
843-
|| !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, imageBoxOverlap, precision[1],aos[1],MODERATE)
844-
|| !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, imageBoxOverlap, precision[2],aos[2],HARD)){
845-
mail->msg("Car evaluation failed.");
846-
return false;
847-
}
848-
fclose(fp_det);
849-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CAR] + "_detection",CLASS_NAMES[CAR],precision,0);
850-
if(compute_aos){
851-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CAR] + "_orientation",CLASS_NAMES[CAR],aos,1);
852-
fclose(fp_ori);
853-
}
854-
}
855-
856-
// eval pedestrians
857-
if(eval_pedestrian_image){
858-
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[PEDESTRIAN] + "_detection.txt").c_str(),"w");
859-
if(compute_aos)
860-
fp_ori = fopen((result_dir + "/stats_" + CLASS_NAMES[PEDESTRIAN] + "_orientation.txt").c_str(),"w");
861-
vector<double> precision[3], aos[3];
862-
if( !eval_class(fp_det,fp_ori,PEDESTRIAN,groundtruth,detections,compute_aos, imageBoxOverlap, precision[0],aos[0],EASY)
863-
|| !eval_class(fp_det,fp_ori,PEDESTRIAN,groundtruth,detections,compute_aos, imageBoxOverlap, precision[1],aos[1],MODERATE)
864-
|| !eval_class(fp_det,fp_ori,PEDESTRIAN,groundtruth,detections,compute_aos, imageBoxOverlap, precision[2],aos[2],HARD)){
865-
mail->msg("Pedestrian evaluation failed.");
866-
return false;
867-
}
868-
fclose(fp_det);
869-
saveAndPlotPlots(plot_dir,CLASS_NAMES[PEDESTRIAN] + "_detection",CLASS_NAMES[PEDESTRIAN],precision,0);
870-
if(compute_aos){
871-
fclose(fp_ori);
872-
saveAndPlotPlots(plot_dir,CLASS_NAMES[PEDESTRIAN] + "_orientation",CLASS_NAMES[PEDESTRIAN],aos,1);
873-
}
874-
}
875-
876-
// eval cyclists
877-
if(eval_cyclist_image){
878-
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[CYCLIST] + "_detection.txt").c_str(),"w");
879-
if(compute_aos)
880-
fp_ori = fopen((result_dir + "/stats_" + CLASS_NAMES[CYCLIST] + "_orientation.txt").c_str(),"w");
881-
vector<double> precision[3], aos[3];
882-
if( !eval_class(fp_det,fp_ori,CYCLIST,groundtruth,detections,compute_aos, imageBoxOverlap, precision[0],aos[0],EASY)
883-
|| !eval_class(fp_det,fp_ori,CYCLIST,groundtruth,detections,compute_aos, imageBoxOverlap, precision[1],aos[1],MODERATE)
884-
|| !eval_class(fp_det,fp_ori,CYCLIST,groundtruth,detections,compute_aos, imageBoxOverlap, precision[2],aos[2],HARD)){
885-
mail->msg("Cyclist evaluation failed.");
886-
return false;
887-
}
888-
fclose(fp_det);
889-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CYCLIST] + "_detection",CLASS_NAMES[CYCLIST],precision,0);
890-
if(compute_aos){
891-
fclose(fp_ori);
892-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CYCLIST] + "_orientation",CLASS_NAMES[CYCLIST],aos,1);
820+
// eval image 2D bounding boxes
821+
for (int c = 0; c < NUM_CLASS; c++) {
822+
CLASSES cls = (CLASSES)c;
823+
if (eval_image[c]) {
824+
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[c] + "_detection.txt").c_str(), "w");
825+
if(compute_aos)
826+
fp_ori = fopen((result_dir + "/stats_" + CLASS_NAMES[c] + "_orientation.txt").c_str(),"w");
827+
vector<double> precision[3], aos[3];
828+
if( !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, imageBoxOverlap, precision[0], aos[0], EASY, IMAGE)
829+
|| !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, imageBoxOverlap, precision[1], aos[1], MODERATE, IMAGE)
830+
|| !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, imageBoxOverlap, precision[2], aos[2], HARD, IMAGE)) {
831+
mail->msg("%s evaluation failed.", CLASS_NAMES[c].c_str());
832+
return false;
833+
}
834+
fclose(fp_det);
835+
saveAndPlotPlots(plot_dir, CLASS_NAMES[c] + "_detection", CLASS_NAMES[c], precision, 0);
836+
if(compute_aos){
837+
saveAndPlotPlots(plot_dir, CLASS_NAMES[c] + "_orientation", CLASS_NAMES[c], aos, 1);
838+
fclose(fp_ori);
839+
}
893840
}
894841
}
895842

896843
// don't evaluate AOS for birdview boxes and 3D boxes
897844
compute_aos = false;
898-
// eval cars on the ground plane
899-
if(eval_car_ground){
900-
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[CAR] + "_ground_detection.txt").c_str(),"w");
901-
if(compute_aos)
902-
fp_ori = fopen((result_dir + "/stats_" + CLASS_NAMES[CAR] + "_ground_orientation.txt").c_str(),"w");
903-
vector<double> precision[3], aos[3];
904-
if( !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, groundBoxOverlap, precision[0],aos[0],EASY)
905-
|| !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, groundBoxOverlap, precision[1],aos[1],MODERATE)
906-
|| !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, groundBoxOverlap, precision[2],aos[2],HARD)){
907-
mail->msg("Ground plane box evaluation for Car failed.");
908-
return false;
909-
}
910-
fclose(fp_det);
911-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CAR] + "_ground_detection",CLASS_NAMES[CAR],precision,0);
912-
if(compute_aos){
913-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CAR] + "_ground_orientation",CLASS_NAMES[CAR],aos,1);
914-
fclose(fp_ori);
845+
846+
// eval bird's eye view bounding boxes
847+
for (int c = 0; c < NUM_CLASS; c++) {
848+
CLASSES cls = (CLASSES)c;
849+
if (eval_ground[c]) {
850+
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[c] + "_detection_ground.txt").c_str(), "w");
851+
vector<double> precision[3], aos[3];
852+
if( !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, groundBoxOverlap, precision[0], aos[0], EASY, GROUND)
853+
|| !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, groundBoxOverlap, precision[1], aos[1], MODERATE, GROUND)
854+
|| !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, groundBoxOverlap, precision[2], aos[2], HARD, GROUND)) {
855+
mail->msg("%s evaluation failed.", CLASS_NAMES[c].c_str());
856+
return false;
857+
}
858+
fclose(fp_det);
859+
saveAndPlotPlots(plot_dir, CLASS_NAMES[c] + "_detection_ground", CLASS_NAMES[c], precision, 0);
915860
}
916861
}
917862

918-
// Use 0.5 overlap for 3D box evaluation
919-
MIN_OVERLAP[0] = 0.5;
920-
// eval cars in 3D
921-
if(eval_car_3d){
922-
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[CAR] + "_3d_detection.txt").c_str(),"w");
923-
if(compute_aos)
924-
fp_ori = fopen((result_dir + "/stats_" + CLASS_NAMES[CAR] + "_3d_orientation.txt").c_str(),"w");
925-
vector<double> precision[3], aos[3];
926-
if( !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, box3DOverlap, precision[0],aos[0],EASY)
927-
|| !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, box3DOverlap, precision[1],aos[1],MODERATE)
928-
|| !eval_class(fp_det,fp_ori,CAR,groundtruth,detections,compute_aos, box3DOverlap, precision[2],aos[2],HARD)){
929-
mail->msg("3D box evaluation for Car failed.");
930-
return false;
931-
}
932-
fclose(fp_det);
933-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CAR] + "_3d_detection",CLASS_NAMES[CAR],precision,0);
934-
if(compute_aos){
935-
saveAndPlotPlots(plot_dir,CLASS_NAMES[CAR] + "_3d_orientation",CLASS_NAMES[CAR],aos,1);
936-
fclose(fp_ori);
863+
// eval 3D bounding boxes
864+
for (int c = 0; c < NUM_CLASS; c++) {
865+
CLASSES cls = (CLASSES)c;
866+
if (eval_3d[c]) {
867+
fp_det = fopen((result_dir + "/stats_" + CLASS_NAMES[c] + "_detection_3d.txt").c_str(), "w");
868+
vector<double> precision[3], aos[3];
869+
if( !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, box3DOverlap, precision[0], aos[0], EASY, BOX3D)
870+
|| !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, box3DOverlap, precision[1], aos[1], MODERATE, BOX3D)
871+
|| !eval_class(fp_det, fp_ori, cls, groundtruth, detections, compute_aos, box3DOverlap, precision[2], aos[2], HARD, BOX3D)) {
872+
mail->msg("%s evaluation failed.", CLASS_NAMES[c].c_str());
873+
return false;
874+
}
875+
fclose(fp_det);
876+
saveAndPlotPlots(plot_dir, CLASS_NAMES[c] + "_detection_3d", CLASS_NAMES[c], precision, 0);
937877
}
938878
}
939879

0 commit comments

Comments
 (0)