Added (rather simplistic) k-fold crossval to see how that goes

previtus · previtus · commit 2c4f0a55d5dd · 2019-05-16T20:27:14.000+02:00
diff --git a/Dataset.py b/Dataset.py
@@ -1,5 +1,7 @@
 import DataLoader, DataPreprocesser, Debugger
 import DatasetInstance_OurAerial, DatasetInstance_ONERA
+import numpy as np
+
 
 class Dataset(object):
     """
@@ -42,12 +44,24 @@ def init_from_stable_datasets(self):
         print("Dataset loaded with", len(self.data[0]), "images.")
 
         # Shuffle
-        self.data = self.shuffle_thyself(self.data)
+        #self.data = self.shuffle_thyself(self.data)
 
         # Split into training, validation and test:
-        self.train, self.val, self.test = self.datasetInstance.split_train_val_test(self.data)
-        self.train_paths, self.val_paths, self.test_paths = self.datasetInstance.split_train_val_test(self.paths)
+
+        K = self.settings.TestDataset_K_Folds
+        test_fold = self.settings.TestDataset_Fold_Index
+        print("K-Fold crossval: [",test_fold,"from",K,"]")
+        self.train, self.val, self.test = self.datasetInstance.split_train_val_test_KFOLDCROSSVAL(self.data, test_fold=test_fold, K=K)
+        self.paths = np.asarray(self.paths)
+        self.train_paths, self.val_paths, self.test_paths = self.datasetInstance.split_train_val_test_KFOLDCROSSVAL(self.paths, test_fold=test_fold, K=K)
+
         print("Has ", len(self.train[0]), "train, ", len(self.val[0]), "val, ", len(self.test[0]), "test, ")
+        #print("Has ", len(self.train_paths[0]), "train_paths, ", len(self.val_paths[0]), "val_paths, ", len(self.test_paths[0]), "test_paths, ")
+
+        #print("Revert...")
+        #self.train, self.val, self.test = self.datasetInstance.split_train_val_test(self.data)
+        #self.train_paths, self.val_paths, self.test_paths = self.datasetInstance.split_train_val_test(self.paths)
+        #print("Has ", len(self.train[0]), "train, ", len(self.val[0]), "val, ", len(self.test[0]), "test, ")
 
         # preprocess the dataset
         self.train, self.val, self.test = self.dataPreprocesser.process_dataset(self.train, self.val, self.test)
diff --git a/DatasetInstance_OurAerial.py b/DatasetInstance_OurAerial.py
@@ -147,6 +147,55 @@ def __init__(self, settings, dataLoader, variant = "256"):
             self.DEBUG_TURN_OFF_BALANCING = True
 
 
+
+    def split_train_val_test_KFOLDCROSSVAL(self, data, test_fold = 0, K = 4):
+        lefts, rights, labels = data
+
+        # now we would like the val jump around the dataset (and the rest can be still separated into train - val
+
+        # split [0 - end] into K folds, one as a test the rest as a train (alt. val, but that can be 0)
+        N = len(lefts)
+        jump_by = int(N / K)
+
+        test_L = np.empty(((0,)+lefts.shape[1:]), lefts.dtype)
+        train_L = np.empty(((0,)+lefts.shape[1:]), lefts.dtype)
+        test_R = np.empty(((0,) + rights.shape[1:]), rights.dtype)
+        train_R = np.empty(((0,) + rights.shape[1:]), rights.dtype)
+        test_V = np.empty(((0,) + labels.shape[1:]), labels.dtype)
+        train_V = np.empty(((0,) + labels.shape[1:]), labels.dtype)
+
+        data_start = 0
+        for fold_index in range(K):
+            data_until = data_start + jump_by
+            if data_until > N:
+                data_until = N
+
+            fold_L = lefts[data_start:data_until]
+            fold_R = rights[data_start:data_until]
+            fold_V = labels[data_start:data_until]
+
+            #print("fold_L.shape", fold_L.shape)
+
+            if fold_index == test_fold:
+                # add to test set
+                test_L = np.append(test_L, fold_L, 0)
+                test_R = np.append(test_R, fold_R, 0)
+                test_V = np.append(test_V, fold_V, 0)
+            else:
+                # add to train set
+                train_L = np.append(train_L, fold_L, 0)
+                train_R = np.append(train_R, fold_R, 0)
+                train_V = np.append(train_V, fold_V, 0)
+
+            data_start += jump_by
+
+        train = [train_L, train_R, train_V]
+        test = [test_L, test_R, test_V]
+        val = test # hmmm
+
+        return train, val, test
+
+
     def split_train_val_test(self, data):
         lefts, rights, labels = data
 
diff --git a/Evaluator.py b/Evaluator.py
@@ -43,24 +43,27 @@ def try_all_thresholds(self, predicted, labels, range_values = [0.0, 0.5, 1.0],
         ys_recalls = []
         ys_precisions = []
         ys_accuracies = []
+        ys_f1s= []
         for thr in range_values: #np.arange(0.0,1.0,0.01):
             xs.append(thr)
             print("threshold=",thr)
             #_, recall, precision, accuracy = self.calculate_metrics(predicted, labels, threshold=thr)
             if "NoChange" in title_txt:
                 print("from the position of NoChange class instead...")
-                recall, precision, accuracy, f1 = self.calculate_recall_precision_accuracy_NOCHANGECLASS(predicted, labels, threshold=thr)
+                recall, precision, accuracy, f1 = self.calculate_recall_precision_accuracy_NOCHANGECLASS(predicted, labels, threshold=thr, need_f1=True)
             else:
-                recall, precision, accuracy, f1 = self.calculate_recall_precision_accuracy(predicted, labels, threshold=thr)
+                recall, precision, accuracy, f1 = self.calculate_recall_precision_accuracy(predicted, labels, threshold=thr, need_f1=True)
 
             ys_recalls.append(recall)
             ys_precisions.append(precision)
             ys_accuracies.append(accuracy)
+            ys_f1s.append(f1)
 
         print("xs", len(xs), xs)
         print("ys_recalls", len(ys_recalls), ys_recalls)
         print("ys_precisions", len(ys_precisions), ys_precisions)
         print("ys_accuracies", len(ys_accuracies), ys_accuracies)
+        print("ys_f1s", len(ys_f1s), ys_f1s)
 
         if title_txt == "":
             plt.title('Changing the threshold values')
@@ -72,6 +75,7 @@ def try_all_thresholds(self, predicted, labels, range_values = [0.0, 0.5, 1.0],
         plt.plot(xs, ys_recalls, '-o', label="Recall")
         plt.plot(xs, ys_precisions, '-o', label="Precision")
         plt.plot(xs, ys_accuracies, '-o', label="Accuracy")
+        plt.plot(xs, ys_f1s, '-o', label="f1")
         plt.legend()
 
         plt.ylim(0.0, 1.0)
@@ -83,6 +87,25 @@ def try_all_thresholds(self, predicted, labels, range_values = [0.0, 0.5, 1.0],
         if show:
            plt.show()
 
+        plt.close()
+
+    def calculate_f1(self, predictions, ground_truths, threshold = 0.5):
+        if len(predictions.shape) > 1:
+            predictions_copy = np.array(predictions)
+        else:
+            predictions_copy = np.array([predictions])
+
+        for image in predictions_copy:
+            image[image >= threshold] = 1
+            image[image < threshold] = 0
+
+        arr_predictions = predictions_copy.flatten()
+        arr_gts = ground_truths.flatten()
+
+        sklearn_f1 = sklearn.metrics.f1_score(arr_gts, arr_predictions)
+
+        return sklearn_f1
+
     def calculate_recall_precision_accuracy(self, predictions, ground_truths, threshold = 0.5, need_f1=False):
         if len(predictions.shape) > 1:
             predictions_copy = np.array(predictions)
@@ -303,8 +326,9 @@ def calculate_metrics_fast(self, predictions, ground_truths, threshold = 0.5, ve
         return predictions_thresholded, recall, precision, accuracy
 
     # select thr which maximizes the f1 score
-    def metrics_autothr_f1_max(self, predictions, ground_truths, verbose=2):
-        range_values = np.arange(0.0, 1.0, 0.01)
+    def metrics_autothr_f1_max(self, predictions, ground_truths, jump_by = 0.1):
+        # force it selecting something 'sensible' for the threshold ...
+        range_values = np.arange(0.1, 0.9, jump_by)
 
         xs = []
         ys_recalls = []
@@ -313,17 +337,18 @@ def metrics_autothr_f1_max(self, predictions, ground_truths, verbose=2):
         ys_f1s = []
         for thr in range_values:
             xs.append(thr)
-            print("threshold=", thr)
-
-            recall, precision, accuracy, f1 = self.calculate_recall_precision_accuracy(predictions, ground_truths, threshold=thr)
+            print("auto threshold=", thr)
 
-            ys_recalls.append(recall)
-            ys_precisions.append(precision)
-            ys_accuracies.append(accuracy)
+            f1 = self.calculate_f1(predictions, ground_truths, threshold=thr)
             ys_f1s.append(f1)
 
         max_f1_idx = np.argmax(ys_f1s)
-        selected_thr = xs[max_f1_idx]
+        best_thr = xs[max_f1_idx]
+
+        selected_recall, selected_precision, selected_accuracy, _ = self.calculate_recall_precision_accuracy(predictions, ground_truths,threshold=thr, need_f1=False)
+        selected_f1 = ys_f1s[max_f1_idx]
+
+        print("Selecting threshold as", best_thr, "as it maximizes the f1 score getting", selected_f1,
+              "(other scores are: recall", selected_recall, ", precision", selected_precision, ", acc", selected_accuracy, ")")
 
-        print("Selecting threshold as", selected_thr, "as it maximizes the f1 score getting", ys_f1s[max_f1_idx],
-              "(other scores are: recall", ys_recalls[max_f1_idx], ", precision", ys_precisions[max_f1_idx], ", acc", ys_accuracies[max_f1_idx], ")")
+        return best_thr, selected_recall, selected_precision, selected_accuracy, selected_f1
diff --git a/Model2_SiamUnet_Encoder.py b/Model2_SiamUnet_Encoder.py
@@ -63,7 +63,6 @@ def __init__(self, settings, dataset):
         BACKBONE = 'resnet34'
         BACKBONE = 'resnet50' #batch 16
         #BACKBONE = 'resnet101' #batch 8
-        #BACKBONE = 'seresnext50' #trying batch 16 as well
         custom_weights_file = "imagenet"
 
         #weights from imagenet finetuned on aerial data specific task - will it work? will it break?
@@ -105,8 +104,6 @@ def train(self, show=True, save=False):
         print("label images (train)")
         self.debugger.explore_set_stats(train_V)
 
-        added_plots = []
-
         from albumentations.core.transforms_interface import DualTransform
         class RandomRotate90x1(DualTransform):
             def apply(self, img, factor=0, **params):
@@ -188,16 +185,16 @@ def get_params(self):
                     del augmented1
                     del augmented2
 
-                if False:
-                    # for sake of showing:
-                    aug_lefts_tmp, aug_rights_tmp = self.dataPreprocesser.postprocess_images(np.asarray(aug_lefts), np.asarray(aug_rights))
+            if False:
+                # for sake of showing:
+                aug_lefts_tmp, aug_rights_tmp = self.dataPreprocesser.postprocess_images(np.asarray(aug_lefts), np.asarray(aug_rights))
 
-                    #self.debugger.viewTripples(aug_lefts, aug_rights, aug_ys)
-                    by = 5
-                    off = i * by
-                    while off < len(aug_lefts):
-                        self.debugger.viewTripples(aug_lefts_tmp, aug_rights_tmp, aug_ys, how_many=by, off=off)
-                        off += by
+                #self.debugger.viewTripples(aug_lefts, aug_rights, aug_ys)
+                by = 5
+                off = i * by
+                while off < len(aug_lefts):
+                    self.debugger.viewTripples(aug_lefts_tmp, aug_rights_tmp, aug_ys, how_many=by, off=off)
+                    off += by
 
             aug_lefts = np.asarray(aug_lefts)
             aug_rights = np.asarray(aug_rights)
@@ -337,8 +334,8 @@ def test(self, evaluator, show = True, save = False):
             print("indices:", misclassified_indices)
             misclassified_indices = misclassified_indices[0]
 
-            for ind in misclassified_indices:
-                print("idx", ind, ":", predicted_classlabels[ind]," != ",test_classlabels[ind])
+            #for ind in misclassified_indices:
+            #    print("idx", ind, ":", predicted_classlabels[ind]," != ",test_classlabels[ind])
 
 
         print("MASK EVALUATION")
@@ -370,7 +367,7 @@ def test(self, evaluator, show = True, save = False):
         print("predicted images (test)")
         self.debugger.explore_set_stats(predicted)
 
-
+        """
         if Tile_Based_Evaluation:
             print("Misclassified samples (in total", len(misclassified_indices),"):")
             if show:
@@ -381,7 +378,7 @@ def test(self, evaluator, show = True, save = False):
                     #self.debugger.viewTripples(test_L, test_R, test_V, how_many=4, off=off)
                     self.debugger.viewQuadrupples(test_L[misclassified_indices], test_R[misclassified_indices], test_V[misclassified_indices], predicted[misclassified_indices], how_many=by, off=off, show=show,save=save)
                     off += by
-
+        """
 
         if show:
             off = 0
@@ -398,7 +395,9 @@ def test(self, evaluator, show = True, save = False):
             until_n = min(by*8, len(test_L))
             while off < until_n:
                 #self.debugger.viewTripples(test_L, test_R, test_V, how_many=4, off=off)
-                self.debugger.viewQuadrupples(test_L, test_R, test_V, predicted, how_many=by, off=off, show=show,save=save, name=self.save_plot_path+"quad"+str(off))
+                kfold_txt = "KFold_" + str(self.settings.TestDataset_Fold_Index) + "z" + str(self.settings.TestDataset_K_Folds)
+
+                self.debugger.viewQuadrupples(test_L, test_R, test_V, predicted, how_many=by, off=off, show=show,save=save, name=self.save_plot_path+"quad"+str(off)+kfold_txt)
                 off += by
 
 
diff --git a/Model2_builder.py b/Model2_builder.py
@@ -226,7 +226,7 @@ def build_siamese_unet(backbone, classes, skip_connection_layers,
     branch_a = branch_a_outputs[0]
     branch_b = branch_b_outputs[0]
 
-    x = Concatenate()([branch_a, branch_b]) # both inputs, in theory 8x8x512 + 8x8x512 -> 8x8x1024
+    x = Concatenate(name="concatHighLvlFeat")([branch_a, branch_b]) # both inputs, in theory 8x8x512 + 8x8x512 -> 8x8x1024
 
     skip_connection_outputs_a = branch_a_outputs[1:]
     skip_connection_outputs_b = branch_b_outputs[1:]
diff --git a/main.py b/main.py
@@ -19,6 +19,13 @@ def main(args):
     print(args)
 
     settings = Settings.Settings(args)
+
+    # We already did these
+    # ResNet50 and indices: 5, 2, 7, 3 (doing ? r.n.)
+    settings.TestDataset_Fold_Index = 999 # can be 0 to 9 (K-1)
+    settings.TestDataset_K_Folds = 10
+    assert settings.TestDataset_Fold_Index < settings.TestDataset_K_Folds
+
     dataset = Dataset.Dataset(settings)
     evaluator = Evaluator.Evaluator(settings)
 
@@ -29,7 +36,7 @@ def main(args):
     #dataset.dataset
     model = ModelHandler.ModelHandler(settings, dataset)
 
-    #model.model.train(show=show,save=save)
+    model.model.train(show=show,save=save)
 
     # Model 2 ...
 
@@ -43,11 +50,19 @@ def main(args):
     # - class weights changed ?
     # - ... any other special cool thing ...
 
+    # K-Fold_Crossval:
+    kfold_txt = "KFold_"+str(settings.TestDataset_Fold_Index)+"z"+str(settings.TestDataset_K_Folds)
+    print(kfold_txt)
+
+    # resnet 101 approx 5-6 hours (per fold - might be a bit less ...)
+    # resnet 50  approx 3-4 hours
+    model.model.save("/scratch/ruzicka/python_projects_large/ChangeDetectionProject_files/weightsModel2_cleanManual_100ep_ImagenetWgenetW_resnet50-8batch_Augmentation1to1_ClassWeights1to3_["+kfold_txt+"].h5")
+
     # Next = train Resnet50 on the same dataset without the whole STRIP2 (to have some large Test images)
 
     #model.model.load("/scratch/ruzicka/python_projects_large/ChangeDetectionProject_files/weightsModel2_cleanManual_100ep_ImagenetWgenetW_seresnext50-8batch_Augmentation1to1_ClassWeights1to3.h5")
 
-    model.model.load("/scratch/ruzicka/python_projects_large/ChangeDetectionProject_files/weightsModel2_cleanManual-noStrip2_100ep_ImagenetWgenetW_resnet50-16batch_Augmentation1to1_ClassWeights1to3.h5")
+    #model.model.load("/scratch/ruzicka/python_projects_large/ChangeDetectionProject_files/weightsModel2_cleanManual-noStrip2_100ep_ImagenetWgenetW_resnet50-16batch_Augmentation1to1_ClassWeights1to3.h5")
     #model.model.load("/scratch/ruzicka/python_projects_large/ChangeDetectionProject_files/weightsModel2_cleanManual_100ep_ImagenetWgenetW_resnet101-8batch_Augmentation1to1_ClassWeights1to3.h5")
 
     # Senet154 crashed, 10hrs train + Imagenet weights + Data Aug 1:1 + Class weight 1:3
