Bad accuracy after retraining a SSDLite MobileDet for the EdgeTPU to detect hand gestures

[lproj]

Hi guys,

I am looking for help about the following problem.

I am trying to retrain the official mobildet model for the Edge TPU to detect 5 hand gestures from 440 annotated images used for training and 160 images used for evaluation. The non-annotated dataset is available here: https://lttm.dei.unipd.it/downloads/gesture/#kinect_leap.

I am using the following Google Colab project to speed up the training with the GPU and to compile the trained TFLite model for the EdgeTPU: https://colab.research.google.com/github/Namburger/edgetpu-ssdlite-mobiledet-retrain/blob/master/ssdlite_mobiledet_transfer_learning_cat_vs_dog.ipynb

The number of training steps was 25000 (it took 6 hours!).

I created the proper train and test .tfrecord's and double checked that they are fine with tfrecord-viewer (I slightly modified the pipeline in the above project to point to my tfrecord and label files instead of the default one). The training and the compilation go well and I can load the model onto the USB Coral Edge TPU.

However when evaluating this model with a live stream from the camera, it cannot detect anything or, when it detects something with some certain confidence score, it's just garbage more often than not! The curious thing is that from the tensorboard it seemed the model being trained could successfully detect the gestures after just some initial training steps.

Some people on youtube got a pretty good model starting from the official mobilenet (not det) with just a few of images and 2000 steps. So what am I doing wrong? Below is the pipeline configuration I used.

model {
  ssd {
    num_classes: 5
    image_resizer {
      fixed_shape_resizer {
        height: 320
        width: 320
      }
    }
    feature_extractor {
      type: "ssd_mobiledet_edgetpu"
      depth_multiplier: 1.0
      min_depth: 16
      conv_hyperparams {
        regularizer {
          l2_regularizer {
            weight: 4e-05
          }
        }
        initializer {
          truncated_normal_initializer {
            mean: 0.0
            stddev: 0.03
          }
        }
        activation: RELU_6
        batch_norm {
          decay: 0.97
          center: true
          scale: true
          epsilon: 0.001
          train: true
        }
      }
      use_depthwise: true
      override_base_feature_extractor_hyperparams: false
    }
    box_coder {
      faster_rcnn_box_coder {
        y_scale: 10.0
        x_scale: 10.0
        height_scale: 5.0
        width_scale: 5.0
      }
    }
    matcher {
      argmax_matcher {
        matched_threshold: 0.5
        unmatched_threshold: 0.5
        ignore_thresholds: false
        negatives_lower_than_unmatched: true
        force_match_for_each_row: true
        use_matmul_gather: true
      }
    }
    similarity_calculator {
      iou_similarity {
      }
    }
    box_predictor {
      convolutional_box_predictor {
        conv_hyperparams {
          regularizer {
            l2_regularizer {
              weight: 4e-05
            }
          }
          initializer {
            random_normal_initializer {
              mean: 0.0
              stddev: 0.03
            }
          }
          activation: RELU_6
          batch_norm {
            decay: 0.97
            center: true
            scale: true
            epsilon: 0.001
            train: true
          }
        }
        min_depth: 0
        max_depth: 0
        num_layers_before_predictor: 0
        use_dropout: false
        dropout_keep_probability: 0.8
        kernel_size: 3
        box_code_size: 4
        apply_sigmoid_to_scores: false
        class_prediction_bias_init: -4.6
        use_depthwise: true
      }
    }
    anchor_generator {
      ssd_anchor_generator {
        num_layers: 6
        min_scale: 0.2
        max_scale: 0.95
        aspect_ratios: 1.0
        aspect_ratios: 2.0
        aspect_ratios: 0.5
        aspect_ratios: 3.0
        aspect_ratios: 0.3333
      }
    }
    post_processing {
      batch_non_max_suppression {
        score_threshold: 1e-08
        iou_threshold: 0.6
        max_detections_per_class: 100
        max_total_detections: 100
        use_static_shapes: true
      }
      score_converter: SIGMOID
    }
    normalize_loss_by_num_matches: true
    loss {
      localization_loss {
        weighted_smooth_l1 {
          delta: 1.0
        }
      }
      classification_loss {
        weighted_sigmoid_focal {
          gamma: 2.0
          alpha: 0.75
        }
      }
      classification_weight: 1.0
      localization_weight: 1.0
    }
    encode_background_as_zeros: true
    normalize_loc_loss_by_codesize: true
    inplace_batchnorm_update: true
    freeze_batchnorm: false
  }
}
train_config {
  batch_size: 32
  data_augmentation_options {
    random_horizontal_flip {
    }
  }
  data_augmentation_options {
    ssd_random_crop {
    }
  }
  sync_replicas: true
  optimizer {
    momentum_optimizer {
      learning_rate {
        cosine_decay_learning_rate {
          learning_rate_base: 0.8
          total_steps: 400000
          warmup_learning_rate: 0.13333
          warmup_steps: 2000
        }
      }
      momentum_optimizer_value: 0.9
    }
    use_moving_average: false
  }
  fine_tune_checkpoint: "/content/train/model.ckpt"
  num_steps: 25000
  startup_delay_steps: 0.0
  replicas_to_aggregate: 32
  max_number_of_boxes: 100
  unpad_groundtruth_tensors: false
}
train_input_reader {
  label_map_path: "/content/gdrive/MyDrive/annotations/label_map.pbtxt"
  tf_record_input_reader {
    input_path: "/content/gdrive/MyDrive/annotations/train.record"
  }
}
eval_config {
  num_examples: 160
  metrics_set: "coco_detection_metrics"
  use_moving_averages: false
}
eval_input_reader {
  label_map_path: "/content/gdrive/MyDrive/annotations/label_map.pbtxt"
  shuffle: false
  num_epochs: 1
  tf_record_input_reader {
    input_path: "/content/gdrive/MyDrive/annotations/test.record"
  }
}
graph_rewriter {
  quantization {
    delay: 0
    weight_bits: 8
    activation_bits: 8
  }
}