Getting poor results training a top down model

[Mattshanevdberg]

Hi,

I am hoping you can assist me.

I am getting very poor results when training a top down model for pose estimation. I am starting with my simple dataset (I am using the topdown as I will expand to multi-animal and I would like results to be comparible). This consists of a single animal in each frame. My training set consists of 360 images. My val of 60 images and test of 80 images. Images are large (1920x1080).

My train_config.json for the centroid:
{
"data": {
"labels": {
"training_labels": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/labels.SimpleDataset_DLC_train.slp",
"validation_labels": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/labels.SimpleDataset_DLC_val.slp",
"validation_fraction": 0.1,
"test_labels": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/labels.SimpleDataset_DLC_test.slp",
"split_by_inds": false,
"training_inds": null,
"validation_inds": null,
"test_inds": null,
"search_path_hints": [],
"skeletons": [
{
"directed": true,
"multigraph": true,
"graph": {
"name": "Skeleton-3",
"num_edges_inserted": 0
},
"nodes": [
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Head",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Beak",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Body_top",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"RFlipper_mid",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"LFlipper_mid",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Body_bottom",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"RFoot",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"LFoot",
1.0
]
}
}
}
],
"links": []
}
]
},
"preprocessing": {
"ensure_rgb": false,
"ensure_grayscale": false,
"imagenet_mode": null,
"input_scaling": 0.5,
"pad_to_stride": 32,
"resize_and_pad_to_target": true,
"target_height": 1080,
"target_width": 1920
},
"instance_cropping": {
"center_on_part": "Body_top",
"crop_size": null,
"crop_size_detection_padding": 16
}
},
"model": {
"backbone": {
"leap": null,
"unet": {
"stem_stride": null,
"max_stride": 32,
"output_stride": 4,
"filters": 24,
"filters_rate": 1.5,
"middle_block": true,
"up_interpolate": true,
"stacks": 1
},
"hourglass": null,
"resnet": null,
"pretrained_encoder": null
},
"heads": {
"single_instance": null,
"centroid": {
"anchor_part": "Body_top",
"sigma": 2.5,
"output_stride": 4,
"loss_weight": 1.0,
"offset_refinement": false
},
"centered_instance": null,
"multi_instance": null,
"multi_class_bottomup": null,
"multi_class_topdown": null
},
"base_checkpoint": null
},
"optimization": {
"preload_data": true,
"augmentation_config": {
"rotate": true,
"rotation_min_angle": -15.0,
"rotation_max_angle": 15.0,
"translate": false,
"translate_min": -5,
"translate_max": 5,
"scale": false,
"scale_min": 0.9,
"scale_max": 1.1,
"uniform_noise": false,
"uniform_noise_min_val": 0.0,
"uniform_noise_max_val": 10.0,
"gaussian_noise": false,
"gaussian_noise_mean": 5.0,
"gaussian_noise_stddev": 1.0,
"contrast": false,
"contrast_min_gamma": 0.5,
"contrast_max_gamma": 2.0,
"brightness": false,
"brightness_min_val": 0.0,
"brightness_max_val": 10.0,
"random_crop": false,
"random_crop_height": 256,
"random_crop_width": 256,
"random_flip": false,
"flip_horizontal": true
},
"online_shuffling": true,
"shuffle_buffer_size": 128,
"prefetch": true,
"batch_size": 1,
"batches_per_epoch": 360,
"min_batches_per_epoch": 200,
"val_batches_per_epoch": 60,
"min_val_batches_per_epoch": 10,
"epochs": 200,
"optimizer": "adam",
"initial_learning_rate": 0.0001,
"learning_rate_schedule": {
"reduce_on_plateau": true,
"reduction_factor": 0.5,
"plateau_min_delta": 1e-08,
"plateau_patience": 5,
"plateau_cooldown": 3,
"min_learning_rate": 1e-08
},
"hard_keypoint_mining": {
"online_mining": false,
"hard_to_easy_ratio": 2.0,
"min_hard_keypoints": 2,
"max_hard_keypoints": null,
"loss_scale": 5.0
},
"early_stopping": {
"stop_training_on_plateau": true,
"plateau_min_delta": 1e-08,
"plateau_patience": 150
}
},
"outputs": {
"save_outputs": true,
"run_name": "Oct3_test1_centroid_200Epoch",
"run_name_prefix": "",
"run_name_suffix": "",
"runs_folder": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/baseline_models",
"tags": [],
"save_visualizations": true,
"keep_viz_images": true,
"zip_outputs": false,
"log_to_csv": true,
"checkpointing": {
"initial_model": false,
"best_model": true,
"every_epoch": false,
"latest_model": false,
"final_model": false
},
"tensorboard": {
"write_logs": false,
"loss_frequency": "epoch",
"architecture_graph": false,
"profile_graph": false,
"visualizations": true
},
"zmq": {
"subscribe_to_controller": false,
"controller_address": "tcp://127.0.0.1:9000",
"controller_polling_timeout": 10,
"publish_updates": false,
"publish_address": "tcp://127.0.0.1:9001"
}
},
"name": "",
"description": "",
"sleap_version": "1.4.1a2",
"filename": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/baseline_models/Oct3_test1_centroid_200Epoch/training_config.json"
}

my training_config.json for the centre_instance:
{
"data": {
"labels": {
"training_labels": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/labels.SimpleDataset_DLC_train.slp",
"validation_labels": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/labels.SimpleDataset_DLC_val.slp",
"validation_fraction": 0.1,
"test_labels": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/labels.SimpleDataset_DLC_test.slp",
"split_by_inds": false,
"training_inds": null,
"validation_inds": null,
"test_inds": null,
"search_path_hints": [],
"skeletons": [
{
"directed": true,
"multigraph": true,
"graph": {
"name": "Skeleton-3",
"num_edges_inserted": 0
},
"nodes": [
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Head",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Beak",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Body_top",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"RFlipper_mid",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"LFlipper_mid",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"Body_bottom",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"RFoot",
1.0
]
}
}
},
{
"id": {
"py/object": "sleap.skeleton.Node",
"py/state": {
"py/tuple": [
"LFoot",
1.0
]
}
}
}
],
"links": []
}
]
},
"preprocessing": {
"ensure_rgb": false,
"ensure_grayscale": false,
"imagenet_mode": null,
"input_scaling": 1.0,
"pad_to_stride": 1,
"resize_and_pad_to_target": true,
"target_height": 1080,
"target_width": 1920
},
"instance_cropping": {
"center_on_part": null,
"crop_size": 992,
"crop_size_detection_padding": 16
}
},
"model": {
"backbone": {
"leap": null,
"unet": {
"stem_stride": null,
"max_stride": 32,
"output_stride": 4,
"filters": 24,
"filters_rate": 1.5,
"middle_block": true,
"up_interpolate": true,
"stacks": 1
},
"hourglass": null,
"resnet": null,
"pretrained_encoder": null
},
"heads": {
"single_instance": null,
"centroid": null,
"centered_instance": {
"anchor_part": null,
"part_names": [
"Head",
"Beak",
"Body_top",
"RFlipper_mid",
"LFlipper_mid",
"Body_bottom",
"RFoot",
"LFoot"
],
"sigma": 2.5,
"output_stride": 4,
"loss_weight": 1.0,
"offset_refinement": false
},
"multi_instance": null,
"multi_class_bottomup": null,
"multi_class_topdown": null
},
"base_checkpoint": null
},
"optimization": {
"preload_data": true,
"augmentation_config": {
"rotate": true,
"rotation_min_angle": -15.0,
"rotation_max_angle": 15.0,
"translate": false,
"translate_min": -5,
"translate_max": 5,
"scale": false,
"scale_min": 0.9,
"scale_max": 1.1,
"uniform_noise": false,
"uniform_noise_min_val": 0.0,
"uniform_noise_max_val": 10.0,
"gaussian_noise": false,
"gaussian_noise_mean": 5.0,
"gaussian_noise_stddev": 1.0,
"contrast": false,
"contrast_min_gamma": 0.5,
"contrast_max_gamma": 2.0,
"brightness": false,
"brightness_min_val": 0.0,
"brightness_max_val": 10.0,
"random_crop": false,
"random_crop_height": 256,
"random_crop_width": 256,
"random_flip": false,
"flip_horizontal": true
},
"online_shuffling": true,
"shuffle_buffer_size": 128,
"prefetch": true,
"batch_size": 1,
"batches_per_epoch": 360,
"min_batches_per_epoch": 200,
"val_batches_per_epoch": 60,
"min_val_batches_per_epoch": 10,
"epochs": 200,
"optimizer": "adam",
"initial_learning_rate": 0.0001,
"learning_rate_schedule": {
"reduce_on_plateau": true,
"reduction_factor": 0.5,
"plateau_min_delta": 1e-08,
"plateau_patience": 5,
"plateau_cooldown": 3,
"min_learning_rate": 1e-08
},
"hard_keypoint_mining": {
"online_mining": false,
"hard_to_easy_ratio": 2.0,
"min_hard_keypoints": 2,
"max_hard_keypoints": null,
"loss_scale": 5.0
},
"early_stopping": {
"stop_training_on_plateau": true,
"plateau_min_delta": 1e-08,
"plateau_patience": 150
}
},
"outputs": {
"save_outputs": true,
"run_name": "Oct3_test3_medium_rf.topdown_200Epoch",
"run_name_prefix": "",
"run_name_suffix": "",
"runs_folder": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/baseline_models",
"tags": [],
"save_visualizations": true,
"keep_viz_images": true,
"zip_outputs": false,
"log_to_csv": true,
"checkpointing": {
"initial_model": false,
"best_model": true,
"every_epoch": false,
"latest_model": true,
"final_model": true
},
"tensorboard": {
"write_logs": false,
"loss_frequency": "epoch",
"architecture_graph": true,
"profile_graph": false,
"visualizations": true
},
"zmq": {
"subscribe_to_controller": false,
"controller_address": "tcp://127.0.0.1:9000",
"controller_polling_timeout": 10,
"publish_updates": false,
"publish_address": "tcp://127.0.0.1:9001"
}
},
"name": "",
"description": "",
"sleap_version": "1.4.1a2",
"filename": "/home/matthew/Desktop/Master_Dev/masters_penguin_pose_estimation/runs/SLEAP_model/baseline_models/Oct3_test3_medium_rf.topdown_200Epoch/training_config.json"
}

My results on the centroid detector don't really make sense to me but I have one instance per image in my validation set (consisting of 60 images), so there should be 60 instances detected however the visibility metrics are as follows:
Visibility Metrics:
True Positives (TP): 280
False Positives (FP): 0
True Negatives (TN): 0
False Negatives (FN): 0
Precision: 1.0
Recall: 1.0

However my gt labels when I describe() them are as follows:
Frames (user/predicted): 60/0
Instances (user/predicted): 60/0
and my pr labels are:
Frames (user/predicted): 0/35
Instances (user/predicted): 0/49
so I would think the visiblity metrics to look more like this:
Visibility Metrics (not what was reported but what I would expect give the instance detecter):
True Positives (TP): 35
False Positives (FP): 14
True Negatives (TN): 0
False Negatives (FN): 25
Precision: 0.71
Recall: 0.58

The centroid error distance metrics are as follows:
Centroid Distance Metrics:
Error distance (avg): 2.6409071517163838e-05
Error distance (50%): 2.5054158188260635e-05
Error distance (75%): 3.379028532049095e-05
Error distance (90%): 4.965237132973848e-05
Error distance (95%): 5.5047631915180444e-05
Error distance (99%): 6.1694600633691e-05

for the centroid both the OKS and PCK
Mean Average Precision (mAP): 0.5841584158415841
Mean Average Recall (mAR): 0.5833333333333333

For the Top_down (centre_instance)
My validation results are as follows:
Visibility Metrics:
True Positives (TP): 193
False Positives (FP): 6
True Negatives (TN): 37
False Negatives (FN): 180
Precision: 0.9698492462311558
Recall: 0.517426273458445

NOTE: We are missing almost half the keypoints. That is not a good sign

interestingly my instance detection is slightly better:
However my gt labels when I describe() them are as follows:
Frames (user/predicted): 60/0
Instances (user/predicted): 60/0
and my pr labels are:
Frames (user/predicted): 0/55
Instances (user/predicted): 0/55

PIXEL DISTANCE:
Error distance (avg): 46.714012124678426
Error distance (50%): 10.731174132897111
Error distance (75%): 19.35681355090176
Error distance (90%): 78.87723090070979
Error distance (95%): 239.23818025309083
Error distance (99%): 730.6019764445392

PCK:
Mean PCK per keypoint (mPCK_parts): [0.24423077 0.27307692 0.09807692 0.00576923 0.03269231 0.1 0.09038462]
Overall mean PCK (mPCK): 0.10552884615384617
I have also uploaded some graphs for this
OKS:
Mean OKS (mOKS): 0.37586655730354923
I have also uploaded some graphs for this

I have also uploaded some pictures of the predictions in the output viz folder

Please let me know if you can spot any obvious errors on my part. Thanks so much!

[github-actions[bot]]

Thank you for starting a new discussion!

We appreciate your input and will review it soon.

Warning

A friendly reminder that this is a public forum. Please be cautious when clicking links, downloading files, or running scripts posted by others.

• Always verify the credibility of links and code.
• Avoid running scripts or installing files from untrusted sources.
• If you're unsure, ask for clarification before proceeding.

Stay safe and happy SLEAPing!

Best regards,
The Team

[eberrigan]

Hi @Mattshanevdberg,

Wow so cute!

• I noticed that in some of your frames I can't see all of the keypoints. Are you toggling the visibility to off in those frames?
• Also, just wondering why you chose the middle of the wing as a keypoint instead of the tip? The tip might be easier to label consistently and easier to detect.
• You could try the bottom-up model instead, especially if your animal changes scale significantly. The top-down model will crop the animal at the same size across the frames.
• How does the receptive field size look for your centroid model in the preview on the training window? It should be approximately the size of your animal.
• Checkout the guide on choosing models here.

Your test set consists of 80 images. This means those 80 images will be held out from the model during training. The other 360 - 80 = 280 images are used to train and validate the model. To evaluate your model you should use the test set since this gives a more realistic measure of how your model will perform on new data.

Best,

Elizabeth