-
Notifications
You must be signed in to change notification settings - Fork 121
Description
Hello,
I created a sleap project on one computer using videos from an external hard drive. I had to switch external hard drives, so I exported that project as a sleap training package in order to continue working on the project on a different computer. It was working earlier this summer, however when I try to open the training package file the training and inference of a new added video fails with an error every time. Is there anyway to work around this?
Thank you!
Last login: Sun Sep 21 21:33:22 on ttys000
(base) name-MacBook-Air ~ % conda activate sleap-env
(sleap-env) name-MacBook-Air ~ % sleap-label
Saving config: /Users/name/.sleap/1.4.1/preferences.yaml
Restoring GUI state...
Software versions:
SLEAP: 1.4.1
TensorFlow: 2.9.2
Numpy: 1.22.4
Python: 3.9.23
OS: macOS-13.7.7-arm64-arm-64bit
Happy SLEAPing! :)
2025-09-22 20:56:51.346 python[7162:831863] +[CATransaction synchronize] called within transaction
2025-09-22 20:58:04.068 python[7162:831863] +[CATransaction synchronize] called within transaction
2025-09-22 21:01:58.583 python[7162:831863] Warning: Window move completed without beginning
Resetting monitor window.
Polling: /Users/name/Downloads/models/FC250922_210648.single_instance.n=4162/viz/validation.*.png
Start training single_instance...
['sleap-train', '/var/folders/bv/50cjsx2j4fs1frdyqyyhgnyh0000gn/T/tmpmq4hlhnd/250922_210648_training_job.json', '/Users/name/Downloads/TrainingPackagetest.v001.pkg (1).slp', '--zmq', '--controller_port', '9000', '--publish_port', '9001', '--save_viz']
INFO:sleap.nn.training:Versions:
SLEAP: 1.4.1
TensorFlow: 2.9.2
Numpy: 1.22.4
Python: 3.9.23
OS: macOS-13.7.7-arm64-arm-64bit
INFO:sleap.nn.training:Training labels file: /Users/name/Downloads/TrainingPackagetest.v001.pkg (1).slp
INFO:sleap.nn.training:Training profile: /var/folders/bv/50cjsx2j4fs1frdyqyyhgnyh0000gn/T/tmpmq4hlhnd/250922_210648_training_job.json
INFO:sleap.nn.training:
INFO:sleap.nn.training:Arguments:
INFO:sleap.nn.training:{
"training_job_path": "/var/folders/bv/50cjsx2j4fs1frdyqyyhgnyh0000gn/T/tmpmq4hlhnd/250922_210648_training_job.json",
"labels_path": "/Users/name/Downloads/TrainingPackagetest.v001.pkg (1).slp",
"video_paths": [
""
],
"val_labels": null,
"test_labels": null,
"base_checkpoint": null,
"tensorboard": false,
"save_viz": true,
"keep_viz": false,
"zmq": true,
"publish_port": 9001,
"controller_port": 9000,
"run_name": "",
"prefix": "",
"suffix": "",
"cpu": false,
"first_gpu": false,
"last_gpu": false,
"gpu": "auto"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Training job:
INFO:sleap.nn.training:{
"data": {
"labels": {
"training_labels": null,
"validation_labels": null,
"validation_fraction": 0.1,
"test_labels": null,
"split_by_inds": false,
"training_inds": null,
"validation_inds": null,
"test_inds": null,
"search_path_hints": [],
"skeletons": []
},
"preprocessing": {
"ensure_rgb": false,
"ensure_grayscale": true,
"imagenet_mode": null,
"input_scaling": 1.0,
"pad_to_stride": null,
"resize_and_pad_to_target": true,
"target_height": null,
"target_width": null
},
"instance_cropping": {
"center_on_part": null,
"crop_size": null,
"crop_size_detection_padding": 16
}
},
"model": {
"backbone": {
"leap": null,
"unet": {
"stem_stride": null,
"max_stride": 16,
"output_stride": 2,
"filters": 16,
"filters_rate": 2.0,
"middle_block": true,
"up_interpolate": true,
"stacks": 1
},
"hourglass": null,
"resnet": null,
"pretrained_encoder": null
},
"heads": {
"single_instance": {
"part_names": null,
"sigma": 2.5,
"output_stride": 2,
"loss_weight": 1.0,
"offset_refinement": false
},
"centroid": null,
"centered_instance": null,
"multi_instance": null,
"multi_class_bottomup": null,
"multi_class_topdown": null
},
"base_checkpoint": null
},
"optimization": {
"preload_data": true,
"augmentation_config": {
"rotate": true,
"rotation_min_angle": -15.0,
"rotation_max_angle": 15.0,
"translate": false,
"translate_min": -5,
"translate_max": 5,
"scale": false,
"scale_min": 0.9,
"scale_max": 1.1,
"uniform_noise": false,
"uniform_noise_min_val": 0.0,
"uniform_noise_max_val": 10.0,
"gaussian_noise": false,
"gaussian_noise_mean": 5.0,
"gaussian_noise_stddev": 1.0,
"contrast": false,
"contrast_min_gamma": 0.5,
"contrast_max_gamma": 2.0,
"brightness": false,
"brightness_min_val": 0.0,
"brightness_max_val": 10.0,
"random_crop": false,
"random_crop_height": 256,
"random_crop_width": 256,
"random_flip": true,
"flip_horizontal": false
},
"online_shuffling": true,
"shuffle_buffer_size": 128,
"prefetch": true,
"batch_size": 4,
"batches_per_epoch": null,
"min_batches_per_epoch": 200,
"val_batches_per_epoch": null,
"min_val_batches_per_epoch": 10,
"epochs": 200,
"optimizer": "adam",
"initial_learning_rate": 0.0001,
"learning_rate_schedule": {
"reduce_on_plateau": true,
"reduction_factor": 0.5,
"plateau_min_delta": 1e-06,
"plateau_patience": 5,
"plateau_cooldown": 3,
"min_learning_rate": 1e-08
},
"hard_keypoint_mining": {
"online_mining": false,
"hard_to_easy_ratio": 2.0,
"min_hard_keypoints": 2,
"max_hard_keypoints": null,
"loss_scale": 5.0
},
"early_stopping": {
"stop_training_on_plateau": true,
"plateau_min_delta": 1e-08,
"plateau_patience": 10
}
},
"outputs": {
"save_outputs": true,
"run_name": "250922_210648.single_instance.n=4162",
"run_name_prefix": "FC",
"run_name_suffix": "",
"runs_folder": "/Users/name/Downloads/models",
"tags": [
""
],
"save_visualizations": true,
"keep_viz_images": false,
"zip_outputs": false,
"log_to_csv": true,
"checkpointing": {
"initial_model": false,
"best_model": true,
"every_epoch": false,
"latest_model": false,
"final_model": false
},
"tensorboard": {
"write_logs": false,
"loss_frequency": "epoch",
"architecture_graph": false,
"profile_graph": false,
"visualizations": true
},
"zmq": {
"subscribe_to_controller": true,
"controller_address": "tcp://127.0.0.1:9000",
"controller_polling_timeout": 10,
"publish_updates": true,
"publish_address": "tcp://127.0.0.1:9001"
}
},
"name": "",
"description": "",
"sleap_version": "1.4.1",
"filename": "/var/folders/bv/50cjsx2j4fs1frdyqyyhgnyh0000gn/T/tmpmq4hlhnd/250922_210648_training_job.json"
}
INFO:sleap.nn.training:
INFO:sleap.nn.training:Failed to query GPU memory from nvidia-smi. Defaulting to first GPU.
INFO:sleap.nn.training:Using GPU 0 for acceleration.
INFO:sleap.nn.training:Disabled GPU memory pre-allocation.
INFO:sleap.nn.training:System:
GPUs: 1/1 available
Device: /physical_device:GPU:0
Available: True
Initialized: False
Memory growth: True
INFO:sleap.nn.training:
INFO:sleap.nn.training:Initializing trainer...
INFO:sleap.nn.training:Loading training labels from: /Users/name/Downloads/TrainingPackagetest.v001.pkg (1).slp
INFO:sleap.nn.training:Creating training and validation splits from validation fraction: 0.1
INFO:sleap.nn.training: Splits: Training = 3746 / Validation = 416.
INFO:sleap.nn.training:Setting up for training...
INFO:sleap.nn.training:Setting up pipeline builders...
INFO:sleap.nn.training:Setting up model...
INFO:sleap.nn.training:Building test pipeline...
Traceback (most recent call last):
File "/Users/name/miniconda3/envs/sleap-env/bin/sleap-train", line 33, in
sys.exit(load_entry_point('sleap==1.4.1', 'console_scripts', 'sleap-train')())
File "/Users/name/miniconda3/envs/sleap-env/lib/python3.9/site-packages/sleap/nn/training.py", line 2039, in main
trainer.train()
File "/Users/name/miniconda3/envs/sleap-env/lib/python3.9/site-packages/sleap/nn/training.py", line 924, in train
self.setup()
File "/Users/name/miniconda3/envs/sleap-env/lib/python3.9/site-packages/sleap/nn/training.py", line 910, in setup
self._setup_model()
File "/Users/name/miniconda3/envs/sleap-env/lib/python3.9/site-packages/sleap/nn/training.py", line 724, in _setup_model
base_pipeline = self.pipeline_builder.make_base_pipeline(
File "/Users/name/miniconda3/envs/sleap-env/lib/python3.9/site-packages/sleap/nn/data/pipelines.py", line 362, in make_base_pipeline
pipeline += SizeMatcher.from_config(
File "/Users/name/miniconda3/envs/sleap-env/lib/python3.9/site-packages/sleap/nn/data/resizing.py", line 324, in from_config
max_height, max_width = provider.max_height_and_width
File "/Users/name/miniconda3/envs/sleap-env/lib/python3.9/site-packages/sleap/nn/data/providers.py", line 140, in max_height_and_width
return max(video.shape[1] for video in self.videos), max(
TypeError: '>' not supported between instances of 'NoneType' and 'NoneType'
Run Path: /Users/name/Downloads/models/FC250922_210648.single_instance.n=4162
Saving config: /Users/name/.sleap/1.4.1/preferences.yaml
(sleap-env) name-MacBook-Air ~ %