Added readme for ios-logger scan inference. Fix bug for depth min/max when no valid depth exists.

Author: mohammed-amr

README.md

@@ -55,7 +55,7 @@ With larger batches speed increases considerably. With batch size 8 on the non-s
 - [x] Simple scan for folks to quickly try the code, instead of downloading the ScanNetv2 test scenes. DONE
 - [x] ScanNetv2 extraction, ~~ETA 10th October~~ DONE
 - [ ] FPN model weights.
-- ~~[ ] Tutorial on how to use Scanniverse data, ETA 5th October 10th October 20th October~~ At present there is no publically available way of exporting scans from Scanniverse. You'll have to use ios-logger; NeuralRecon have a good tutorial on [this](https://github.com/zju3dv/NeuralRecon/blob/master/DEMO.md), and a dataloader that accepts the processed format is at ```datasets/arkit_dataset.py```. 
+- ~~[ ] Tutorial on how to use Scanniverse data, ETA 5th October 10th October 20th October~~ At present there is no publically available way of exporting scans from Scanniverse. You'll have to use ios-logger; NeuralRecon have a good tutorial on [this](https://github.com/zju3dv/NeuralRecon/blob/master/DEMO.md), and a dataloader that accepts the processed format is at ```datasets/arkit_dataset.py```. UPDATE: There is now  quick readme [data_scripts/IOS_LOGGER_ARKIT_README.md](data_scripts/IOS_LOGGER_ARKIT_README.md) for how to process and run inference an ios-logger scan using the script at ```data_scripts/ios_logger_preprocessing.py```.
 
 ## 🏃 Running out of the box!
 

configs/data/arkit_dense.yaml

@@ -0,0 +1,9 @@
+!!python/object:options.Options
+dataset_path: /mnt/res_nas/mohameds/datasets/arkit
+tuple_info_file_location: data_splits/arkit/
+dataset_scan_split_file: data_splits/arkit/scans.txt
+dataset: arkit
+mv_tuple_file_suffix: _eight_view_deepvmvs.txt
+num_images_in_tuple: 8
+frame_tuple_type: default
+split: test

configs/data/neucon_arkit_default.yaml

@@ -0,0 +1,9 @@
+!!python/object:options.Options
+dataset_path: /mnt/res_nas/mohameds/datasets/arkit
+tuple_info_file_location: data_splits/arkit/
+dataset_scan_split_file: data_splits/arkit/scans.txt
+dataset: arkit
+mv_tuple_file_suffix: _eight_view_deepvmvs_dense.txt
+num_images_in_tuple: 8
+frame_tuple_type: dense
+split: test

configs/data/neucon_arkit_dense.yaml

@@ -0,0 +1,69 @@
+# Running with NeuralRecon's demo data and data from ios-logger.
+
+Download the demo scene out of ios-logger from here: https://github.com/zju3dv/NeuralRecon/blob/master/DEMO.md
+
+Follow the instructions in the NeuralRecon repo for how to use ios-logger to make your own captures.
+
+Unzip the folder into your arkit dataset path so that it looks somthing like this:
+
+```
+    dataset_path
+        scans 
+            neucon_demodata_b5f1
+                ...
+            ....
+```
+
+Run the extraction script that uses modified versions of the functions provided by the NeuralRecon authors:
+
+```bash
+python ./data_scripts/ios_logger_preprocessing.py --data_config configs/data/neucon_arkit_default.yaml 
+```
+
+Make sure you set your correct `dataset_path` folder.
+
+Run tuple file generation (we've already computed one for you in data_splits):
+
+```bash
+python ./data_scripts/generate_test_tuples.py  --num_workers 16 --data_config configs/data/neucon_arkit_default.yaml
+```
+
+Then run the model using this config file, see the full readme for more. 
+
+There is unfortunately a break in the pose in the NR demo scene, so you'll to trim the first 350 frames using `--skip_to_frame 350` when running dense frames and `--skip_to_frame 83` when running default.
+
+Run:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 python test.py --name HERO_MODEL \
+            --output_base_path OUTPUT_PATH \
+            --config_file configs/models/hero_model.yaml \
+            --load_weights_from_checkpoint weights/hero_model.ckpt \
+            --data_config configs/data/neucon_arkit_default.yaml \
+            --num_workers 8 \
+            --batch_size 2 \
+            --fast_cost_volume \
+            --run_fusion \
+            --depth_fuser open3d \
+            --fuse_color \
+            --skip_to_frame 83;
+```
+
+```bash
+CUDA_VISIBLE_DEVICES=0 python test.py --name HERO_MODEL \
+            --output_base_path OUTPUT_PATH \
+            --config_file configs/models/hero_model.yaml \
+            --load_weights_from_checkpoint weights/hero_model.ckpt \
+            --data_config configs/data/neucon_arkit_dense.yaml \
+            --num_workers 8 \
+            --batch_size 2 \
+            --fast_cost_volume \
+            --run_fusion \
+            --depth_fuser open3d \
+            --fuse_color \
+            --skip_to_frame 350;
+```
+
+Should get an output that looks like this for default frames: 
+
+![alt text](../media/arkit_ioslogger_snapshot.png)

data_scripts/IOS_LOGGER_ARKIT_README.md

@@ -0,0 +1,32 @@
+import os
+import sys
+sys.path.append("/".join(sys.path[0].split("/")[:-1]))
+from datasets.arkit_dataset import process_data
+import options
+
+""" Download scans and extract each to a folder with their name like so:
+    dataset_path
+        scans 
+            neucon_demodata_b5f1
+                ...
+            ....
+"""
+
+option_handler = options.OptionsHandler()
+option_handler.parse_and_merge_options()
+opts = option_handler.options
+
+
+if opts.dataset_scan_split_file is not None:
+    f = open(opts.dataset_scan_split_file, "r")
+    scans = f.readlines()
+    scans = [scan.strip() for scan in scans]
+    f.close()
+elif opts.single_debug_scan_id is not None:
+    scans = [opts.single_debug_scan_id]
+else:
+    print("No valid scans pointers.")
+
+for scan in scans:
+    path_dir = os.path.join(opts.dataset_path, "scans", scan)
+    process_data(path_dir)

data_scripts/ios_logger_preprocessing.py

@@ -0,0 +1 @@
+neucon_demodata_b5f1

data_splits/arkit/scans.txt

@@ -44,6 +44,7 @@ def __init__(self,
             include_high_res_color=False,
             pass_frame_id=False,
             skip_frames=None,
+            skip_to_frame=None,
             verbose_init=True,
             min_valid_depth=1e-3,
             max_valid_depth=10,
@@ -64,7 +65,7 @@ def __init__(self,
                 include_full_depth_K=include_full_depth_K, 
                 include_high_res_color=include_high_res_color, 
                 pass_frame_id=pass_frame_id, skip_frames=skip_frames, 
-                verbose_init=verbose_init, 
+                skip_to_frame=skip_to_frame, verbose_init=verbose_init, 
                 native_depth_width=native_depth_width,
                 native_depth_height=native_depth_height
             )

data_splits/arkit/test_eight_view_deepvmvs.txt

@@ -65,6 +65,7 @@ def __init__(
             include_high_res_color=False,
             pass_frame_id=False,
             skip_frames=None,
+            skip_to_frame=None,
             verbose_init=True,
             native_depth_width=640,
             native_depth_height=480,
@@ -84,7 +85,7 @@ def __init__(
                 include_full_depth_K=include_full_depth_K, 
                 include_high_res_color=include_high_res_color, 
                 pass_frame_id=pass_frame_id, skip_frames=skip_frames, 
-                verbose_init=verbose_init,
+                skip_to_frame=skip_to_frame, verbose_init=verbose_init,
                 native_depth_width=native_depth_width,
                 native_depth_height=native_depth_height,
             )

data_splits/arkit/test_eight_view_deepvmvs_dense.txt

@@ -59,6 +59,7 @@ def __init__(self,
                 shuffle_tuple=False,
                 pass_frame_id=False,
                 skip_frames=None,
+                skip_to_frame=None,
                 verbose_init=True,
                 native_depth_width=640,
                 native_depth_height=480,
@@ -140,6 +141,17 @@ def __init__(self,
                         self.frame_tuples if limit_to_scan_id == 
                             frame_tuple.split(" ")[0]]
 
+            if skip_to_frame is not None:
+                if verbose_init:
+                    print(f"".center(80, "#"))
+                    print(f"".center(80, "#"))
+                    print(f"".center(80, "#"))
+                    print(f" Skipping to frame {skip_to_frame} ".center(80, "#"))
+                    print(f"".center(80, "#"))
+                    print(f"".center(80, "#"))
+                    print(f"".center(80, "#"), "\n")
+                self.frame_tuples = self.frame_tuples[skip_to_frame:]
+
             # optionally skip every frame with interval skip_frame 
             if skip_frames is not None:
                 if verbose_init:

datasets/arkit_dataset.py

@@ -94,6 +94,7 @@ def __init__(
             include_high_res_color=False,
             pass_frame_id=False,
             skip_frames=None,
+            skip_to_frame=None,
             verbose_init=True,
             min_valid_depth=1e-3,
             max_valid_depth=10,
@@ -113,7 +114,7 @@ def __init__(
                 include_full_depth_K=include_full_depth_K, 
                 include_high_res_color=include_high_res_color, 
                 pass_frame_id=pass_frame_id, skip_frames=skip_frames, 
-                verbose_init=verbose_init,
+                skip_to_frame=skip_to_frame, verbose_init=verbose_init,
             )
 
         """

datasets/colmap_dataset.py

@@ -44,6 +44,7 @@ def __init__(
             include_high_res_color=False,
             pass_frame_id=False,
             skip_frames=None,
+            skip_to_frame=None,
             verbose_init=True,
             min_valid_depth=1e-3,
             max_valid_depth=10,
@@ -65,7 +66,7 @@ def __init__(
                 include_full_depth_K=include_full_depth_K, 
                 include_high_res_color=include_high_res_color, 
                 pass_frame_id=pass_frame_id, skip_frames=skip_frames, 
-                verbose_init=verbose_init,
+                skip_to_frame=skip_to_frame, verbose_init=verbose_init,
                 native_depth_width=native_depth_width,
                 native_depth_height=native_depth_height,
             )

datasets/generic_mvs_dataset.py

@@ -64,6 +64,7 @@ def __init__(
             include_high_res_color=False,
             pass_frame_id=False,
             skip_frames=None,
+            skip_to_frame=None,
             verbose_init=True,
             min_valid_depth=1e-3,
             max_valid_depth=10,
@@ -83,7 +84,7 @@ def __init__(
                 include_full_depth_K=include_full_depth_K, 
                 include_high_res_color=include_high_res_color, 
                 pass_frame_id=pass_frame_id, skip_frames=skip_frames, 
-                verbose_init=verbose_init,
+                skip_to_frame=skip_to_frame, verbose_init=verbose_init,
             )
         """
         Args:

datasets/scannet_dataset.py

@@ -46,6 +46,7 @@ def __init__(
             include_high_res_color=False,
             pass_frame_id=False,
             skip_frames=None,
+            skip_to_frame=None,
             verbose_init=True,
             native_depth_width=256,
             native_depth_height=192,
@@ -64,7 +65,7 @@ def __init__(
                 include_full_depth_K=include_full_depth_K, 
                 include_high_res_color=include_high_res_color, 
                 pass_frame_id=pass_frame_id, skip_frames=skip_frames, 
-                verbose_init=verbose_init,
+                skip_to_frame=skip_to_frame, verbose_init=verbose_init,
                 native_depth_width=native_depth_width,
                 native_depth_height=native_depth_height,
             )

datasets/scanniverse_dataset.py

@@ -164,6 +164,9 @@ class Options():
     # tuples
     skip_frames: int = None
 
+    # skip to a frame in the scan
+    skip_to_frame: int = None
+
     # point cloud fusion hyperparams.
     pc_fusion_z_thresh: float = 0.04
     n_consistent_thresh: int = 3

datasets/seven_scenes_dataset.py

@@ -84,6 +84,7 @@ def main(opts):
                         include_high_res_color=True,
                         include_full_depth_K=True,
                         skip_frames=opts.skip_frames,
+                        skip_to_frame=opts.skip_to_frame,
                         image_width=opts.image_width,
                         image_height=opts.image_height,
                         pass_frame_id=True,

datasets/vdr_dataset.py

@@ -234,6 +234,7 @@ def main(opts):
                                     ),
                         include_full_depth_K=True,
                         skip_frames=opts.skip_frames,
+                        skip_to_frame=opts.skip_to_frame,
                         image_width=opts.image_width,
                         image_height=opts.image_height,
                         pass_frame_id=True,

media/arkit_ioslogger_snapshot.png

@@ -97,6 +97,10 @@ def quick_viz_export(
         batch_vmin = cur_data["full_res_depth_b1hw"][valid_mask_b].min()
         batch_vmax = cur_data["full_res_depth_b1hw"][valid_mask_b].max()
 
+    if batch_vmax == batch_vmin:
+        batch_vmin = 0.0
+        batch_vmax = 5.0
+
     for elem_ind in range(outputs["depth_pred_s0_b1hw"].shape[0]):
         if "frame_id_string" in cur_data:
             frame_id = cur_data["frame_id_string"][elem_ind]

options.py

@@ -65,6 +65,7 @@ def main(opts):
                         pass_frame_id=True,
                         include_full_depth_K=False,
                         skip_frames=opts.skip_frames,
+                        skip_to_frame=opts.skip_to_frame,
                     )
 
             dataloader = torch.utils.data.DataLoader(

pc_fusion.py

@@ -108,6 +108,7 @@ def main(opts):
                                 include_high_res_color=True,
                                 pass_frame_id=True,
                                 include_full_depth_K=True,
+                                skip_to_frame=opts.skip_to_frame,
                             )
 
             dataloader = torch.utils.data.DataLoader(

test.py

@@ -111,6 +111,7 @@ def main(opts):
                     include_high_res_color=opts.fuse_color and opts.run_fusion,
                     include_full_depth_K=True,
                     skip_frames=opts.skip_frames,
+                    skip_to_frame=opts.skip_to_frame,
                     image_width=opts.image_width,
                     image_height=opts.image_height,
                     pass_frame_id=True,