feat(examples): added suncg_with_light_projector

Author: apenzko

blenderproc/python/modules/lighting/LightInterface.py

@@ -63,4 +63,8 @@ def _add_light_source(self, config):
         light.set_rotation_euler(config.get_list("rotation", [0, 0, 0]))
         light.set_energy(config.get_float("energy", 10.))
         light.set_color(config.get_list("color", [1, 1, 1])[:3])
-        light.set_distance(config.get_float("distance", 0))
+        light.set_distance(config.get_float("distance", 0))
+        if config.get_bool("use_projector", False):
+            light.setup_as_projector(config.get_string("path", "examples/advanced/stereo_matching_with_projector"
+                                                               "/patterns/random_pattern_00256.png"))
+

blenderproc/python/types/LightUtility.py

@@ -62,6 +62,59 @@ def set_type(self, type: str, frame: int = None):
         self.blender_obj.data.type = type
         Utility.insert_keyframe(self.blender_obj.data, "type", frame)
 
+    def setup_as_projector(self, path: str, frame: int = None):
+        if path is None or path == "none":
+            print('Path is None')
+            return
+
+        # Set location to camera -- COPY TRANSFORMS
+        self.blender_obj.constraints.new('COPY_TRANSFORMS')
+        self.blender_obj.constraints['Copy Transforms'].target = bpy.context.scene.camera
+
+        # Setup nodes for projecting image
+        self.blender_obj.data.use_nodes = True
+        self.blender_obj.data.shadow_soft_size = 0
+        self.blender_obj.data.spot_size = 2.0944
+        self.blender_obj.data.cycles.cast_shadow = False
+
+        nodes = self.blender_obj.data.node_tree.nodes
+        links = self.blender_obj.data.node_tree.links
+
+        img_name = path.split('/')[-1]
+        print(path)
+        print(f'\nUsing {img_name} as projector image\n')
+        bpy.data.images.load(path, check_existing=False)
+
+        node_ox = nodes.get('Emission')
+
+        # Set Up Nodes
+        node_pattern = nodes.new(type="ShaderNodeTexImage")  # Texture Image
+        node_pattern.image = bpy.data.images[img_name]
+        node_pattern.extension = 'CLIP'
+
+        node_mapping = nodes.new(type="ShaderNodeMapping")  # Mapping
+        node_mapping.inputs[1].default_value[0] = 0.5  # Location X
+        node_mapping.inputs[1].default_value[1] = 0.5  # Location Y
+        node_mapping.inputs[3].default_value[0] = 0.4  # Scaling X
+        node_mapping.inputs[3].default_value[1] = 0.4  # Scaling Y
+
+        node_coord = nodes.new(type="ShaderNodeTexCoord")  # Texture Coordinate
+
+        divide = nodes.new(type="ShaderNodeVectorMath")
+        divide.operation = 'DIVIDE'
+
+        z_component = nodes.new(type="ShaderNodeSeparateXYZ")
+
+        # Set Up Links
+        links.new(node_pattern.outputs["Color"], node_ox.inputs["Color"])  # Link Image Texture to Emission
+        links.new(node_mapping.outputs["Vector"], node_pattern.inputs["Vector"])  # Link Mapping to Image Texture
+        links.new(divide.outputs["Vector"], node_mapping.inputs["Vector"])  # Link Texture Coordinate to Mapping
+        links.new(node_coord.outputs["Normal"], divide.inputs[0])
+        links.new(node_coord.outputs["Normal"], z_component.inputs["Vector"])
+        links.new(z_component.outputs["Z"], divide.inputs[1])
+
+        Utility.insert_keyframe(self.blender_obj.data, "use_projector", frame)
+
 
     def get_energy(self, frame: int = None) -> float:
         """ Returns the energy of the light.

examples/advanced/stereo_matching_with_projector/README.md

@@ -0,0 +1,59 @@
+# Stereo Matching
+![](../../../images/stereo_matching_stereo_pair.jpg)
+![](../../../images/stereo_matching_stereo_depth.jpg)
+
+In the first row we can see the rendered stereo RGB images, left and right respectively, and beneath them we can view
+the computed depth image using stereo matching. Note that due to a high discrepancy between the TV and the rest
+of the rendered scene, the visualization is not descriptive enough. This discrepancy or high depth values at the TV
+is due to a lack of gradient or useful features for stereo matching in this area. However, the depth values in other
+areas are consistent and close to the rendered depth images.
+
+## Usage
+
+Execute in the BlenderProc main directory:
+
+```
+blenderproc run examples/advanced/stereo_matching/main.py <path to cam_pose file> <path to house.json> examples/advanced/stereo_matching/output
+```
+
+* `examples/advanced/stereo_matching/main.py`: path to the main python file to run.
+* `<path to cam_pose file>`: Should point to a file which describes one camera pose per line (here the output of `scn2cam` from the `SUNCGToolbox` can be used).
+* `<path to house.json>`: Path to the house.json file of the SUNCG scene you want to render. Which should be either located inside the SUNCG directory, or the SUNCG directory path should be added to the config file.
+* `examples/advanced/stereo_matching/output`: path to the output directory.
+
+## Visualizaton
+Visualize the generated data:
+```
+blenderproc vis hdf5 examples/advanced/stereo_matching/output/1.hdf5
+```
+
+## Implementation
+
+```python
+# Enable stereo mode and set baseline
+bproc.camera.set_stereo_parameters(interocular_distance=0.05, convergence_mode="PARALLEL")
+```
+
+Here we enable stereo rendering and specify the camera parameters, some notable points are:
+* Setting the `interocular_distance` which is the stereo baseline.
+* Specifying `convergence_mode` to be `"PARALLEL"` (i.e. both cameras lie on the same line and are just shifted by `interocular_distance`, and are trivially coplanar).
+    * Other options are `OFF-AXIS` where the cameras rotate inwards (converge) up to some plane.  
+  * `convergence_distance` is the distance from the cameras to the aforementioned plane they converge to in case of `OFF-AXIS` convergence mode. In this case, this parameter is ignored by Blender, but it is added here for clarification.
+
+```python
+# Apply stereo matching to each pair of images
+data["stereo-depth"], data["disparity"] = bproc.postprocessing.stereo_global_matching(data["colors"], disparity_filter=False)
+```
+
+Here we apply the stereo matching.
+* It is based on OpenCV's [implementation](https://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html?highlight=sgbm#stereosgbm-stereosgbm) of [stereo semi global matching](https://elib.dlr.de/73119/1/180Hirschmueller.pdf).
+* Its pipeline runs as follows:
+    * Compute the disparity map between the two images. After specifying the required parameters.
+    * Optional use of a disparity filter (namely `wls_filter`). Enabled by setting `disparity_filter` (Enabling it could possibly lead to less accurate depth values. One should experiment with this parameter).
+    * Triangulate the depth values using the focal length and disparity.
+    * Clip the depth map from 0 to `depth_max`, where this value is retrieved from `renderer.Renderer`.
+    * Apply an optional [depth completion routine](https://github.com/kujason/ip_basic/blob/master/ip_basic/depth_map_utils.py), based on simple image processing techniques. This is enabled by setting `depth_completion`.
+* There are some stereo semi global matching parameters that can be tuned (see fct docs), such as:
+    * `window_size`
+    * `num_disparities`
+    * `min_disparity`

examples/advanced/stereo_matching_with_projector/config.yaml

@@ -0,0 +1,73 @@
+# Args: <cam_file> <obj_file> <output_dir>
+{
+  "version": 3,
+  "setup": {
+    "blender_install_path": "/home_local/<env:USER>/blender/",
+    "pip": [
+      "h5py",
+      "python-dateutil==2.1",
+      "numpy",
+      "Pillow",
+      "opencv-contrib-python",
+      "scipy"
+    ]
+  },
+  "modules": [
+    {
+      "module": "main.Initializer",
+      "config": {
+        "global": {
+          "output_dir": "<args:2>"
+        }
+      }
+    },
+    {
+      "module": "loader.SuncgLoader",
+      "config": {
+        "path": "<args:1>"
+      }
+    },
+    {
+      "module": "camera.CameraLoader",
+      "config": {
+        "path": "<args:0>",
+        "file_format": "location rotation/value _ _ _ _ _ _",
+        "world_frame_change": ["X", "-Z", "Y"],
+        "default_cam_param": {
+          "rotation": {
+            "format": "forward_vec"
+          }
+        },
+        "intrinsics": {
+          "interocular_distance": 0.05,
+          "stereo_convergence_mode": "PARALLEL",
+          "convergence_distance": 0.00001,
+          "cam_K": [650.018, 0, 637.962, 0, 650.018, 355.984, 0, 0 ,1],
+          "resolution_x": 1280,
+          "resolution_y": 720
+        },
+      }
+    },
+    {
+      "module": "lighting.SuncgLighting",
+      "config": {}
+    },
+    {
+      "module": "renderer.RgbRenderer",
+      "config": {
+        "render_distance": true,
+        "stereo": true,
+        "use_alpha": true,
+      }
+    },
+    {
+      "module": "writer.StereoGlobalMatchingWriter",
+      "config": {
+        "disparity_filter": false
+      }
+    },
+    {
+      "module": "writer.Hdf5Writer",
+    }
+  ]
+}

examples/advanced/stereo_matching_with_projector/main.py

@@ -0,0 +1,59 @@
+import blenderproc as bproc
+import argparse
+import os
+import numpy as np
+
+parser = argparse.ArgumentParser()
+parser.add_argument('camera', help="Path to the camera file which describes one camera pose per line, here the output of scn2cam from the SUNCGToolbox can be used")
+parser.add_argument('house', help="Path to the house.json file of the SUNCG scene to load")
+parser.add_argument('projector', help="Path to the image, which should be projected onto the scene")
+parser.add_argument('output_dir', nargs='?', default="examples/datasets/suncg_basic/output", help="Path to where the final files, will be saved")
+args = parser.parse_args()
+#print(f"\nUsing {args.projector} as projector image\n")
+bproc.init()
+
+# load the objects into the scene
+label_mapping = bproc.utility.LabelIdMapping.from_csv(bproc.utility.resolve_resource(os.path.join('id_mappings', 'nyu_idset.csv')))
+objs = bproc.loader.load_suncg(args.house, label_mapping=label_mapping)
+
+# define the camera intrinsics
+K = np.array([
+    [650.018, 0, 637.962],
+    [0, 650.018, 355.984],
+    [0, 0, 1]
+])
+bproc.camera.set_intrinsics_from_K_matrix(K, 1280, 720)
+# Enable stereo mode and set baseline
+bproc.camera.set_stereo_parameters(interocular_distance=0.05, convergence_mode="PARALLEL", convergence_distance=0.00001)
+
+# define a new light and set it as projector
+light = bproc.types.Light()
+light.set_type('SPOT')
+light.set_energy(3000)
+light.setup_as_projector(args.projector)
+
+# read the camera positions file and convert into homogeneous camera-world transformation
+with open(args.camera, "r") as f:
+    for line in f.readlines():
+        line = [float(x) for x in line.split()]
+        position = bproc.math.change_coordinate_frame_of_point(line[:3], ["X", "-Z", "Y"])
+        rotation = bproc.math.change_coordinate_frame_of_point(line[3:6], ["X", "-Z", "Y"])
+        matrix_world = bproc.math.build_transformation_mat(position, bproc.camera.rotation_from_forward_vec(rotation))
+        bproc.camera.add_camera_pose(matrix_world)
+
+# makes Suncg objects emit light
+bproc.lighting.light_suncg_scene()
+
+# activate normal and depth rendering
+bproc.renderer.enable_depth_output(activate_antialiasing=False)
+bproc.material.add_alpha_channel_to_textures(blurry_edges=True)
+bproc.renderer.toggle_stereo(True)
+
+# render the whole pipeline
+data = bproc.renderer.render()
+
+# Apply stereo matching to each pair of images
+data["stereo-depth"], data["disparity"] = bproc.postprocessing.stereo_global_matching(data["colors"], disparity_filter=False)
+
+# write the data to a .hdf5 container
+bproc.writer.write_hdf5(args.output_dir, data)