[yfl] add 3dgs rendering backend

Author: yifanlu0227

README.md

@@ -29,6 +29,13 @@ pip install -r requirements.txt
 imageio_download_bin freeimage
 ```
 
+### Step 2: Faster alternative of McNeRF, 3D Gaussians splatting
+This requires the your CUDA NVCC version matches your pytorch cuda version.
+```bash
+cd chatsim/background/gaussian-splatting/
+pip install submodules/simple-knn
+```
+
 ### Step 2: Install McNeRF 
 The installation is the same as [F2-NeRF](https://github.com/totoro97/f2-nerf). Please go through the following steps.
 
@@ -260,7 +267,7 @@ data
         |-- shutters                    # normalized exposure time (mean=0 std=1)
         |-- tracking_info.pkl           # tracking data
         |-- vehi2veh0.npy               # transformation matrix from i-th frame's vehicle coordinate to the first frame's vehicle 
-        |-- sparse/0                    # calibration files from COLMAP (intermediate file, not required by simulation inference)
+        |-- colmap/sparse_undistorted   # calibration files from COLMAP (intermediate file, only required when using 3dgs rendering)
         `-- camera.xml                  # calibration file from Metashape (intermediate file, not required by simulation inference)
         coordinate.
 ```
@@ -275,7 +282,7 @@ data
 **`cams_meta.npy` instruction**
 ```
 cams_meta.shape = (N, 27)
-cams_meta[:, 0 :12]: flatten camera poses in RUB
+cams_meta[:, 0 :12]: flatten camera poses in RUB, world coordinate is the starting frame's vehicle coordinate.
 cams_meta[:, 12:21]: flatten camse intrinsics
 cams_meta[:, 21:25]: distortion params [k1, k2, p1, p2]
 cams_meta[:, 25:27]: bounds [z_near, z_far] (not used.)

chatsim/agents/background_rendering_3dgs_agent.py

@@ -0,0 +1,65 @@
+"""
+This is a 3dgs version of our background rendering, which offers real-time performance
+"""
+import numpy as np
+from termcolor import colored
+import imageio.v2 as imageio
+import os
+
+class BackgroundRendering3DGSAgent:
+    def __init__(self, config):
+        self.config = config
+
+        self.is_wide_angle = config["nerf_config"]['is_wide_angle']
+
+        self.gs_dir = config["gs_config"]['gs_dir']
+        self.model_folder = os.path.join(config["gs_config"]['gs_dir'],
+                                         config["gs_config"]["output_folder"],
+                                         config["gs_config"]["gs_model_name"])
+        self.gs_novel_view_dir = os.path.join(self.model_folder, "chatsim_novel_views")
+
+    def func_render_background(self, scene):
+        """
+        Call the NeRF, store results in scene.current_images
+        """
+        # first update scene.is_ego_motion
+        scene.is_ego_motion = not np.all(scene.current_extrinsics == scene.current_extrinsics[0])
+
+        if scene.is_ego_motion:
+            print(f"{colored('[Background Gaussian Splatting]', 'red', attrs=['bold'])} is_ego_motion is True, rendering multiple frames")
+
+            camera_extrinsics = scene.current_extrinsics[:, :3, :] # [N_frames, 3, 4]
+            camera_intrinsics = scene.intrinsics # [3, 3]
+        else:
+            print(f"{colored('[Background Gaussian Splatting]', 'red', attrs=['bold'])} is_ego_motion is False, rendering one frame")
+            camera_extrinsics = scene.current_extrinsics[0:1, :3, :] # [1, 3, 4]
+            camera_intrinsics = scene.intrinsics # [3, 3]
+
+        np.savez(os.path.join(self.model_folder, 'chatsim_extint.npz'), 
+                    camera_extrinsics = camera_extrinsics, 
+                    camera_intrinsics = camera_intrinsics,
+                    H = scene.height, 
+                    W = scene.width
+                )
+
+        # remove previous rendered images
+        if os.path.exists(self.gs_novel_view_dir) and len(os.listdir(self.gs_novel_view_dir)) > 0:
+            os.system(f"rm -r {self.gs_novel_view_dir}/*")
+
+        current_dir = os.getcwd()
+        os.chdir(self.gs_dir) # do not generate intermediate file at root directory
+        render_command = f'python render_chatsim.py \
+                            --model_path {self.model_folder}'
+
+        os.system(render_command)
+        os.chdir(current_dir)
+        
+        scene.current_images = [] # to be updated
+        img_path_list = os.listdir(self.gs_novel_view_dir)
+        img_path_list.sort(key=lambda x:int(x[:-4]))
+
+        for img_path in img_path_list:
+            scene.current_images.append(imageio.imread(os.path.join(self.gs_novel_view_dir, img_path))[:, :scene.width])
+
+        if not scene.is_ego_motion:
+            scene.current_images = scene.current_images * scene.frames

chatsim/background/gaussian-splatting/.gitignore

@@ -0,0 +1,9 @@
+*.pyc
+.vscode
+output
+build
+diff_rasterization/diff_rast.egg-info
+diff_rasterization/dist
+tensorboard_3d
+screenshots
+data

chatsim/background/gaussian-splatting/LICENSE.md

@@ -0,0 +1,91 @@
+Gaussian-Splatting License  
+===========================  
+
+**Inria** and **the Max Planck Institut for Informatik (MPII)** hold all the ownership rights on the *Software* named **gaussian-splatting**.  
+The *Software* is in the process of being registered with the Agence pour la Protection des  
+Programmes (APP).  
+
+The *Software* is still being developed by the *Licensor*.  
+
+*Licensor*'s goal is to allow the research community to use, test and evaluate  
+the *Software*.  
+
+## 1.  Definitions  
+
+*Licensee* means any person or entity that uses the *Software* and distributes  
+its *Work*.  
+
+*Licensor* means the owners of the *Software*, i.e Inria and MPII  
+
+*Software* means the original work of authorship made available under this  
+License ie gaussian-splatting.  
+
+*Work* means the *Software* and any additions to or derivative works of the  
+*Software* that are made available under this License.  
+
+
+## 2.  Purpose  
+This license is intended to define the rights granted to the *Licensee* by  
+Licensors under the *Software*.  
+
+## 3.  Rights granted  
+
+For the above reasons Licensors have decided to distribute the *Software*.  
+Licensors grant non-exclusive rights to use the *Software* for research purposes  
+to research users (both academic and industrial), free of charge, without right  
+to sublicense.. The *Software* may be used "non-commercially", i.e., for research  
+and/or evaluation purposes only.  
+
+Subject to the terms and conditions of this License, you are granted a  
+non-exclusive, royalty-free, license to reproduce, prepare derivative works of,  
+publicly display, publicly perform and distribute its *Work* and any resulting  
+derivative works in any form.  
+
+## 4.  Limitations  
+
+**4.1 Redistribution.** You may reproduce or distribute the *Work* only if (a) you do  
+so under this License, (b) you include a complete copy of this License with  
+your distribution, and (c) you retain without modification any copyright,  
+patent, trademark, or attribution notices that are present in the *Work*.  
+
+**4.2 Derivative Works.** You may specify that additional or different terms apply  
+to the use, reproduction, and distribution of your derivative works of the *Work*  
+("Your Terms") only if (a) Your Terms provide that the use limitation in  
+Section 2 applies to your derivative works, and (b) you identify the specific  
+derivative works that are subject to Your Terms. Notwithstanding Your Terms,  
+this License (including the redistribution requirements in Section 3.1) will  
+continue to apply to the *Work* itself.  
+
+**4.3** Any other use without of prior consent of Licensors is prohibited. Research  
+users explicitly acknowledge having received from Licensors all information  
+allowing to appreciate the adequacy between of the *Software* and their needs and  
+to undertake all necessary precautions for its execution and use.  
+
+**4.4** The *Software* is provided both as a compiled library file and as source  
+code. In case of using the *Software* for a publication or other results obtained  
+through the use of the *Software*, users are strongly encouraged to cite the  
+corresponding publications as explained in the documentation of the *Software*.  
+
+## 5.  Disclaimer  
+
+THE USER CANNOT USE, EXPLOIT OR DISTRIBUTE THE *SOFTWARE* FOR COMMERCIAL PURPOSES  
+WITHOUT PRIOR AND EXPLICIT CONSENT OF LICENSORS. YOU MUST CONTACT INRIA FOR ANY  
+UNAUTHORIZED USE: [email protected] . ANY SUCH ACTION WILL  
+CONSTITUTE A FORGERY. THIS *SOFTWARE* IS PROVIDED "AS IS" WITHOUT ANY WARRANTIES  
+OF ANY NATURE AND ANY EXPRESS OR IMPLIED WARRANTIES, WITH REGARDS TO COMMERCIAL  
+USE, PROFESSIONNAL USE, LEGAL OR NOT, OR OTHER, OR COMMERCIALISATION OR  
+ADAPTATION. UNLESS EXPLICITLY PROVIDED BY LAW, IN NO EVENT, SHALL INRIA OR THE  
+AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR  
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE  
+GOODS OR SERVICES, LOSS OF USE, DATA, OR PROFITS OR BUSINESS INTERRUPTION)  
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT  
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING FROM, OUT OF OR  
+IN CONNECTION WITH THE *SOFTWARE* OR THE USE OR OTHER DEALINGS IN THE *SOFTWARE*.  
+
+## 6.  Files subject to permissive licenses
+The contents of the file ```utils/loss_utils.py``` are based on publicly available code authored by Evan Su, which falls under the permissive MIT license. 
+
+Title: pytorch-ssim\
+Project code: https://github.com/Po-Hsun-Su/pytorch-ssim\
+Copyright Evan Su, 2017\
+License: https://github.com/Po-Hsun-Su/pytorch-ssim/blob/master/LICENSE.txt (MIT)