Use 3 bytes, fix mypy/ruff

examples/17_background_composite.py

@@ -24,7 +24,7 @@
 
 mesh = trimesh.creation.box((0.5, 0.5, 0.5))
 server.add_mesh_trimesh(
-    name=f"/cube",
+    name="/cube",
     mesh=mesh,
     position=(0, 0, 0.0),
 )

viser/_message_api.py

@@ -398,27 +398,28 @@ def set_background_image(
         jpeg_quality: Optional[int] = None,
         depth: Optional[onp.ndarray] = None,
     ) -> None:
-        """Set a background image for the scene. Useful for NeRF visualization."""
+        """Set a background image for the scene, optionally with depth compositing."""
         media_type, base64_data = _encode_image_base64(
             image, format, jpeg_quality=jpeg_quality
         )
 
-        # Encode depth if provided. We use a 4-channel PNG to represent a 4-byte (fixed
-        # point) depth at each pixel.
+        # Encode depth if provided. We use a 3-channel PNG to represent a fixed point
+        # depth at each pixel.
         depth_base64data = None
         if depth is not None:
             # Convert to fixed-point.
-            # We'll support from 0 -> (2^32 - 1) / 1_000_000.
+            # We'll support from 0 -> (2^24 - 1) / 100_000.
             #
-            # This translates to a range of [0, 4294.96], with a precision of 1e-6.
+            # This translates to a range of [0, 167.77215], with a precision of 1e-5.
             assert len(depth.shape) == 2 or (
                 len(depth.shape) == 3 and depth.shape[2] == 1
             ), "Depth should have shape (H,W) or (H,W,1)."
-            depth = onp.clip(depth * 1_000_000, 0, 2**32 - 1).astype(onp.uint32)
-            intdepth = depth.reshape((*depth.shape[:2], 1)).view(onp.uint8)
+            depth = onp.clip(depth * 100_000, 0, 2**24 - 1).astype(onp.uint32)
+            assert depth is not None  # Appease mypy.
+            intdepth: onp.ndarray = depth.reshape((*depth.shape[:2], 1)).view(onp.uint8)
             assert intdepth.shape == (*depth.shape[:2], 4)
             with io.BytesIO() as data_buffer:
-                iio.imwrite(data_buffer, intdepth, extension=".png")
+                iio.imwrite(data_buffer, intdepth[:, :, :3], extension=".png")
                 depth_base64data = base64.b64encode(data_buffer.getvalue()).decode(
                     "ascii"
                 )

viser/client/src/App.tsx

@@ -209,11 +209,12 @@ function BackgroundImage() {
   uniform bool enabled;
   uniform bool hasDepth;
 
-  float readDepth( sampler2D depthSampler, vec2 coord, float zNear, float zFar) {
-    vec4 rgbaPacked = texture(depthSampler, coord);
-    // For the k-th channel, coefficients are calculated as: 255 * 1e-6 * 2^(8 * k).
-    // Note that: [0, 255] channels are scaled to [0, 1], and we multiply by 1e6 on the server side.
-    float depth = (rgbaPacked.r * 0.000255 + rgbaPacked.g * 0.06528 + rgbaPacked.b * 16.71168 + rgbaPacked.a * 4278.19008);
+  float readDepth(sampler2D depthMap, vec2 coord) {
+    vec4 rgbPacked = texture(depthMap, coord);
+
+    // For the k-th channel, coefficients are calculated as: 255 * 1e-5 * 2^(8 * k).
+    // Note that: [0, 255] channels are scaled to [0, 1], and we multiply by 1e5 on the server side.
+    float depth = rgbPacked.r * 0.00255 + rgbPacked.g * 0.6528 + rgbPacked.b * 167.1168;
     return depth;
   }
 
@@ -227,10 +228,10 @@ function BackgroundImage() {
 
     float bufDepth;
     if(hasDepth){
-      float depth = readDepth(depthMap, vUv, cameraNear, cameraFar);
+      float depth = readDepth(depthMap, vUv);
       bufDepth = viewZToPerspectiveDepth(-depth, cameraNear, cameraFar);
-    }else{
-      // If no depth enabled, set depth to 1.0 (infinity) to treat it like a background image
+    } else {
+      // If no depth enabled, set depth to 1.0 (infinity) to treat it like a background image.
       bufDepth = 1.0;
     }
     gl_FragDepth = bufDepth;

viser/client/src/WebsocketInterface.tsx

@@ -437,7 +437,7 @@ function useMessageHandler() {
         if (message.base64_depth !== null) {
           // If depth is available set the texture
           new TextureLoader().load(
-            `data:$image/png;base64,${message.base64_depth}`,
+            `data:image/png;base64,${message.base64_depth}`,
             (texture) => {
               // TODO: this onLoad callback prevents flickering, but could cause messages to be handled slightly out-of-order.);
               const olddepthtex =