diff --git a/examples/17_background_composite.py b/examples/17_background_composite.py index fc02d9c11..9f4022e12 100644 --- a/examples/17_background_composite.py +++ b/examples/17_background_composite.py @@ -24,7 +24,7 @@ mesh = trimesh.creation.box((0.5, 0.5, 0.5)) server.add_mesh_trimesh( - name=f"/cube", + name="/cube", mesh=mesh, position=(0, 0, 0.0), ) diff --git a/viser/_message_api.py b/viser/_message_api.py index 2216fea05..d76f9f3ae 100644 --- a/viser/_message_api.py +++ b/viser/_message_api.py @@ -398,27 +398,28 @@ def set_background_image( jpeg_quality: Optional[int] = None, depth: Optional[onp.ndarray] = None, ) -> None: - """Set a background image for the scene. Useful for NeRF visualization.""" + """Set a background image for the scene, optionally with depth compositing.""" media_type, base64_data = _encode_image_base64( image, format, jpeg_quality=jpeg_quality ) - # Encode depth if provided. We use a 4-channel PNG to represent a 4-byte (fixed - # point) depth at each pixel. + # Encode depth if provided. We use a 3-channel PNG to represent a fixed point + # depth at each pixel. depth_base64data = None if depth is not None: # Convert to fixed-point. - # We'll support from 0 -> (2^32 - 1) / 1_000_000. + # We'll support from 0 -> (2^24 - 1) / 100_000. # - # This translates to a range of [0, 4294.96], with a precision of 1e-6. + # This translates to a range of [0, 167.77215], with a precision of 1e-5. assert len(depth.shape) == 2 or ( len(depth.shape) == 3 and depth.shape[2] == 1 ), "Depth should have shape (H,W) or (H,W,1)." - depth = onp.clip(depth * 1_000_000, 0, 2**32 - 1).astype(onp.uint32) - intdepth = depth.reshape((*depth.shape[:2], 1)).view(onp.uint8) + depth = onp.clip(depth * 100_000, 0, 2**24 - 1).astype(onp.uint32) + assert depth is not None # Appease mypy. + intdepth: onp.ndarray = depth.reshape((*depth.shape[:2], 1)).view(onp.uint8) assert intdepth.shape == (*depth.shape[:2], 4) with io.BytesIO() as data_buffer: - iio.imwrite(data_buffer, intdepth, extension=".png") + iio.imwrite(data_buffer, intdepth[:, :, :3], extension=".png") depth_base64data = base64.b64encode(data_buffer.getvalue()).decode( "ascii" ) diff --git a/viser/client/src/App.tsx b/viser/client/src/App.tsx index d204e8b6e..6be284efd 100644 --- a/viser/client/src/App.tsx +++ b/viser/client/src/App.tsx @@ -209,11 +209,12 @@ function BackgroundImage() { uniform bool enabled; uniform bool hasDepth; - float readDepth( sampler2D depthSampler, vec2 coord, float zNear, float zFar) { - vec4 rgbaPacked = texture(depthSampler, coord); - // For the k-th channel, coefficients are calculated as: 255 * 1e-6 * 2^(8 * k). - // Note that: [0, 255] channels are scaled to [0, 1], and we multiply by 1e6 on the server side. - float depth = (rgbaPacked.r * 0.000255 + rgbaPacked.g * 0.06528 + rgbaPacked.b * 16.71168 + rgbaPacked.a * 4278.19008); + float readDepth(sampler2D depthMap, vec2 coord) { + vec4 rgbPacked = texture(depthMap, coord); + + // For the k-th channel, coefficients are calculated as: 255 * 1e-5 * 2^(8 * k). + // Note that: [0, 255] channels are scaled to [0, 1], and we multiply by 1e5 on the server side. + float depth = rgbPacked.r * 0.00255 + rgbPacked.g * 0.6528 + rgbPacked.b * 167.1168; return depth; } @@ -227,10 +228,10 @@ function BackgroundImage() { float bufDepth; if(hasDepth){ - float depth = readDepth(depthMap, vUv, cameraNear, cameraFar); + float depth = readDepth(depthMap, vUv); bufDepth = viewZToPerspectiveDepth(-depth, cameraNear, cameraFar); - }else{ - // If no depth enabled, set depth to 1.0 (infinity) to treat it like a background image + } else { + // If no depth enabled, set depth to 1.0 (infinity) to treat it like a background image. bufDepth = 1.0; } gl_FragDepth = bufDepth; diff --git a/viser/client/src/WebsocketInterface.tsx b/viser/client/src/WebsocketInterface.tsx index ac8557327..c447eb5d2 100644 --- a/viser/client/src/WebsocketInterface.tsx +++ b/viser/client/src/WebsocketInterface.tsx @@ -437,7 +437,7 @@ function useMessageHandler() { if (message.base64_depth !== null) { // If depth is available set the texture new TextureLoader().load( - `data:$image/png;base64,${message.base64_depth}`, + `data:image/png;base64,${message.base64_depth}`, (texture) => { // TODO: this onLoad callback prevents flickering, but could cause messages to be handled slightly out-of-order.); const olddepthtex =