Skip to content

Commit

Permalink
WE GOT IT WORKINGgit add ../../src/viser/client/src/Splatting/git add…
Browse files Browse the repository at this point in the history 
… ../../src/viser/client/src/Splatting/ OMG OMG OMG!
  • Loading branch information
@AntonioMacaronio
AntonioMacaronio committed Oct 24, 2024
1 parent e097109 commit cf7cdad
Show file tree
Hide file tree
Showing 2 changed files with 193 additions and 6 deletions.
17 changes: 14 additions & 3 deletions src/viser/client/src/Splatting/GaussianSplats.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial(
transparent: true,
sh_degree: 0,
textureBuffer: null,
shTextureBuffer: null,
textureT_camera_groups: null,
transitionInState: 0.0,
},
Expand Down Expand Up @@ -180,8 +181,7 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial(
vec2 chol45 = unpackHalf2x16(intBufferData.z);
// Get spherical harmonic terms from the buffer, there are 48 coeffecients per vertex
// uint shTexStart = sortedIndex * 6u;
uint shTexStart = sortedIndex * 0u;
uint shTexStart = sortedIndex * 6u;
ivec2 shTexSize = textureSize(shTextureBuffer, 0);
float sh_coeffs_unpacked[48];
for (int i = 0; i < 6; i++) {
Expand Down Expand Up @@ -280,9 +280,15 @@ const GaussianSplatMaterial = /* @__PURE__ */ shaderMaterial(
vec3 rgb = C0 * sh_coeffs[0];
vec3 pointFive = vec3(0.5, 0.5, 0.5);
vRgba = vec4(rgb + pointFive, float(rgbaUint32 >> uint(24)) / 255.0);
// vRgba = vec4(sh_coeffs[0], 1.0); // im using this to see what the sh_coeffs actually are
//vRgba = vec4(sh_coeffs[0], 1.0); // im using this to see what the sh_coeffs actually are
// if (sh_coeffs[0].x > 1.0 && sh_coeffs[0].x < 1.6) {
// vRgba = vec4(1.0, 0.0, 0.0, 1.0);
// } else {
// vRgba = vec4(1.0, 1.0, 1.0, 1.0);
// }
// rgb = rgb -
// C1 * y * sh_coeffs[1] +
Expand Down Expand Up @@ -409,6 +415,7 @@ function SplatRenderer() {
if (!initializedBufferTexture) {
meshProps.material.uniforms.numGaussians.value = merged.numGaussians;
meshProps.textureBuffer.needsUpdate = true;
meshProps.shTextureBuffer.needsUpdate = true;
initializedBufferTexture = true;
}
};
Expand All @@ -425,6 +432,7 @@ function SplatRenderer() {
React.useEffect(() => {
return () => {
meshProps.textureBuffer.dispose();
meshProps.shTextureBuffer.dispose();
meshProps.geometry.dispose();
meshProps.material.dispose();
postToWorker({ close: true });
Expand Down Expand Up @@ -554,6 +562,7 @@ function mergeGaussianGroups(groupBufferFromName: {
const groupBufferFromNameFiltered = Object.fromEntries(
Object.entries(groupBufferFromName).filter(([key]) => !key.startsWith("sh_buffer_"))
);
// groupBufferFromNameFiltered only contains buffers for the gaussians (not sh buffers)
for (const buffer of Object.values(groupBufferFromNameFiltered)) {
totalBufferLength += buffer.length;
}
Expand Down Expand Up @@ -599,6 +608,7 @@ function mergeGaussianGroups(groupBufferFromName: {
combinedSHBuffer.set(sh_buffer, sh_offset)
sh_offset += sh_buffer.length;
}
console.log(combinedSHBuffer)

const numGroups = Object.keys(groupBufferFromNameFiltered).length;
return { numGaussians, gaussianBuffer, numGroups, groupIndices, combinedSHBuffer};
Expand Down Expand Up @@ -695,6 +705,7 @@ function useGaussianMeshProps(gaussianBuffer: Uint32Array, combinedSHBuffer: Uin
geometry,
material,
textureBuffer,
shTextureBuffer,
sortedIndexAttribute,
textureT_camera_groups,
rowMajorT_camera_groups,
Expand Down
182 changes: 179 additions & 3 deletions src/viser/client/src/Splatting/spherical_harmonics_testing.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -128,14 +128,14 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[-0.18104345 0.54372042 -0.14186212]\n"
"[0.70985258 0.20630107 0.06283242]\n"
]
}
],
Expand All @@ -147,6 +147,182 @@
"print(eval_sh(deg, sh, dirs))\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"import struct\n",
"import numpy as np\n",
"\n",
"def unpackHalf2x16(value):\n",
" \"\"\"The first float is the least significant 16bits, the second is the most significant 16bits.\"\"\"\n",
" # Convert int32 to its binary representation\n",
" binary = format(value, '032b')\n",
" \n",
" # Split the binary string into two 16-bit parts\n",
" binary1 = binary[:16]\n",
" binary2 = binary[16:]\n",
" \n",
" # Convert each 16-bit binary string to an integer\n",
" int1 = int(binary1, 2)\n",
" int2 = int(binary2, 2)\n",
" \n",
" # Use numpy to convert uint16 to float16\n",
" float1 = np.frombuffer(struct.pack('H', int1), dtype=np.float16)[0]\n",
" float2 = np.frombuffer(struct.pack('H', int2), dtype=np.float16)[0]\n",
" \n",
" return float1, float2\n",
"\n",
"def int32_to_rgba(value):\n",
" # Ensure the input is a 32-bit integer\n",
" value = int(value) & 0xFFFFFFFF\n",
" \n",
" # Extract each 8-bit piece\n",
" r = (value >> 24) & 0xFF\n",
" g = (value >> 16) & 0xFF\n",
" b = (value >> 8) & 0xFF\n",
" a = value & 0xFF\n",
" \n",
" return r / 255.0, g / 255.0, b / 255.0, a / 255.0"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(0.803, 0.8213)"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"unpackHalf2x16(980236946)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(-0.00319, 0.786)"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"unpackHalf2x16(2592619082)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"(0.3686274509803922,\n",
" 0.7215686274509804,\n",
" 0.7254901960784313,\n",
" 0.7294117647058823)"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"int32_to_rgba(1589164474)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "c3317264494f4ea997d61191fa24a3c5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(FloatSlider(value=0.8212358, description='Red:', max=1.0, step=0.01), FloatSlider(value=0.80300…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import ipywidgets as widgets\n",
"from IPython.display import display, HTML\n",
"import matplotlib.colors as mcolors\n",
"\n",
"def color_picker(r=0.5, g=0.5, b=0.5):\n",
" def update_color(r, g, b):\n",
" color = mcolors.to_hex([r, g, b])\n",
" color_display.value = f'<div style=\"width: 100px; height: 100px; background-color: {color};\"></div>'\n",
" rgb_display.value = f'RGB: ({r:.2f}, {g:.2f}, {b:.2f})'\n",
" hex_display.value = f'Hex: {color}'\n",
"\n",
" r_slider = widgets.FloatSlider(value=r, min=0, max=1, step=0.01, description='Red:')\n",
" g_slider = widgets.FloatSlider(value=g, min=0, max=1, step=0.01, description='Green:')\n",
" b_slider = widgets.FloatSlider(value=b, min=0, max=1, step=0.01, description='Blue:')\n",
"\n",
" color_display = widgets.HTML()\n",
" rgb_display = widgets.Label()\n",
" hex_display = widgets.Label()\n",
"\n",
" widgets.interactive(update_color, r=r_slider, g=g_slider, b=b_slider)\n",
"\n",
" display(widgets.VBox([r_slider, g_slider, b_slider, color_display, rgb_display, hex_display]))\n",
"\n",
"# Usage\n",
"color_picker(0.8212358, 0.8030037, 0.78623223) # You can change these initial values"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "49ebcdf479814f618fab56eb4f17427b",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"VBox(children=(FloatSlider(value=0.3686274509803922, description='Red:', max=1.0, step=0.01), FloatSlider(valu…"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"color_picker(0.3686274509803922, 0.7215686274509804, 0.7254901960784313)"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down

0 comments on commit cf7cdad

Please sign in to comment.