From 25fc55d1497288a1c57fdc44d142046fbf0f7f6b Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Sun, 13 Apr 2025 13:39:44 +0530 Subject: [PATCH 01/11] Working single pass instanced and multiview rendering in Quest3. Existing functionality for multipass and non VR intact --- package/Runtime/GaussianSplatRenderer.cs | 51 ++++++++++-- package/Runtime/GaussianSplatURPFeature.cs | 65 ++++++++++++--- package/Shaders/GaussianComposite.shader | 50 +++++++++++- package/Shaders/RenderGaussianSplats.shader | 7 +- package/Shaders/SplatUtilities.compute | 88 ++++++++++++++------- 5 files changed, 210 insertions(+), 51 deletions(-) diff --git a/package/Runtime/GaussianSplatRenderer.cs b/package/Runtime/GaussianSplatRenderer.cs index 9e080b97..b895b256 100644 --- a/package/Runtime/GaussianSplatRenderer.cs +++ b/package/Runtime/GaussianSplatRenderer.cs @@ -105,7 +105,7 @@ public bool GatherSplatsForCamera(Camera cam) } // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled - public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) + public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex = -1) { Material matComposite = null; foreach (var kvp in m_ActiveSplats) @@ -134,7 +134,7 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) if (displayMat == null) continue; - gs.SetAssetDataOnMaterial(mpb); + gs.SetAssetDataOnMaterial(mpb, eyeIndex); mpb.SetBuffer(GaussianSplatRenderer.Props.SplatChunks, gs.m_GpuChunks); mpb.SetBuffer(GaussianSplatRenderer.Props.SplatViewData, gs.m_GpuView); @@ -200,7 +200,7 @@ void OnPreCullCamera(Camera cam) m_CommandBuffer.SetGlobalTexture(GaussianSplatRenderer.Props.CameraTargetTexture, BuiltinRenderTextureType.CameraTarget); // add sorting, view calc and drawing commands for each splat object - Material matComposite = SortAndRenderSplats(cam, m_CommandBuffer); + Material matComposite = SortAndRenderSplats(cam, m_CommandBuffer, -1); // compose m_CommandBuffer.BeginSample(s_ProfCompose); @@ -297,6 +297,8 @@ internal static class Props public static readonly int SplatBitsValid = Shader.PropertyToID("_SplatBitsValid"); public static readonly int SplatFormat = Shader.PropertyToID("_SplatFormat"); public static readonly int SplatChunks = Shader.PropertyToID("_SplatChunks"); + public static readonly int EyeIndex = Shader.PropertyToID("_EyeIndex"); + public static readonly int IsStereo = Shader.PropertyToID("_IsStereo"); public static readonly int SplatChunkCount = Shader.PropertyToID("_SplatChunkCount"); public static readonly int SplatViewData = Shader.PropertyToID("_SplatViewData"); public static readonly int OrderBuffer = Shader.PropertyToID("_OrderBuffer"); @@ -328,6 +330,8 @@ internal static class Props public static readonly int SelectionMode = Shader.PropertyToID("_SelectionMode"); public static readonly int SplatPosMouseDown = Shader.PropertyToID("_SplatPosMouseDown"); public static readonly int SplatOtherMouseDown = Shader.PropertyToID("_SplatOtherMouseDown"); + public static readonly int ViewProjMatrixLeft = Shader.PropertyToID("_ViewProjMatrixLeft"); + public static readonly int ViewProjMatrixRight = Shader.PropertyToID("_ViewProjMatrixRight"); } [field: NonSerialized] public bool editModified { get; private set; } @@ -404,7 +408,8 @@ void CreateResourcesForAsset() m_GpuChunksValid = false; } - m_GpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, m_Asset.splatCount, kGpuViewDataSize); + // Double the size to hold both left and right eye data for stereo rendering + m_GpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, m_Asset.splatCount * 2, kGpuViewDataSize); m_GpuIndexBuffer = new GraphicsBuffer(GraphicsBuffer.Target.Index, 36, 2); // cube indices, most often we use only the first quad m_GpuIndexBuffer.SetData(new ushort[] @@ -509,7 +514,7 @@ void SetAssetDataOnCS(CommandBuffer cmb, KernelIndices kernel) cmb.SetComputeBufferParam(cs, kernelIndex, Props.SplatCutouts, m_GpuEditCutouts); } - internal void SetAssetDataOnMaterial(MaterialPropertyBlock mat) + internal void SetAssetDataOnMaterial(MaterialPropertyBlock mat, int eyeIndex) { mat.SetBuffer(Props.SplatPos, m_GpuPosData); mat.SetBuffer(Props.SplatOther, m_GpuOtherData); @@ -517,6 +522,15 @@ internal void SetAssetDataOnMaterial(MaterialPropertyBlock mat) mat.SetTexture(Props.SplatColor, m_GpuColorData); mat.SetBuffer(Props.SplatSelectedBits, m_GpuEditSelected ?? m_GpuPosData); mat.SetBuffer(Props.SplatDeletedBits, m_GpuEditDeleted ?? m_GpuPosData); + if (eyeIndex != -1) + { + mat.SetInteger(Props.EyeIndex, eyeIndex); + mat.SetInteger(Props.IsStereo, 1); + } + else + { + mat.SetInteger(Props.IsStereo, 0); + } mat.SetInt(Props.SplatBitsValid, m_GpuEditSelected != null && m_GpuEditDeleted != null ? 1 : 0); uint format = (uint)m_Asset.posFormat | ((uint)m_Asset.scaleFormat << 8) | ((uint)m_Asset.shFormat << 16); mat.SetInteger(Props.SplatFormat, (int)format); @@ -597,6 +611,28 @@ internal void CalcViewData(CommandBuffer cmb, Camera cam) cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixMV, matView * matO2W); cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixObjectToWorld, matO2W); cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixWorldToObject, matW2O); + bool isStereo = XRSettings.enabled && cam.stereoEnabled && + (XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassInstanced || + XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview); + + if (isStereo) + { + // Get correct stereo matrices for each eye + Matrix4x4 stereoViewLeft = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Left); + Matrix4x4 stereoProjLeft = GL.GetGPUProjectionMatrix(cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Left), true); + Matrix4x4 matVPLeft = stereoProjLeft * stereoViewLeft; + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.ViewProjMatrixLeft, matVPLeft); + + Matrix4x4 stereoViewRight = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Right); + Matrix4x4 stereoProjRight = GL.GetGPUProjectionMatrix(cam.GetStereoProjectionMatrix(Camera.StereoscopicEye.Right), true); + Matrix4x4 matVPRight = stereoProjRight * stereoViewRight; + cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.ViewProjMatrixRight, matVPRight); + cmb.SetComputeIntParam(m_CSSplatUtilities, Props.IsStereo, 1); + } + else + { + cmb.SetComputeIntParam(m_CSSplatUtilities, Props.IsStereo, 0); + } cmb.SetComputeVectorParam(m_CSSplatUtilities, Props.VecScreenParams, screenPar); cmb.SetComputeVectorParam(m_CSSplatUtilities, Props.VecWorldSpaceCameraPos, camPos); @@ -606,7 +642,7 @@ internal void CalcViewData(CommandBuffer cmb, Camera cam) cmb.SetComputeIntParam(m_CSSplatUtilities, Props.SHOnly, m_SHOnly ? 1 : 0); m_CSSplatUtilities.GetKernelThreadGroupSizes((int)KernelIndices.CalcViewData, out uint gsX, out _, out _); - cmb.DispatchCompute(m_CSSplatUtilities, (int)KernelIndices.CalcViewData, (m_GpuView.count + (int)gsX - 1)/(int)gsX, 1, 1); + cmb.DispatchCompute(m_CSSplatUtilities, (int)KernelIndices.CalcViewData, (m_SplatCount + (int)gsX - 1)/(int)gsX, 1, 1); } internal void SortPoints(CommandBuffer cmd, Camera cam, Matrix4x4 matrix) @@ -997,7 +1033,8 @@ public void EditSetSplatCount(int newSplatCount) ClearGraphicsBuffer(newEditSelectedMouseDown); ClearGraphicsBuffer(newEditDeleted); - var newGpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, newSplatCount, kGpuViewDataSize); + // Double the size to hold both left and right eye data + var newGpuView = new GraphicsBuffer(GraphicsBuffer.Target.Structured, newSplatCount * 2, kGpuViewDataSize); InitSortBuffers(newSplatCount); // copy existing data over into new buffers diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index cab17475..1c2cbf33 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -10,6 +10,7 @@ using UnityEngine.Rendering; using UnityEngine.Rendering.Universal; using UnityEngine.Rendering.RenderGraphModule; +using UnityEngine.XR; namespace GaussianSplatting.Runtime { @@ -34,6 +35,7 @@ class PassData internal TextureHandle SourceTexture; internal TextureHandle SourceDepth; internal TextureHandle GaussianSplatRT; + internal bool IsStereo; } public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer frameData) @@ -43,31 +45,74 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer var cameraData = frameData.Get(); var resourceData = frameData.Get(); - RenderTextureDescriptor rtDesc = cameraData.cameraTargetDescriptor; + bool isStereo = XRSettings.enabled && cameraData.camera.stereoEnabled && + (XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassInstanced || + XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview); + RenderTextureDescriptor rtDesc = isStereo? XRSettings.eyeTextureDesc: cameraData.cameraTargetDescriptor; rtDesc.depthBufferBits = 0; rtDesc.msaaSamples = 1; rtDesc.graphicsFormat = GraphicsFormat.R16G16B16A16_SFloat; - var textureHandle = UniversalRenderer.CreateRenderGraphTexture(renderGraph, rtDesc, GaussianSplatRTName, true); + Debug.Log($"isStereo: {isStereo}"); + Debug.Log($"Dimension: {rtDesc.dimension}"); + Debug.Log($"Volume: {rtDesc.volumeDepth}"); + Debug.Log($"stereoRenderingMode: {XRSettings.stereoRenderingMode}"); + TextureDesc desc = resourceData.activeColorTexture.GetDescriptor(renderGraph); + Debug.Log($"xrReady: {desc.dimension}"); + Debug.Log($"rtDesc vrUsage: {rtDesc.vrUsage}"); + + // Create render texture + var gaussianSplatRT = UniversalRenderer.CreateRenderGraphTexture(renderGraph, rtDesc, GaussianSplatRTName, true); passData.CameraData = cameraData; passData.SourceTexture = resourceData.activeColorTexture; passData.SourceDepth = resourceData.activeDepthTexture; - passData.GaussianSplatRT = textureHandle; + passData.GaussianSplatRT = gaussianSplatRT; + passData.IsStereo = isStereo; builder.UseTexture(resourceData.activeColorTexture, AccessFlags.ReadWrite); builder.UseTexture(resourceData.activeDepthTexture); - builder.UseTexture(textureHandle, AccessFlags.Write); + builder.UseTexture(gaussianSplatRT, AccessFlags.ReadWrite); builder.AllowPassCulling(false); builder.SetRenderFunc(static (PassData data, UnsafeGraphContext context) => { var commandBuffer = CommandBufferHelpers.GetNativeCommandBuffer(context.cmd); using var _ = new ProfilingScope(commandBuffer, s_profilingSampler); - commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); - CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, data.SourceDepth, ClearFlag.Color, Color.clear); - Material matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer); - commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); - Blitter.BlitCameraTexture(commandBuffer, data.GaussianSplatRT, data.SourceTexture, matComposite, 0); - commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); + + if (data.IsStereo) + { + // Left eye rendering + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 0); + Material matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer, 0); + + // Right eye rendering + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 1); + GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer, 1); + + matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); + + // Composite to the final target + commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); + commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); + commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 0); + commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 0); // emulate left + commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); + + commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 1); + commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 1); // emulate right + commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); + commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); + } + else + { + // Legacy single-eye rendering for backward compatibility + commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, data.SourceDepth, ClearFlag.Color, Color.clear); + Material matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer); + + commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); + Blitter.BlitCameraTexture(commandBuffer, data.GaussianSplatRT, data.SourceTexture, matComposite, 0); + commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); + } }); } } diff --git a/package/Shaders/GaussianComposite.shader b/package/Shaders/GaussianComposite.shader index be1702c7..efc1fe1a 100644 --- a/package/Shaders/GaussianComposite.shader +++ b/package/Shaders/GaussianComposite.shader @@ -15,26 +15,70 @@ CGPROGRAM #pragma fragment frag #pragma require compute #pragma use_dxc +#pragma require 2darray +#pragma multi_compile_instancing +#pragma require instancing + +// Enable proper multi-compile support for all stereo rendering modes +#pragma multi_compile_local _ UNITY_SINGLE_PASS_STEREO STEREO_INSTANCING_ON STEREO_MULTIVIEW_ON + #include "UnityCG.cginc" struct v2f { float4 vertex : SV_POSITION; + float2 uv : TEXCOORD0; }; -v2f vert (uint vtxID : SV_VertexID) +struct appdata +{ + float4 vertex : POSITION; + float2 uv : TEXCOORD0; + uint vtxID : SV_VertexID; +}; + +v2f vert (appdata v) { v2f o; + uint vtxID = v.vtxID; + float2 quadPos = float2(vtxID&1, (vtxID>>1)&1) * 4.0 - 1.0; - o.vertex = float4(quadPos, 1, 1); + o.vertex = UnityObjectToClipPos(float4(quadPos, 1, 1)); + + o.uv = float2(vtxID&1, (vtxID>>1)&1); return o; } +// Separate textures for left and right eyes +#if defined(UNITY_SINGLE_PASS_STEREO) || defined(STEREO_INSTANCING_ON) || defined(STEREO_MULTIVIEW_ON) +UNITY_DECLARE_TEX2DARRAY(_GaussianSplatRT); +#else Texture2D _GaussianSplatRT; +#endif +int _CustomStereoEyeIndex; half4 frag (v2f i) : SV_Target { - half4 col = _GaussianSplatRT.Load(int3(i.vertex.xy, 0)); + uint eyeIndex = _CustomStereoEyeIndex; + // return float4(eyeIndex == 0 ? 1 : 0, 0, eyeIndex == 1 ? 1 : 0, 1); // Red = left, Blue = right + + // Normalize the pixel coordinates to [0,1] range + float2 normalizedUV = float2(i.vertex.x / _ScreenParams.x, i.vertex.y / _ScreenParams.y); + +#if 1 + half4 col1, col2, col; + + // // Check if using separate eye textures + #if defined(UNITY_SINGLE_PASS_STEREO) || defined(STEREO_INSTANCING_ON) || defined(STEREO_MULTIVIEW_ON) + col1 = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, 0)); + col2 = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, 1)); + col = col2 * eyeIndex + col1 * (1 - eyeIndex); + #else + // Fallback to legacy single-texture approach for backward compatibility + col = _GaussianSplatRT.Load(int3(i.vertex.xy, 0)); + #endif +#endif + col.rgb = GammaToLinearSpace(col.rgb); col.a = saturate(col.a * 1.5); return col; diff --git a/package/Shaders/RenderGaussianSplats.shader b/package/Shaders/RenderGaussianSplats.shader index 540a9f5b..66b25e47 100644 --- a/package/Shaders/RenderGaussianSplats.shader +++ b/package/Shaders/RenderGaussianSplats.shader @@ -31,12 +31,15 @@ struct v2f StructuredBuffer _SplatViewData; ByteAddressBuffer _SplatSelectedBits; uint _SplatBitsValid; - +uint _EyeIndex; +uint _IsStereo; v2f vert (uint vtxID : SV_VertexID, uint instID : SV_InstanceID) { v2f o = (v2f)0; instID = _OrderBuffer[instID]; - SplatViewData view = _SplatViewData[instID]; + uint eyeIndex = _EyeIndex; + uint viewIndex = _IsStereo ? instID * 2 + eyeIndex : instID; + SplatViewData view = _SplatViewData[viewIndex]; float4 centerClipPos = view.pos; bool behindCam = centerClipPos.w <= 0; if (behindCam) diff --git a/package/Shaders/SplatUtilities.compute b/package/Shaders/SplatUtilities.compute index 9aa0a9bf..c7f7dc49 100644 --- a/package/Shaders/SplatUtilities.compute +++ b/package/Shaders/SplatUtilities.compute @@ -81,13 +81,19 @@ void CSCalcDistances (uint3 id : SV_DispatchThreadID) _SplatSortDistances[idx] = FloatToSortableUint(pos.z); } +cbuffer StereoMatrices +{ + float4x4 _ViewProjMatrixLeft; + float4x4 _ViewProjMatrixRight; +}; + RWStructuredBuffer _SplatViewData; float _SplatScale; float _SplatOpacityScale; uint _SHOrder; uint _SHOnly; - +uint _IsStereo; uint _SplatCutoutsCount; #define SPLAT_CUTOUT_TYPE_ELLIPSOID 0 @@ -186,40 +192,18 @@ bool IsSplatCut(float3 pos) return finalCut; } -[numthreads(GROUP_SIZE,1,1)] -void CSCalcViewData (uint3 id : SV_DispatchThreadID) +SplatViewData CalculateEyeViewData(SplatData splat, float3 centerWorldPos, float4x4 viewProjMatrix, bool isDeleted, bool isCut, float splatScale, half opacityScale) { - uint idx = id.x; - if (idx >= _SplatCount) - return; - - SplatData splat = LoadSplatData(idx); SplatViewData view = (SplatViewData)0; - float3 centerWorldPos = mul(_MatrixObjectToWorld, float4(splat.pos,1)).xyz; - float4 centerClipPos = mul(UNITY_MATRIX_VP, float4(centerWorldPos, 1)); - half opacityScale = _SplatOpacityScale; - float splatScale = _SplatScale; - - // deleted? - if (_SplatBitsValid) - { - uint wordIdx = idx / 32; - uint bitIdx = idx & 31; - uint wordVal = _SplatDeletedBits.Load(wordIdx * 4); - if (wordVal & (1 << bitIdx)) - { - centerClipPos.w = 0; - } - } - - // cutouts - if (IsSplatCut(splat.pos)) + // Calculate projection + float4 centerClipPos = mul(viewProjMatrix, float4(centerWorldPos, 1)); + if (isDeleted || isCut) { centerClipPos.w = 0; } - view.pos = centerClipPos; + bool behindCam = centerClipPos.w <= 0; if (!behindCam) { @@ -248,7 +232,53 @@ void CSCalcViewData (uint3 id : SV_DispatchThreadID) view.color.y = (f32tof16(col.b) << 16) | f32tof16(col.a); } - _SplatViewData[idx] = view; + return view; +} + +[numthreads(GROUP_SIZE,1,1)] +void CSCalcViewData (uint3 id : SV_DispatchThreadID) +{ + uint idx = id.x; + if (idx >= _SplatCount) + return; + + SplatData splat = LoadSplatData(idx); + + // Transform to world space + float3 centerWorldPos = mul(_MatrixObjectToWorld, float4(splat.pos,1)).xyz; + float splatScale = _SplatScale; + half opacityScale = _SplatOpacityScale; + + // Check if deleted + bool isDeleted = false; + if (_SplatBitsValid) + { + uint wordIdx = idx / 32; + uint bitIdx = idx & 31; + uint wordVal = _SplatDeletedBits.Load(wordIdx * 4); + if (wordVal & (1 << bitIdx)) + { + isDeleted = true; + } + } + + // Check if cut + bool isCut = IsSplatCut(splat.pos); + + if (_IsStereo) + { + // Calculate view data for both eyes + SplatViewData viewLeft = CalculateEyeViewData(splat, centerWorldPos, _ViewProjMatrixLeft, isDeleted, isCut, splatScale, opacityScale); + SplatViewData viewRight = CalculateEyeViewData(splat, centerWorldPos, _ViewProjMatrixRight, isDeleted, isCut, splatScale, opacityScale); + + // Store both views + _SplatViewData[idx * 2] = viewLeft; + _SplatViewData[idx * 2 + 1] = viewRight; + } + else + { + _SplatViewData[idx] = CalculateEyeViewData(splat, centerWorldPos, UNITY_MATRIX_VP, isDeleted, isCut, splatScale, opacityScale); + } } From c306f8488fd60aeb5968817785106d6e00cded6e Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Sun, 13 Apr 2025 14:16:12 +0530 Subject: [PATCH 02/11] Optimize sorting and rendering splats where only the rendering is now done twice and other steps are done only once. --- package/Runtime/GaussianSplatRenderer.cs | 71 +++++++++++++++++++--- package/Runtime/GaussianSplatURPFeature.cs | 22 ++++--- 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/package/Runtime/GaussianSplatRenderer.cs b/package/Runtime/GaussianSplatRenderer.cs index b895b256..01fa55e4 100644 --- a/package/Runtime/GaussianSplatRenderer.cs +++ b/package/Runtime/GaussianSplatRenderer.cs @@ -31,6 +31,15 @@ class GaussianSplatRenderSystem CommandBuffer m_CommandBuffer; + // Keep track of the prepared splats for stereo rendering + public class PreparedRenderData + { + public Material matComposite; + public List<(GaussianSplatRenderer gs, Material displayMat, MaterialPropertyBlock mpb, int indexCount, int instanceCount, MeshTopology topology)> renderItems = new(); + } + + private PreparedRenderData m_LastPreparedData; + public void RegisterSplat(GaussianSplatRenderer r) { if (m_Splats.Count == 0) @@ -104,10 +113,18 @@ public bool GatherSplatsForCamera(Camera cam) return true; } + // New optimized method that prepares everything once for stereo rendering + // This does the sorting and calculates view data, but doesn't actually render // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled - public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex = -1) + public PreparedRenderData PrepareSplats(Camera cam, CommandBuffer cmb) { + if (m_LastPreparedData == null) + m_LastPreparedData = new PreparedRenderData(); + else + m_LastPreparedData.renderItems.Clear(); + Material matComposite = null; + foreach (var kvp in m_ActiveSplats) { var gs = kvp.Item1; @@ -115,13 +132,13 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex matComposite = gs.m_MatComposite; var mpb = kvp.Item2; - // sort + // Sort the splats var matrix = gs.transform.localToWorldMatrix; if (gs.m_FrameCounter % gs.m_SortNthFrame == 0) gs.SortPoints(cmb, cam, matrix); ++gs.m_FrameCounter; - // cache view + // Prepare material and view data kvp.Item2.Clear(); Material displayMat = gs.m_RenderMode switch { @@ -134,11 +151,10 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex if (displayMat == null) continue; - gs.SetAssetDataOnMaterial(mpb, eyeIndex); + // Set up everything except eye-specific parameters + gs.SetAssetDataOnMaterial(mpb, -1); // -1 for initial setup without eye index mpb.SetBuffer(GaussianSplatRenderer.Props.SplatChunks, gs.m_GpuChunks); - mpb.SetBuffer(GaussianSplatRenderer.Props.SplatViewData, gs.m_GpuView); - mpb.SetBuffer(GaussianSplatRenderer.Props.OrderBuffer, gs.m_GpuSortKeys); mpb.SetFloat(GaussianSplatRenderer.Props.SplatScale, gs.m_SplatScale); mpb.SetFloat(GaussianSplatRenderer.Props.SplatOpacityScale, gs.m_OpacityScale); @@ -148,11 +164,12 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex mpb.SetInteger(GaussianSplatRenderer.Props.DisplayIndex, gs.m_RenderMode == GaussianSplatRenderer.RenderMode.DebugPointIndices ? 1 : 0); mpb.SetInteger(GaussianSplatRenderer.Props.DisplayChunks, gs.m_RenderMode == GaussianSplatRenderer.RenderMode.DebugChunkBounds ? 1 : 0); + // Calculate view data once for stereo (will calculate for both eyes) cmb.BeginSample(s_ProfCalcView); gs.CalcViewData(cmb, cam); cmb.EndSample(s_ProfCalcView); - // draw + // Set up draw parameters int indexCount = 6; int instanceCount = gs.splatCount; MeshTopology topology = MeshTopology.Triangles; @@ -161,11 +178,45 @@ public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex if (gs.m_RenderMode == GaussianSplatRenderer.RenderMode.DebugChunkBounds) instanceCount = gs.m_GpuChunksValid ? gs.m_GpuChunks.count : 0; + // Store the prepared data for rendering later + m_LastPreparedData.renderItems.Add((gs, displayMat, mpb, indexCount, instanceCount, topology)); + } + + m_LastPreparedData.matComposite = matComposite; + return m_LastPreparedData; + } + + // New optimized method that just draws the prepared splats for a specific eye + // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled + public void RenderPreparedSplats(CommandBuffer cmb, int eyeIndex) + { + if (m_LastPreparedData == null || m_LastPreparedData.renderItems.Count == 0) + return; + + foreach (var (gs, displayMat, mpb, indexCount, instanceCount, topology) in m_LastPreparedData.renderItems) + { + // Set the eye index for this specific render + mpb.SetInteger(GaussianSplatRenderer.Props.EyeIndex, eyeIndex); + mpb.SetInteger(GaussianSplatRenderer.Props.IsStereo, 1); + + // Draw cmb.BeginSample(s_ProfDraw); - cmb.DrawProcedural(gs.m_GpuIndexBuffer, matrix, displayMat, 0, topology, indexCount, instanceCount, mpb); + cmb.DrawProcedural(gs.m_GpuIndexBuffer, gs.transform.localToWorldMatrix, displayMat, 0, topology, indexCount, instanceCount, mpb); cmb.EndSample(s_ProfDraw); } - return matComposite; + } + + // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled + public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) + { + // Prepare the splats (sort and calculate view data) + var renderData = PrepareSplats(cam, cmb); + + // Render the prepared splats + RenderPreparedSplats(cmb, -1); + + // Return the composite material + return renderData.matComposite; } // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled @@ -200,7 +251,7 @@ void OnPreCullCamera(Camera cam) m_CommandBuffer.SetGlobalTexture(GaussianSplatRenderer.Props.CameraTargetTexture, BuiltinRenderTextureType.CameraTarget); // add sorting, view calc and drawing commands for each splat object - Material matComposite = SortAndRenderSplats(cam, m_CommandBuffer, -1); + Material matComposite = SortAndRenderSplats(cam, m_CommandBuffer); // compose m_CommandBuffer.BeginSample(s_ProfCompose); diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index 1c2cbf33..8fc9b4dc 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -80,26 +80,32 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer if (data.IsStereo) { - // Left eye rendering + // Clear the render target for both eyes + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear); + + // Prepare the splats once - sort them and calculate view data + var renderData = GaussianSplatRenderSystem.instance.PrepareSplats(data.CameraData.camera, commandBuffer); + + // Render to left eye CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 0); - Material matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer, 0); + GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 0); - // Right eye rendering + // Render to right eye CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 1); - GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer, 1); + GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 1); - matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); - // Composite to the final target commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); + renderData.matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); + commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 0); commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 0); // emulate left - commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); + commandBuffer.DrawProcedural(Matrix4x4.identity, renderData.matComposite, 0, MeshTopology.Triangles, 3, 1); commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 1); commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 1); // emulate right - commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); + commandBuffer.DrawProcedural(Matrix4x4.identity, renderData.matComposite, 0, MeshTopology.Triangles, 3, 1); commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); } else From 25344b4bd56b00ce3507a505ce46386fb9a5c2a8 Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Sun, 13 Apr 2025 15:56:16 +0530 Subject: [PATCH 03/11] Cleanups --- package/Runtime/GaussianSplatURPFeature.cs | 9 +-------- package/Shaders/GaussianComposite.shader | 17 +++-------------- 2 files changed, 4 insertions(+), 22 deletions(-) diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index 8fc9b4dc..6493bf6a 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -52,13 +52,6 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer rtDesc.depthBufferBits = 0; rtDesc.msaaSamples = 1; rtDesc.graphicsFormat = GraphicsFormat.R16G16B16A16_SFloat; - Debug.Log($"isStereo: {isStereo}"); - Debug.Log($"Dimension: {rtDesc.dimension}"); - Debug.Log($"Volume: {rtDesc.volumeDepth}"); - Debug.Log($"stereoRenderingMode: {XRSettings.stereoRenderingMode}"); - TextureDesc desc = resourceData.activeColorTexture.GetDescriptor(renderGraph); - Debug.Log($"xrReady: {desc.dimension}"); - Debug.Log($"rtDesc vrUsage: {rtDesc.vrUsage}"); // Create render texture var gaussianSplatRT = UniversalRenderer.CreateRenderGraphTexture(renderGraph, rtDesc, GaussianSplatRTName, true); @@ -110,7 +103,7 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer } else { - // Legacy single-eye rendering for backward compatibility + // Single-eye rendering commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, data.SourceDepth, ClearFlag.Color, Color.clear); Material matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer); diff --git a/package/Shaders/GaussianComposite.shader b/package/Shaders/GaussianComposite.shader index efc1fe1a..3f9e8998 100644 --- a/package/Shaders/GaussianComposite.shader +++ b/package/Shaders/GaussianComposite.shader @@ -16,8 +16,6 @@ CGPROGRAM #pragma require compute #pragma use_dxc #pragma require 2darray -#pragma multi_compile_instancing -#pragma require instancing // Enable proper multi-compile support for all stereo rendering modes #pragma multi_compile_local _ UNITY_SINGLE_PASS_STEREO STEREO_INSTANCING_ON STEREO_MULTIVIEW_ON @@ -44,8 +42,6 @@ v2f vert (appdata v) float2 quadPos = float2(vtxID&1, (vtxID>>1)&1) * 4.0 - 1.0; o.vertex = UnityObjectToClipPos(float4(quadPos, 1, 1)); - - o.uv = float2(vtxID&1, (vtxID>>1)&1); return o; } @@ -60,24 +56,17 @@ int _CustomStereoEyeIndex; half4 frag (v2f i) : SV_Target { uint eyeIndex = _CustomStereoEyeIndex; - // return float4(eyeIndex == 0 ? 1 : 0, 0, eyeIndex == 1 ? 1 : 0, 1); // Red = left, Blue = right - // Normalize the pixel coordinates to [0,1] range float2 normalizedUV = float2(i.vertex.x / _ScreenParams.x, i.vertex.y / _ScreenParams.y); + half4 col; -#if 1 - half4 col1, col2, col; - - // // Check if using separate eye textures + // Check if using separate eye textures #if defined(UNITY_SINGLE_PASS_STEREO) || defined(STEREO_INSTANCING_ON) || defined(STEREO_MULTIVIEW_ON) - col1 = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, 0)); - col2 = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, 1)); - col = col2 * eyeIndex + col1 * (1 - eyeIndex); + col = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, eyeIndex)); #else // Fallback to legacy single-texture approach for backward compatibility col = _GaussianSplatRT.Load(int3(i.vertex.xy, 0)); #endif -#endif col.rgb = GammaToLinearSpace(col.rgb); col.a = saturate(col.a * 1.5); From 102a41aab5090d72a72405604658f6ffa939b07a Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Sun, 13 Apr 2025 19:53:04 +0530 Subject: [PATCH 04/11] more cleanups --- package/Shaders/GaussianComposite.shader | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/package/Shaders/GaussianComposite.shader b/package/Shaders/GaussianComposite.shader index 3f9e8998..212296e7 100644 --- a/package/Shaders/GaussianComposite.shader +++ b/package/Shaders/GaussianComposite.shader @@ -25,20 +25,17 @@ CGPROGRAM struct v2f { float4 vertex : SV_POSITION; - float2 uv : TEXCOORD0; }; struct appdata { float4 vertex : POSITION; - float2 uv : TEXCOORD0; uint vtxID : SV_VertexID; }; -v2f vert (appdata v) +v2f vert (uint vtxID : SV_VertexID) { v2f o; - uint vtxID = v.vtxID; float2 quadPos = float2(vtxID&1, (vtxID>>1)&1) * 4.0 - 1.0; o.vertex = UnityObjectToClipPos(float4(quadPos, 1, 1)); @@ -55,16 +52,14 @@ Texture2D _GaussianSplatRT; int _CustomStereoEyeIndex; half4 frag (v2f i) : SV_Target { - uint eyeIndex = _CustomStereoEyeIndex; - // Normalize the pixel coordinates to [0,1] range - float2 normalizedUV = float2(i.vertex.x / _ScreenParams.x, i.vertex.y / _ScreenParams.y); - half4 col; - + half4 col; // Check if using separate eye textures #if defined(UNITY_SINGLE_PASS_STEREO) || defined(STEREO_INSTANCING_ON) || defined(STEREO_MULTIVIEW_ON) - col = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, eyeIndex)); + // Normalize the pixel coordinates to [0,1] range + float2 normalizedUV = float2(i.vertex.x / _ScreenParams.x, i.vertex.y / _ScreenParams.y); + col = UNITY_SAMPLE_TEX2DARRAY(_GaussianSplatRT, float3(normalizedUV, _CustomStereoEyeIndex)); #else - // Fallback to legacy single-texture approach for backward compatibility + // single-texture for non-stereo col = _GaussianSplatRT.Load(int3(i.vertex.xy, 0)); #endif From d0450126732c0ada62e823724e98a2da61e8ed7b Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Sun, 13 Apr 2025 21:02:54 +0530 Subject: [PATCH 05/11] Parameter exposed for Sorting Per eye --- package/Editor/GaussianSplatRendererEditor.cs | 4 +- package/Runtime/GaussianSplatRenderer.cs | 17 +++++- package/Runtime/GaussianSplatURPFeature.cs | 54 +++++++++++++------ package/Shaders/RenderGaussianSplats.shader | 8 +-- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/package/Editor/GaussianSplatRendererEditor.cs b/package/Editor/GaussianSplatRendererEditor.cs index f5bad0d8..9f70fbe6 100644 --- a/package/Editor/GaussianSplatRendererEditor.cs +++ b/package/Editor/GaussianSplatRendererEditor.cs @@ -27,6 +27,7 @@ public class GaussianSplatRendererEditor : UnityEditor.Editor SerializedProperty m_PropSHOrder; SerializedProperty m_PropSHOnly; SerializedProperty m_PropSortNthFrame; + SerializedProperty m_PropSortPerEye; SerializedProperty m_PropRenderMode; SerializedProperty m_PropPointDisplaySize; SerializedProperty m_PropCutouts; @@ -67,6 +68,7 @@ public void OnEnable() m_PropSHOrder = serializedObject.FindProperty("m_SHOrder"); m_PropSHOnly = serializedObject.FindProperty("m_SHOnly"); m_PropSortNthFrame = serializedObject.FindProperty("m_SortNthFrame"); + m_PropSortPerEye = serializedObject.FindProperty("m_SortPerEye"); m_PropRenderMode = serializedObject.FindProperty("m_RenderMode"); m_PropPointDisplaySize = serializedObject.FindProperty("m_PointDisplaySize"); m_PropCutouts = serializedObject.FindProperty("m_Cutouts"); @@ -111,7 +113,7 @@ public override void OnInspectorGUI() EditorGUILayout.PropertyField(m_PropSHOrder); EditorGUILayout.PropertyField(m_PropSHOnly); EditorGUILayout.PropertyField(m_PropSortNthFrame); - + EditorGUILayout.PropertyField(m_PropSortPerEye); EditorGUILayout.Space(); GUILayout.Label("Debugging Tweaks", EditorStyles.boldLabel); EditorGUILayout.PropertyField(m_PropRenderMode); diff --git a/package/Runtime/GaussianSplatRenderer.cs b/package/Runtime/GaussianSplatRenderer.cs index 01fa55e4..6b74e2d0 100644 --- a/package/Runtime/GaussianSplatRenderer.cs +++ b/package/Runtime/GaussianSplatRenderer.cs @@ -207,13 +207,13 @@ public void RenderPreparedSplats(CommandBuffer cmb, int eyeIndex) } // ReSharper disable once MemberCanBePrivate.Global - used by HDRP/URP features that are not always compiled - public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb) + public Material SortAndRenderSplats(Camera cam, CommandBuffer cmb, int eyeIndex = -1) { // Prepare the splats (sort and calculate view data) var renderData = PrepareSplats(cam, cmb); // Render the prepared splats - RenderPreparedSplats(cmb, -1); + RenderPreparedSplats(cmb, eyeIndex); // Return the composite material return renderData.matComposite; @@ -260,6 +260,17 @@ void OnPreCullCamera(Camera cam) m_CommandBuffer.EndSample(s_ProfCompose); m_CommandBuffer.ReleaseTemporaryRT(GaussianSplatRenderer.Props.GaussianSplatRT); } + + // Checks if any active splats require per-eye sorting + public bool RequiresPerEyeSorting() + { + foreach (var item in m_ActiveSplats) + { + if (item.Item1.m_SortPerEye) + return true; + } + return false; + } } [ExecuteInEditMode] @@ -288,6 +299,8 @@ public enum RenderMode public bool m_SHOnly; [Range(1,30)] [Tooltip("Sort splats only every N frames")] public int m_SortNthFrame = 1; + [Tooltip("When in VR, sort splats separately for each eye. This increases accuracy but reduces performance.")] + public bool m_SortPerEye = false; public RenderMode m_RenderMode = RenderMode.Splats; [Range(1.0f,15.0f)] public float m_PointDisplaySize = 3.0f; diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index 6493bf6a..6f2c879a 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -73,32 +73,54 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer if (data.IsStereo) { - // Clear the render target for both eyes - CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear); - - // Prepare the splats once - sort them and calculate view data - var renderData = GaussianSplatRenderSystem.instance.PrepareSplats(data.CameraData.camera, commandBuffer); - - // Render to left eye - CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 0); - GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 0); - - // Render to right eye - CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 1); - GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 1); + // Check if any of the active splats require per-eye sorting + bool requiresPerEyeSorting = GaussianSplatRenderSystem.instance.RequiresPerEyeSorting(); + Material matComposite = null; + + if (requiresPerEyeSorting) + { + // Per-eye sorting mode - sort and render each eye separately + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear); + + // Left eye + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 0); + matComposite = GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer, 0); + + // Right eye + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 1); + GaussianSplatRenderSystem.instance.SortAndRenderSplats(data.CameraData.camera, commandBuffer, 1); + } + else + { + // Standard stereo rendering - sort once, render twice + // Clear the render target for both eyes + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear); + + // Prepare the splats once - sort them and calculate view data + var renderData = GaussianSplatRenderSystem.instance.PrepareSplats(data.CameraData.camera, commandBuffer); + + // Render to left eye + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 0); + GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 0); + + // Render to right eye + CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 1); + GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 1); + matComposite = renderData.matComposite; + } // Composite to the final target commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); - renderData.matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); + matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 0); commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 0); // emulate left - commandBuffer.DrawProcedural(Matrix4x4.identity, renderData.matComposite, 0, MeshTopology.Triangles, 3, 1); + commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 1); commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 1); // emulate right - commandBuffer.DrawProcedural(Matrix4x4.identity, renderData.matComposite, 0, MeshTopology.Triangles, 3, 1); + commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); commandBuffer.EndSample(GaussianSplatRenderSystem.s_ProfCompose); } else diff --git a/package/Shaders/RenderGaussianSplats.shader b/package/Shaders/RenderGaussianSplats.shader index 66b25e47..8c1e55dd 100644 --- a/package/Shaders/RenderGaussianSplats.shader +++ b/package/Shaders/RenderGaussianSplats.shader @@ -35,10 +35,10 @@ uint _EyeIndex; uint _IsStereo; v2f vert (uint vtxID : SV_VertexID, uint instID : SV_InstanceID) { - v2f o = (v2f)0; - instID = _OrderBuffer[instID]; - uint eyeIndex = _EyeIndex; - uint viewIndex = _IsStereo ? instID * 2 + eyeIndex : instID; + v2f o = (v2f)0; + instID = _OrderBuffer[instID]; + uint eyeIndex = _EyeIndex; + uint viewIndex = _IsStereo ? instID * 2 + eyeIndex : instID; SplatViewData view = _SplatViewData[viewIndex]; float4 centerClipPos = view.pos; bool behindCam = centerClipPos.w <= 0; From e586ba9a5228d0b2c330a29ffe3d5065c4666681 Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Sun, 13 Apr 2025 21:14:24 +0530 Subject: [PATCH 06/11] review comments --- package/Runtime/GaussianSplatRenderer.cs | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/package/Runtime/GaussianSplatRenderer.cs b/package/Runtime/GaussianSplatRenderer.cs index 6b74e2d0..775235f7 100644 --- a/package/Runtime/GaussianSplatRenderer.cs +++ b/package/Runtime/GaussianSplatRenderer.cs @@ -32,10 +32,20 @@ class GaussianSplatRenderSystem CommandBuffer m_CommandBuffer; // Keep track of the prepared splats for stereo rendering + public struct RenderItem + { + public GaussianSplatRenderer gs; + public Material displayMat; + public MaterialPropertyBlock mpb; + public int indexCount; + public int instanceCount; + public MeshTopology topology; + } + public class PreparedRenderData { public Material matComposite; - public List<(GaussianSplatRenderer gs, Material displayMat, MaterialPropertyBlock mpb, int indexCount, int instanceCount, MeshTopology topology)> renderItems = new(); + public List renderItems = new(); } private PreparedRenderData m_LastPreparedData; @@ -119,9 +129,13 @@ public bool GatherSplatsForCamera(Camera cam) public PreparedRenderData PrepareSplats(Camera cam, CommandBuffer cmb) { if (m_LastPreparedData == null) + { m_LastPreparedData = new PreparedRenderData(); + } else + { m_LastPreparedData.renderItems.Clear(); + } Material matComposite = null; @@ -179,7 +193,7 @@ public PreparedRenderData PrepareSplats(Camera cam, CommandBuffer cmb) instanceCount = gs.m_GpuChunksValid ? gs.m_GpuChunks.count : 0; // Store the prepared data for rendering later - m_LastPreparedData.renderItems.Add((gs, displayMat, mpb, indexCount, instanceCount, topology)); + m_LastPreparedData.renderItems.Add(new RenderItem { gs = gs, displayMat = displayMat, mpb = mpb, indexCount = indexCount, instanceCount = instanceCount, topology = topology }); } m_LastPreparedData.matComposite = matComposite; @@ -193,15 +207,15 @@ public void RenderPreparedSplats(CommandBuffer cmb, int eyeIndex) if (m_LastPreparedData == null || m_LastPreparedData.renderItems.Count == 0) return; - foreach (var (gs, displayMat, mpb, indexCount, instanceCount, topology) in m_LastPreparedData.renderItems) + foreach (var item in m_LastPreparedData.renderItems) { // Set the eye index for this specific render - mpb.SetInteger(GaussianSplatRenderer.Props.EyeIndex, eyeIndex); - mpb.SetInteger(GaussianSplatRenderer.Props.IsStereo, 1); + item.mpb.SetInteger(GaussianSplatRenderer.Props.EyeIndex, eyeIndex); + item.mpb.SetInteger(GaussianSplatRenderer.Props.IsStereo, 1); // Draw cmb.BeginSample(s_ProfDraw); - cmb.DrawProcedural(gs.m_GpuIndexBuffer, gs.transform.localToWorldMatrix, displayMat, 0, topology, indexCount, instanceCount, mpb); + cmb.DrawProcedural(item.gs.m_GpuIndexBuffer, item.gs.transform.localToWorldMatrix, item.displayMat, 0, item.topology, item.indexCount, item.instanceCount, item.mpb); cmb.EndSample(s_ProfDraw); } } From 815384f74dd2a3f9fe36464bcb5793a680e4a46c Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Sun, 13 Apr 2025 21:24:31 +0530 Subject: [PATCH 07/11] minor cleanup --- package/Runtime/GaussianSplatURPFeature.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index 6f2c879a..8b0c296e 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -111,7 +111,6 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer // Composite to the final target commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); - commandBuffer.SetGlobalTexture(s_gaussianSplatRT, data.GaussianSplatRT); matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 0); From 6bc220f8423fbe749fcdc15b1abb81c205c70f36 Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Mon, 14 Apr 2025 00:01:23 +0530 Subject: [PATCH 08/11] Fix for non stereo --- package/Runtime/GaussianSplatRenderer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/Runtime/GaussianSplatRenderer.cs b/package/Runtime/GaussianSplatRenderer.cs index 775235f7..4487220c 100644 --- a/package/Runtime/GaussianSplatRenderer.cs +++ b/package/Runtime/GaussianSplatRenderer.cs @@ -211,7 +211,7 @@ public void RenderPreparedSplats(CommandBuffer cmb, int eyeIndex) { // Set the eye index for this specific render item.mpb.SetInteger(GaussianSplatRenderer.Props.EyeIndex, eyeIndex); - item.mpb.SetInteger(GaussianSplatRenderer.Props.IsStereo, 1); + item.mpb.SetInteger(GaussianSplatRenderer.Props.IsStereo, (eyeIndex == -1) ? 0 : 1); // Draw cmb.BeginSample(s_ProfDraw); From c5276a94f015b155d62b76e7e1c4671665293145 Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Mon, 14 Apr 2025 00:25:35 +0530 Subject: [PATCH 09/11] removing unnecessary UnityObjectToClipPos --- package/Shaders/GaussianComposite.shader | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/Shaders/GaussianComposite.shader b/package/Shaders/GaussianComposite.shader index 212296e7..dbc9d055 100644 --- a/package/Shaders/GaussianComposite.shader +++ b/package/Shaders/GaussianComposite.shader @@ -38,7 +38,7 @@ v2f vert (uint vtxID : SV_VertexID) v2f o; float2 quadPos = float2(vtxID&1, (vtxID>>1)&1) * 4.0 - 1.0; - o.vertex = UnityObjectToClipPos(float4(quadPos, 1, 1)); + o.vertex = float4(quadPos, 1, 1); return o; } From e70381157881f8bd913049a5d1329451924442ee Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Mon, 14 Apr 2025 09:23:16 +0530 Subject: [PATCH 10/11] Add workaround and TODO text for stereo rendering via 2 draw calls. --- package/Runtime/GaussianSplatURPFeature.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index 8b0c296e..387cd0e9 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -99,6 +99,10 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer // Prepare the splats once - sort them and calculate view data var renderData = GaussianSplatRenderSystem.instance.PrepareSplats(data.CameraData.camera, commandBuffer); + // [Quest3] Workaround for stereo rendering. Unity is not able to correctly set unity_stereoEyeIndex when drawing to + // a render texture array, so we need to do it manually. Also, we need to draw the same material twice, + // once for each eye. TODO: Revisit this when Unity fixes the issue. + // Render to left eye CoreUtils.SetRenderTarget(commandBuffer, data.GaussianSplatRT, ClearFlag.Color, Color.clear, 0, CubemapFace.Unknown, 0); GaussianSplatRenderSystem.instance.RenderPreparedSplats(commandBuffer, 0); @@ -112,7 +116,10 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer // Composite to the final target commandBuffer.BeginSample(GaussianSplatRenderSystem.s_ProfCompose); matComposite.SetTexture(s_gaussianSplatRT, data.GaussianSplatRT); - + + // [Quest3] Workaround for stereo rendering. Unity is not able to correctly set unity_stereoEyeIndex when drawing to + // a render texture array, so we need to do it manually. Also, we need to draw the same material twice, + // once for each eye. TODO: Revisit this when Unity fixes the issue. commandBuffer.SetRenderTarget(data.SourceTexture, 0, CubemapFace.Unknown, 0); commandBuffer.SetGlobalInt("_CustomStereoEyeIndex", 0); // emulate left commandBuffer.DrawProcedural(Matrix4x4.identity, matComposite, 0, MeshTopology.Triangles, 3, 1); From 051d245bd29785dbe198a8191a4e0e2e1da38376 Mon Sep 17 00:00:00 2001 From: Arghya Sur Date: Tue, 22 Apr 2025 23:28:19 +0530 Subject: [PATCH 11/11] Editor potential fix --- package/Runtime/GaussianSplatRenderer.cs | 3 ++- package/Runtime/GaussianSplatURPFeature.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package/Runtime/GaussianSplatRenderer.cs b/package/Runtime/GaussianSplatRenderer.cs index 4487220c..02f8e869 100644 --- a/package/Runtime/GaussianSplatRenderer.cs +++ b/package/Runtime/GaussianSplatRenderer.cs @@ -691,7 +691,8 @@ internal void CalcViewData(CommandBuffer cmb, Camera cam) cmb.SetComputeMatrixParam(m_CSSplatUtilities, Props.MatrixWorldToObject, matW2O); bool isStereo = XRSettings.enabled && cam.stereoEnabled && (XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassInstanced || - XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview); + XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview) && + !Application.isEditor; if (isStereo) { diff --git a/package/Runtime/GaussianSplatURPFeature.cs b/package/Runtime/GaussianSplatURPFeature.cs index 387cd0e9..fde72e1d 100644 --- a/package/Runtime/GaussianSplatURPFeature.cs +++ b/package/Runtime/GaussianSplatURPFeature.cs @@ -47,7 +47,8 @@ public override void RecordRenderGraph(RenderGraph renderGraph, ContextContainer bool isStereo = XRSettings.enabled && cameraData.camera.stereoEnabled && (XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassInstanced || - XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview); + XRSettings.stereoRenderingMode == XRSettings.StereoRenderingMode.SinglePassMultiview) && + !Application.isEditor; RenderTextureDescriptor rtDesc = isStereo? XRSettings.eyeTextureDesc: cameraData.cameraTargetDescriptor; rtDesc.depthBufferBits = 0; rtDesc.msaaSamples = 1;