Expose Bitrate and Change Render Streaming (#6)

- Added Bit Rate
- Added New VR Camera Streaming Component
- Updated Samples with new Prefab Changes

Author: vmohan7

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to com.unity.renderstreaming package will be documented in t
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2021-08-17
+
+### Changed
+
+- **MAJOR** Combined left eye and right eye into a single render texture on one video track to increase encoding performance & maintain synconization between eyes
+- Removed Video Resizing as this implementation needs to be re-evaluated
+- Increased default resolution to 2700 x 1500 (both eyes)
+
+### Added
+
+- Added new VRCamStreaming component that exposes bitrate and framerate encoder parameters
+
 ## [0.3.0] - 2021-08-10
 
 ### Changed

Runtime/Prefabs/Render Streaming Services.prefab

@@ -11,6 +11,7 @@ GameObject:
   - component: {fileID: 1793859182182230346}
   - component: {fileID: 1793859182182230347}
   - component: {fileID: 1270632014}
+  - component: {fileID: 1084652618}
   m_Layer: 0
   m_Name: WebVR Camera
   m_TagString: Untagged
@@ -49,9 +50,7 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   local: 0
   label: 
-  VRCameras:
-  - {fileID: 1793859182864016192}
-  - {fileID: 1793859183251251332}
+  VRCameras: {fileID: 1084652618}
   VRPoseEvent:
     m_PersistentCalls:
       m_Calls:
@@ -106,6 +105,24 @@ MonoBehaviour:
   VRAxisEvent:
     m_PersistentCalls:
       m_Calls: []
+--- !u!114 &1084652618
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 1793859182182230341}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 88dc1ef0a0e7d1b4a897f095b852ab38, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  streamingSize: {x: 2700, y: 1500}
+  leftEye: {fileID: 1793859182864016192}
+  rightEye: {fileID: 1793859183251251332}
+  depth: 0
+  antiAliasing: 1
+  BIT_RATE: 9000000
 --- !u!1 &1793859182664022047
 GameObject:
   m_ObjectHideFlags: 0
@@ -256,7 +273,6 @@ GameObject:
   m_Component:
   - component: {fileID: 1793859182864016220}
   - component: {fileID: 1793859182864016192}
-  - component: {fileID: 1793859182864016198}
   m_Layer: 0
   m_Name: Left Eye Render Streaming Camera
   m_TagString: Untagged
@@ -321,21 +337,6 @@ Camera:
   m_OcclusionCulling: 1
   m_StereoConvergence: 10
   m_StereoSeparation: 0.022
---- !u!114 &1793859182864016198
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 1793859182864016223}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: e3379f8b7ab35724b965e26351c2004e, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  streamingSize: {x: 1440, y: 1600}
-  depth: 0
-  antiAliasing: 2
 --- !u!1 &1793859182968099965
 GameObject:
   m_ObjectHideFlags: 0
@@ -529,7 +530,6 @@ GameObject:
   m_Component:
   - component: {fileID: 1793859183251251333}
   - component: {fileID: 1793859183251251332}
-  - component: {fileID: 1793859183251251328}
   m_Layer: 0
   m_Name: Right Eye Render Streaming Camera
   m_TagString: Untagged
@@ -594,21 +594,6 @@ Camera:
   m_OcclusionCulling: 1
   m_StereoConvergence: 10
   m_StereoSeparation: 0.022
---- !u!114 &1793859183251251328
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 1793859183251251330}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: e3379f8b7ab35724b965e26351c2004e, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  streamingSize: {x: 1440, y: 1600}
-  depth: 0
-  antiAliasing: 2
 --- !u!1 &1793859183478774782
 GameObject:
   m_ObjectHideFlags: 0
@@ -730,9 +715,9 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 31b94e633688a294abc5410d3190b6f5, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  gameID: 
   streams:
-  - {fileID: 1793859182864016198}
-  - {fileID: 1793859183251251328}
+  - {fileID: 1084652618}
   - {fileID: 1793859183478774780}
   - {fileID: 1793859182182230347}
 --- !u!1 &9010668948091610382

Runtime/Scripts/VRBroadcast.cs

@@ -114,9 +114,16 @@ public void OnAddChannel(SignalingEventData data) {
         private string GetGameID(string sdp) {
             string[] parameters = sdp.Split('\n'); //split on new line
             string customParam = parameters[parameters.Length - 2].Split('=')[1]; //get line before last new line and split on =
-            string value = ""; //get dictionary value
-            JsonConvert.DeserializeObject<Dictionary<string, string>>(customParam).TryGetValue("user", out value);
-            return value;
+
+            Dictionary<string, string> jsonMap = JsonConvert.DeserializeObject<Dictionary<string, string>>(customParam);
+            if (jsonMap != null) {
+                string value = ""; //get dictionary value
+                jsonMap.TryGetValue("user", out value);
+                return value;
+            } else {
+                return ""; //TODO: check if this is a valid fail case
+            }
+
         }
     }
 }

Runtime/Scripts/VRCamStream.cs

@@ -0,0 +1,98 @@
+using UnityEngine;
+using Unity.RenderStreaming;
+using Unity.WebRTC;
+
+namespace FusedVR.VRStreaming {
+    public class VRCamStream : VideoStreamBase {
+
+        #region Variables
+        [SerializeField]
+        [Tooltip("The Left Eye of the VR Camera")]
+        private Camera leftEye;
+
+        [SerializeField]
+        [Tooltip("The Right Eye of the VR Camera")]
+        private Camera rightEye;
+
+        [SerializeField]
+        [Tooltip("Defines the depth buffer used for render streaming (0, 16, 24, 32)")]
+        private int depth = 0;
+
+        [SerializeField]
+        [Tooltip("Defines the number of samples for anti-aliasing (1, 2, 4, 8)")]
+        private int antiAliasing = 1;
+
+        [SerializeField]
+        [Tooltip("Defines the default bitrate to be used by the encoders in bits per second")]
+        private ulong BIT_RATE = 9000000; //default bitrate of 9 Mbps
+
+        /// <summary>
+        /// The Main Connection ID that this instance is connected with
+        /// </summary>
+        private string mainConnection = "";
+
+
+        public const uint MAX_FRAMERATE = 90; //default max framerate target
+
+        public override Texture SendTexture => leftEye.targetTexture; //should be the same as right eye
+        #endregion
+
+        #region Events
+        // Start is called before the first frame update
+        void Start() {
+            OnStartedStream += StartStream;
+        }
+
+        private void StartStream(string connectionId) {
+            mainConnection = connectionId;
+            ChangeSendParameters(BIT_RATE, MAX_FRAMERATE);
+        }
+
+        /// <summary>
+        /// Assigns Render Textures to our cameras and then 
+        /// Creates a Video Streaming Track that references the render textures
+        /// </summary>
+        protected override MediaStreamTrack CreateTrack() {
+
+            RenderTextureFormat format = WebRTC.GetSupportedRenderTextureFormat(SystemInfo.graphicsDeviceType);
+            RenderTexture rt = new RenderTexture(streamingSize.x, streamingSize.y, depth, format) {
+                antiAliasing = antiAliasing
+            };
+            rt.Create();
+
+            leftEye.targetTexture = rt;
+            leftEye.rect = new Rect(Vector2.zero, new Vector2(0.5f, 1f));
+
+            rightEye.targetTexture = rt;
+            rightEye.rect = new Rect(new Vector2(0.5f, 0f), new Vector2(0.5f, 1f));
+
+            return new VideoStreamTrack("VR Camera", rt);
+        }
+        #endregion
+
+        #region Public Methods
+        /// <summary>
+        /// Change Parameters associated with encoders for sending data to browser
+        /// </summary>
+        public void ChangeSendParameters(ulong? bitrate , uint? framerate) {
+            if (Senders.TryGetValue(mainConnection, out var sender)) {
+                RTCRtpSendParameters parameters = sender.GetParameters();
+                foreach (var encoding in parameters.encodings) {
+                    if (bitrate != null) {
+                        encoding.minBitrate = bitrate;
+                        encoding.maxBitrate = bitrate;
+                    }
+
+                    if (framerate != null) {
+                        encoding.maxFramerate = framerate;
+                    }
+                }
+
+                sender.SetParameters(parameters);
+            }
+        }
+        #endregion
+
+    }
+}
+

Runtime/Scripts/VRCamStream.cs.meta

@@ -24,7 +24,7 @@ public class VRInputManager : InputChannelReceiverBase {
         /// Camera that are used for VR Render Streaming
         /// </summary>
         [Tooltip("The Cameras that are responsible for VR Render Streaming")]
-        public Camera[] VRCameras;
+        public VRCamStream VRCameras;
 
         #region Constants
         /// <summary>
@@ -126,11 +126,12 @@ protected override void OnMessage(byte[] bytes) {
                     case VRDataType.Display:
                         int width = (int)BitConverter.ToUInt32(bytes, 2);
                         int height = (int)BitConverter.ToUInt32(bytes, 6);
-                        foreach (Camera cam in VRCameras) {
-                            cam.targetTexture.Release();
-                            cam.targetTexture.width = width / 2; // half since the the width covers both eyes
-                            cam.targetTexture.height = height;
-                        }
+                        //foreach (Camera cam in VRCameras) {
+                        //    cam.targetTexture.Release();
+                        //    cam.targetTexture.width = width;
+                        //    cam.targetTexture.height = height;
+                        //}
+                        //TODO: need to find proper way to resize texture based on data so that the video channel updates
 
                         break;
                 }