Jira/ep 768/release/4.5.1.230

Agora-C_Sharp-RTC-SDK/Code/Rtc/Impl/Private/Impl/RtcEngineImpl.cs

@@ -837,9 +837,9 @@ public MediaRecorderImpl GetMediaRecorder()
         }
 
 #if UNITY_EDITOR_WIN || UNITY_EDITOR_OSX || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_IOS || UNITY_ANDROID || UNITY_VISIONOS
-        internal IVideoStreamManager GetVideoStreamManager()
+        internal IVideoStreamManager GetVideoStreamManager(TextureManager textureManager)
         {
-            return new VideoStreamManager(this);
+            return new VideoStreamManager(this, textureManager);
         }
 #endif
 

Agora-C_Sharp-RTC-SDK/Code/Rtc/Impl/Private/Native/AgoraRtcApiNative.cs

@@ -111,6 +111,20 @@ internal static extern IRIS_VIDEO_PROCESS_ERR GetVideoFrameCache(IrisRtcRenderin
                                     ref IrisRtcVideoFrameConfig config,
                                     ref IrisCVideoFrame video_frame, out bool is_new_frame
                                  );
+        [DllImport(AgoraRtcLibName, CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern IntPtr CreateVideoFrameEventHandler(IntPtr c_event_handler);
+
+        [DllImport(AgoraRtcLibName, CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern void DestroyVideoFrameEventHandler(IntPtr c_event_handler);
+
+        [DllImport(AgoraRtcLibName, CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern int AddVideoFrameObserverDelegate(IrisRtcRenderingHandle handle,
+            ref IrisRtcVideoFrameConfig config, IntPtr video_frame_handler);
+        [DllImport(AgoraRtcLibName, CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]
+        internal static extern int RemoveVideoFrameObserverDelegate(IrisRtcRenderingHandle handle,
+            int delegate_id);
+
+
 
         internal static void AllocEventHandlerHandle(ref RtcEventHandlerHandle eventHandlerHandle, Rtc_Func_Event_Native onEvent)
         {
@@ -140,6 +154,34 @@ internal static void FreeEventHandlerHandle(ref RtcEventHandlerHandle eventHandl
             eventHandlerHandle.cEvent = new IrisRtcCEventHandler();
         }
 
+        internal static void AllocVideoFrameHandlerHandle(ref VideoFrameEventHandlerHandler eventHandlerHandle, Video_Frame_Event_Native onEvent)
+        {
+            eventHandlerHandle.cEvent = new IrisVideoCEventHandler
+            {
+                OnEvent = onEvent,
+            };
+
+            var cEventHandlerNativeLocal = new IrisRtcCEventHandlerNative
+            {
+                onEvent = Marshal.GetFunctionPointerForDelegate(eventHandlerHandle.cEvent.OnEvent),
+            };
+
+            eventHandlerHandle.marshal = Marshal.AllocHGlobal(Marshal.SizeOf(cEventHandlerNativeLocal));
+            Marshal.StructureToPtr(cEventHandlerNativeLocal, eventHandlerHandle.marshal, true);
+            eventHandlerHandle.handle = AgoraRtcNative.CreateVideoFrameEventHandler(eventHandlerHandle.marshal);
+        }
+
+        internal static void FreeVideoFrameHandlerHandle(ref VideoFrameEventHandlerHandler eventHandlerHandle)
+        {
+            AgoraRtcNative.DestroyVideoFrameEventHandler(eventHandlerHandle.handle);
+            eventHandlerHandle.handle = IntPtr.Zero;
+
+            Marshal.FreeHGlobal(eventHandlerHandle.marshal);
+            eventHandlerHandle.marshal = IntPtr.Zero;
+
+            eventHandlerHandle.cEvent = new IrisVideoCEventHandler();
+        }
+
         #endregion
 
         #region iris_rtc_high_performance_c_api
@@ -240,6 +282,9 @@ internal static extern int IMediaEngine_PushEncodedVideoImage(RtcEngineHandle en
     [UnmanagedFunctionPointer(CallingConvention.Cdecl, CharSet = CharSet.Ansi)]
     internal delegate void Rtc_Func_Event_Native(IntPtr param);
 
+    [UnmanagedFunctionPointer(CallingConvention.Cdecl, CharSet = CharSet.Ansi)]
+    internal delegate void Video_Frame_Event_Native(ref IrisCVideoFrame videoFrame, ref IrisRtcVideoFrameConfig config, bool resize);
+
     [StructLayout(LayoutKind.Sequential)]
     internal struct IrisRtcCEventParam
     {
@@ -326,8 +371,19 @@ internal struct RtcEventHandlerHandle
         internal IrisEventHandlerHandle handle;
     }
 
-    #region callback native
+    internal struct IrisVideoCEventHandler
+    {
+        internal Video_Frame_Event_Native OnEvent;
+    }
+
+    internal struct VideoFrameEventHandlerHandler
+    {
+        internal IrisVideoCEventHandler cEvent;
+        internal IrisEventHandlerMarshal marshal;
+        internal IrisEventHandlerHandle handle;
+    }
 
+    #region callback native
     [StructLayout(LayoutKind.Sequential)]
     public struct IrisSpatialAudioZone
     {

Agora-C_Sharp-RTC-SDK/Code/Rtc/VideoRender/TextureManager.cs

@@ -1,4 +1,3 @@
-#define USE_UNSAFE_CODE
 #if UNITY_EDITOR_WIN || UNITY_EDITOR_OSX || UNITY_STANDALONE_WIN || UNITY_STANDALONE_OSX || UNITY_IOS || UNITY_ANDROID || UNITY_VISIONOS
 using System;
 using System.Runtime.InteropServices;
@@ -9,11 +8,26 @@
 #endif
 
 using UnityEngine;
+using UnityEngine.UIElements;
 
 namespace Agora.Rtc
 {
     public class TextureManager : MonoBehaviour
     {
+        protected class TextureVideoFrame
+        {
+            public int type;
+            public int width;
+            public int height;
+            public int yStride;
+            public int uStride;
+            public int vStride;
+            public byte[] yBuffer;
+            public byte[] uBuffer;
+            public byte[] vBuffer;
+            public byte[] alphaBuffer;
+        }
+
         // texture identity
         protected int _videoPixelWidth = 2;
         protected int _videoPixelHeight = 2;
@@ -40,10 +54,12 @@ public int Height
 
         protected bool _needResize = false;
         protected bool _needUpdateInfo = true;
-        protected bool isFresh = false;
+        protected bool _isFresh = false;
 
         protected IVideoStreamManager _videoStreamManager;
-        protected IrisCVideoFrame _cachedVideoFrame;
+        protected TextureVideoFrame _cachedVideoFrame;
+        internal IrisRtcVideoFrameConfig _videoFrameConfig;
+        protected System.Object _videoFrameLock = new System.Object();
 
         // reference count
         protected int _refCount = 0;
@@ -59,10 +75,6 @@ public Texture2D Texture
             }
         }
 
-#if USE_UNSAFE_CODE && UNITY_2018_1_OR_NEWER
-        protected NativeArray<byte> _textureNative;
-#endif
-
         protected virtual void Awake()
         {
             DontDestroyOnLoad(this.gameObject);
@@ -82,11 +94,10 @@ protected virtual void OnDestroy()
 
             if (_videoStreamManager != null)
             {
-                _videoStreamManager.DisableVideoFrameBuffer(_sourceType, _uid, _channelId, _frameType);
+                _videoStreamManager.RemoveVideoFrameObserverDelegate();
                 _videoStreamManager.Dispose();
                 _videoStreamManager = null;
             }
-
             FreeMemory();
             DestroyTexture();
         }
@@ -106,7 +117,7 @@ internal virtual void InitTexture()
 
         internal virtual void InitIrisVideoFrame()
         {
-            _cachedVideoFrame = new IrisCVideoFrame
+            _cachedVideoFrame = new TextureVideoFrame
             {
                 type = (int)VIDEO_OBSERVER_FRAME_TYPE.FRAME_TYPE_RGBA,
                 yStride = _videoPixelWidth * 4,
@@ -115,18 +126,6 @@ internal virtual void InitIrisVideoFrame()
                 height = _videoPixelHeight,
                 width = _videoPixelWidth
             };
-#if USE_UNSAFE_CODE && UNITY_2018_1_OR_NEWER
-
-            _textureNative = _texture.GetRawTextureData<byte>();
-            unsafe
-            {
-                _cachedVideoFrame.yBuffer = (IntPtr)NativeArrayUnsafeUtility.GetUnsafeBufferPointerWithoutChecks(_textureNative);
-            }
-#else
-
-            _cachedVideoFrame.yBuffer = Marshal.AllocHGlobal(_videoPixelWidth * _videoPixelHeight * 4);
-
-#endif
         }
 
         internal int GetRefCount()
@@ -146,93 +145,59 @@ internal void EnableVideoFrameWithIdentity()
             {
                 if (_videoStreamManager == null)
                 {
-                    _videoStreamManager = ((RtcEngineImpl)engine).GetVideoStreamManager();
+                    _videoStreamManager = ((RtcEngineImpl)engine).GetVideoStreamManager(this);
                 }
 
                 if (_videoStreamManager != null)
                 {
-                    _videoStreamManager.EnableVideoFrameBuffer(_sourceType, _uid, _channelId, _frameType);
+                    _videoFrameConfig = new IrisRtcVideoFrameConfig()
+                    {
+                        video_view_setup_mode = 0,
+                        observed_frame_position = (uint)(VideoStreamManager.position | VIDEO_MODULE_POSITION.POSITION_PRE_RENDERER),
+                        video_source_type = (int)_sourceType,
+                        video_frame_format = (int)_frameType,
+                        uid = _uid,
+                        channelId = _channelId,
+                    };
+                    _videoStreamManager.AddVideoFrameObserverDelegate(ref _videoFrameConfig);
                     _needUpdateInfo = false;
                 }
             }
         }
 
         internal virtual void ReFreshTexture()
         {
-            var ret = _videoStreamManager.GetVideoFrame(ref _cachedVideoFrame, ref isFresh, _sourceType, _uid, _channelId, _frameType);
-
-            if (ret == IRIS_VIDEO_PROCESS_ERR.ERR_NO_CACHE)
-            {
-                _canAttach = false;
-                return;
-            }
+            TimeConsuming.End(1, "TextureRGBA onVideoFame_finsih -> refresh start");
+            TimeConsuming.Start(2);
+            TextureVideoFrame tempVideoFrame = null;
 
-            else if (ret == IRIS_VIDEO_PROCESS_ERR.ERR_RESIZED)
+            lock (_videoFrameLock)
             {
-                _videoPixelWidth = _cachedVideoFrame.width;
-                _videoPixelHeight = _cachedVideoFrame.height;
-
-#if USE_UNSAFE_CODE && UNITY_2018_1_OR_NEWER
-                _cachedVideoFrame.type = (int)VIDEO_OBSERVER_FRAME_TYPE.FRAME_TYPE_RGBA;
-#if UNITY_2021_2_OR_NEWER
-                _texture.Reinitialize(_videoPixelWidth, _videoPixelHeight);
-#else
-                _texture.Resize(_videoPixelWidth, _videoPixelHeight);
-#endif
-                _texture.Apply();
-                _textureNative = _texture.GetRawTextureData<byte>();
-                unsafe
+                if (_isFresh)
                 {
-                    _cachedVideoFrame.yBuffer = (IntPtr)NativeArrayUnsafeUtility.GetUnsafeBufferPointerWithoutChecks(_textureNative);
+                    tempVideoFrame = _cachedVideoFrame;
+                    _isFresh = false;
+                }
+                else
+                {
+                    return;
                 }
-#else
-
-                _needResize = true;
-                FreeMemory();
-                _cachedVideoFrame.type = (int)VIDEO_OBSERVER_FRAME_TYPE.FRAME_TYPE_RGBA;
-                _cachedVideoFrame.yBuffer = Marshal.AllocHGlobal(_videoPixelWidth * _videoPixelHeight * 4);
-                return;
-#endif
-            }
-            else
-            {
-                _canAttach = true;
-            }
-
-
-            if (isFresh == false)
-            {
-                return;
             }
 
-            try
+            _canAttach = true;
+            if (tempVideoFrame.width != _videoPixelWidth || tempVideoFrame.height != _videoPixelHeight)
             {
-#if USE_UNSAFE_CODE && UNITY_2018_1_OR_NEWER
-                _texture.Apply();
-#else
-                if (_needResize)
-                {
+                _videoPixelWidth = tempVideoFrame.width;
+                _videoPixelHeight = tempVideoFrame.height;
 #if UNITY_2021_2_OR_NEWER
-                    _texture.Reinitialize(_videoPixelWidth, _videoPixelHeight);
+                _texture.Reinitialize(_videoPixelWidth, _videoPixelHeight);
 #else
-                    _texture.Resize(_videoPixelWidth, _videoPixelHeight);
-#endif
-                    _texture.Apply();
-                    _needResize = false;
-                }
-
-                _texture.LoadRawTextureData(_cachedVideoFrame.yBuffer,
-                    (int)_videoPixelWidth * (int)_videoPixelHeight * 4);
-                _texture.Apply();
+                _texture.Resize(_videoPixelWidth, _videoPixelHeight);
 #endif
-
-
-            }
-            catch (Exception e)
-            {
-                AgoraLog.Log("Exception e = " + e);
             }
-
+            _texture.LoadRawTextureData(tempVideoFrame.yBuffer);
+            _texture.Apply();
+            TimeConsuming.End(2, "TextureRGBA: refresh start -> refresh end");
         }
 
         internal void SetVideoStreamIdentity(uint uid = 0, string channelId = "",
@@ -273,15 +238,40 @@ protected virtual void DestroyTexture()
 
         private void FreeMemory()
         {
-#if USE_UNSAFE_CODE && UNITY_2018_1_OR_NEWER
-            _cachedVideoFrame.yBuffer = IntPtr.Zero;
-#else
-            if (_cachedVideoFrame.yBuffer != IntPtr.Zero)
+
+        }
+
+        internal virtual void OnVideoFameEvent(ref IrisCVideoFrame videoFrame, ref IrisRtcVideoFrameConfig config, bool resize)
+        {
+            if (_videoFrameConfig.video_source_type != config.video_source_type)
+                return;
+            if (_videoFrameConfig.video_frame_format != config.video_frame_format)
+                return;
+            if (_videoFrameConfig.uid != config.uid)
+                return;
+            if (_videoFrameConfig.channelId != config.channelId)
+                return;
+
+
+            TextureVideoFrame tempVideoFrame = new TextureVideoFrame()
             {
-                Marshal.FreeHGlobal(_cachedVideoFrame.yBuffer);
-                _cachedVideoFrame.yBuffer = IntPtr.Zero;
+                type = (int)videoFrame.type,
+                width = videoFrame.width,
+                height = videoFrame.height,
+                yStride = videoFrame.yStride,
+                uStride = videoFrame.uStride,
+                vStride = videoFrame.vStride
+            };
+            int length = videoFrame.width * videoFrame.height * 4;
+            tempVideoFrame.yBuffer = new byte[length];
+            Marshal.Copy(videoFrame.yBuffer, tempVideoFrame.yBuffer, 0, length);
+            lock (_videoFrameLock)
+            {
+                _cachedVideoFrame = tempVideoFrame;
+                _isFresh = true;
             }
-#endif
+
+            TimeConsuming.Start(1);
         }
 
     }