diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/GraphRunner.cs b/Assets/MediaPipeUnity/Samples/Common/Scripts/GraphRunner.cs index 992f84b65..d75b5c05d 100644 --- a/Assets/MediaPipeUnity/Samples/Common/Scripts/GraphRunner.cs +++ b/Assets/MediaPipeUnity/Samples/Common/Scripts/GraphRunner.cs @@ -189,13 +189,13 @@ protected void AddPacketToInputStream(string streamName, Packet packet) calculatorGraph.AddPacketToInputStream(streamName, packet); } - protected void AddTextureFrameToInputStream(string streamName, TextureFrame textureFrame) + protected void AddTextureFrameToInputStream(string streamName, Experimental.TextureFrame textureFrame, GlContext glContext = null) { latestTimestamp = GetCurrentTimestampMicrosec(); - if (configType == ConfigType.OpenGLES) + if (glContext != null) { - var gpuBuffer = textureFrame.BuildGpuBuffer(GpuManager.GlCalculatorHelper.GetGlContext()); + var gpuBuffer = textureFrame.BuildGpuBuffer(glContext); AddPacketToInputStream(streamName, Packet.CreateGpuBufferAt(gpuBuffer, latestTimestamp)); return; } diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/ImageSourceSolution.cs b/Assets/MediaPipeUnity/Samples/Common/Scripts/ImageSourceSolution.cs deleted file mode 100644 index 375694782..000000000 --- a/Assets/MediaPipeUnity/Samples/Common/Scripts/ImageSourceSolution.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright (c) 2021 homuler -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file or at -// https://opensource.org/licenses/MIT. - -using System.Collections; -using UnityEngine; - -namespace Mediapipe.Unity.Sample -{ - public abstract class ImageSourceSolution : Solution where T : GraphRunner - { - [SerializeField] protected Screen screen; - [SerializeField] protected T graphRunner; - [SerializeField] protected TextureFramePool textureFramePool; - - private Coroutine _coroutine; - - public RunningMode runningMode; - - public long timeoutMillisec - { - get => graphRunner.timeoutMillisec; - set => graphRunner.timeoutMillisec = value; - } - - public override void Play() - { - if (_coroutine != null) - { - Stop(); - } - base.Play(); - _coroutine = StartCoroutine(Run()); - } - - public override void Pause() - { - base.Pause(); - ImageSourceProvider.ImageSource.Pause(); - } - - public override void Resume() - { - base.Resume(); - var _ = StartCoroutine(ImageSourceProvider.ImageSource.Resume()); - } - - public override void Stop() - { - base.Stop(); - StopCoroutine(_coroutine); - ImageSourceProvider.ImageSource.Stop(); - graphRunner.Stop(); - } - - private IEnumerator Run() - { - var graphInitRequest = graphRunner.WaitForInit(runningMode); - var imageSource = ImageSourceProvider.ImageSource; - - yield return imageSource.Play(); - - if (!imageSource.isPrepared) - { - Debug.LogError("Failed to start ImageSource, exiting..."); - yield break; - } - - // Use RGBA32 as the input format. - // TODO: When using GpuBuffer, MediaPipe assumes that the input format is BGRA, so the following code must be fixed. - textureFramePool.ResizeTexture(imageSource.textureWidth, imageSource.textureHeight, TextureFormat.RGBA32); - SetupScreen(imageSource); - - yield return graphInitRequest; - if (graphInitRequest.isError) - { - Debug.LogError(graphInitRequest.error); - yield break; - } - - OnStartRun(); - graphRunner.StartRun(imageSource); - - var waitWhilePausing = new WaitWhile(() => isPaused); - - while (true) - { - if (isPaused) - { - yield return waitWhilePausing; - } - - if (!textureFramePool.TryGetTextureFrame(out var textureFrame)) - { - yield return new WaitForEndOfFrame(); - continue; - } - - // Copy current image to TextureFrame - ReadFromImageSource(imageSource, textureFrame); - AddTextureFrameToInputStream(textureFrame); - yield return new WaitForEndOfFrame(); - - if (runningMode.IsSynchronous()) - { - RenderCurrentFrame(textureFrame); - yield return WaitForNextValue(); - } - } - } - - protected virtual void SetupScreen(ImageSource imageSource) - { - // NOTE: The screen will be resized later, keeping the aspect ratio. - screen.Initialize(imageSource); - } - - protected virtual void RenderCurrentFrame(TextureFrame textureFrame) - { - screen.ReadSync(textureFrame); - } - - protected abstract void OnStartRun(); - - protected abstract void AddTextureFrameToInputStream(TextureFrame textureFrame); - - protected abstract IEnumerator WaitForNextValue(); - } -} diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/LegacySolutionRunner.cs b/Assets/MediaPipeUnity/Samples/Common/Scripts/LegacySolutionRunner.cs new file mode 100644 index 000000000..135a16122 --- /dev/null +++ b/Assets/MediaPipeUnity/Samples/Common/Scripts/LegacySolutionRunner.cs @@ -0,0 +1,65 @@ +// Copyright (c) 2023 homuler +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file or at +// https://opensource.org/licenses/MIT. + +using System.Collections; +using UnityEngine; + +namespace Mediapipe.Unity.Sample +{ + public abstract class LegacySolutionRunner : TaskApiRunner where TGraphRunner : GraphRunner + { + [SerializeField] protected Screen screen; + [SerializeField] protected TGraphRunner graphRunner; + + private Coroutine _coroutine; + + public RunningMode runningMode; + + public long timeoutMillisec + { + get => graphRunner.timeoutMillisec; + set => graphRunner.timeoutMillisec = value; + } + + public override void Play() + { + if (_coroutine != null) + { + Stop(); + } + base.Play(); + _coroutine = StartCoroutine(Run()); + } + + public override void Pause() + { + base.Pause(); + ImageSourceProvider.ImageSource.Pause(); + } + + public override void Resume() + { + base.Resume(); + var _ = StartCoroutine(ImageSourceProvider.ImageSource.Resume()); + } + + public override void Stop() + { + base.Stop(); + StopCoroutine(_coroutine); + ImageSourceProvider.ImageSource.Stop(); + graphRunner.Stop(); + } + + protected abstract IEnumerator Run(); + + protected static void SetupAnnotationController(AnnotationController annotationController, ImageSource imageSource, bool expectedToBeMirrored = false) where T : HierarchicalAnnotation + { + annotationController.isMirrored = expectedToBeMirrored ^ imageSource.isHorizontallyFlipped ^ imageSource.isFrontFacing; + annotationController.rotationAngle = imageSource.rotation.Reverse(); + } + } +} diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/ImageSourceSolution.cs.meta b/Assets/MediaPipeUnity/Samples/Common/Scripts/LegacySolutionRunner.cs.meta similarity index 83% rename from Assets/MediaPipeUnity/Samples/Common/Scripts/ImageSourceSolution.cs.meta rename to Assets/MediaPipeUnity/Samples/Common/Scripts/LegacySolutionRunner.cs.meta index a6af36b65..457329b5f 100644 --- a/Assets/MediaPipeUnity/Samples/Common/Scripts/ImageSourceSolution.cs.meta +++ b/Assets/MediaPipeUnity/Samples/Common/Scripts/LegacySolutionRunner.cs.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: a7573abd3c3f9972e964d5c526c34e0a +guid: ab85c32cd451f15a8ba19394cb3d8fde MonoImporter: externalObjects: {} serializedVersion: 2 diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/Screen.cs b/Assets/MediaPipeUnity/Samples/Common/Scripts/Screen.cs index 642363b07..f83673f9f 100644 --- a/Assets/MediaPipeUnity/Samples/Common/Scripts/Screen.cs +++ b/Assets/MediaPipeUnity/Samples/Common/Scripts/Screen.cs @@ -46,7 +46,7 @@ public void Rotate(RotationAngle rotationAngle) _screen.rectTransform.localEulerAngles = rotationAngle.GetEulerAngles(); } - public void ReadSync(TextureFrame textureFrame) + public void ReadSync(Experimental.TextureFrame textureFrame) { if (!(texture is Texture2D)) { diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/Solution.cs b/Assets/MediaPipeUnity/Samples/Common/Scripts/Solution.cs deleted file mode 100644 index 61b772dba..000000000 --- a/Assets/MediaPipeUnity/Samples/Common/Scripts/Solution.cs +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2021 homuler -// -// Use of this source code is governed by an MIT-style -// license that can be found in the LICENSE file or at -// https://opensource.org/licenses/MIT. - -using System.Collections; -using UnityEngine; - -namespace Mediapipe.Unity.Sample -{ - public abstract class Solution : MonoBehaviour - { - private static readonly string _BootstrapName = nameof(Bootstrap); - - [SerializeField] private GameObject _bootstrapPrefab; - -#pragma warning disable IDE1006 - // TODO: make it static - protected virtual string TAG => GetType().Name; -#pragma warning restore IDE1006 - - public Bootstrap bootstrap { get; private set; } - protected bool isPaused; - - protected virtual IEnumerator Start() - { - bootstrap = FindBootstrap(); - yield return new WaitUntil(() => bootstrap.isFinished); - - Play(); - } - - /// - /// Start the main program from the beginning. - /// - public virtual void Play() - { - isPaused = false; - } - - /// - /// Pause the main program. - /// - public virtual void Pause() - { - isPaused = true; - } - - /// - /// Resume the main program. - /// If the main program has not begun, it'll do nothing. - /// - public virtual void Resume() - { - isPaused = false; - } - - /// - /// Stops the main program. - /// - public virtual void Stop() - { - isPaused = true; - } - - protected static void SetupAnnotationController(AnnotationController annotationController, ImageSource imageSource, bool expectedToBeMirrored = false) where T : HierarchicalAnnotation - { - annotationController.isMirrored = expectedToBeMirrored ^ imageSource.isHorizontallyFlipped ^ imageSource.isFrontFacing; - annotationController.rotationAngle = imageSource.rotation.Reverse(); - } - - protected static void ReadFromImageSource(ImageSource imageSource, TextureFrame textureFrame) - { - var sourceTexture = imageSource.GetCurrentTexture(); - - // For some reason, when the image is coiped on GPU, latency tends to be high. - // So even when OpenGL ES is available, use CPU to copy images. - var textureType = sourceTexture.GetType(); - - if (textureType == typeof(WebCamTexture)) - { - textureFrame.ReadTextureFromOnCPU((WebCamTexture)sourceTexture); - } - else if (textureType == typeof(Texture2D)) - { - textureFrame.ReadTextureFromOnCPU((Texture2D)sourceTexture); - } - else - { - textureFrame.ReadTextureFromOnCPU(sourceTexture); - } - } - - protected Bootstrap FindBootstrap() - { - var bootstrapObj = GameObject.Find(_BootstrapName); - - if (bootstrapObj == null) - { - Debug.Log("Initializing the Bootstrap GameObject"); - bootstrapObj = Instantiate(_bootstrapPrefab); - bootstrapObj.name = _BootstrapName; - DontDestroyOnLoad(bootstrapObj); - } - - return bootstrapObj.GetComponent(); - } - } -} diff --git a/Assets/MediaPipeUnity/Samples/Common/Scripts/Solution.cs.meta b/Assets/MediaPipeUnity/Samples/Common/Scripts/Solution.cs.meta deleted file mode 100644 index 8dabeb8fd..000000000 --- a/Assets/MediaPipeUnity/Samples/Common/Scripts/Solution.cs.meta +++ /dev/null @@ -1,11 +0,0 @@ -fileFormatVersion: 2 -guid: 589adc9f9488f9d8eaee5a408719b452 -MonoImporter: - externalObjects: {} - serializedVersion: 2 - defaultReferences: [] - executionOrder: 0 - icon: {instanceID: 0} - userData: - assetBundleName: - assetBundleVariant: diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/Holistic.unity b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/Holistic.unity index 1d9aba392..e85df4580 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/Holistic.unity +++ b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/Holistic.unity @@ -38,7 +38,6 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.44402248, g: 0.49316555, b: 0.5722324, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -394,7 +393,6 @@ GameObject: - component: {fileID: 275874967} - component: {fileID: 275874969} - component: {fileID: 275874968} - - component: {fileID: 275874970} m_Layer: 0 m_Name: Solution m_TagString: Untagged @@ -454,7 +452,6 @@ MonoBehaviour: type: 3} screen: {fileID: 1375628860} graphRunner: {fileID: 275874968} - textureFramePool: {fileID: 275874970} runningMode: 0 _worldAnnotationArea: {fileID: 1936307554} _poseDetectionAnnotationController: {fileID: 1375628863} @@ -462,19 +459,6 @@ MonoBehaviour: _poseWorldLandmarksAnnotationController: {fileID: 1936307555} _segmentationMaskAnnotationController: {fileID: 1375628865} _poseRoiAnnotationController: {fileID: 1375628861} ---- !u!114 &275874970 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 275874966} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: d5da564da19cb6b7d8e4f97f269edc5d, type: 3} - m_Name: - m_EditorClassIdentifier: - _poolSize: 10 --- !u!1 &298505179 GameObject: m_ObjectHideFlags: 0 @@ -1189,6 +1173,11 @@ PrefabInstance: propertyPath: _solution value: objectReference: {fileID: 275874969} + - target: {fileID: 8571076843237194833, guid: 140d1d2c406167c50819d89f86d9092e, + type: 3} + propertyPath: _taskApiRunner + value: + objectReference: {fileID: 275874969} - target: {fileID: 8571076843525237352, guid: 140d1d2c406167c50819d89f86d9092e, type: 3} propertyPath: m_Name diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingGraph.cs b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingGraph.cs index a880a58b4..9338966fa 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingGraph.cs +++ b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingGraph.cs @@ -173,9 +173,9 @@ public override void Stop() _poseRoiStream = null; } - public void AddTextureFrameToInputStream(TextureFrame textureFrame) + public void AddTextureFrameToInputStream(Experimental.TextureFrame textureFrame, GlContext glContext = null) { - AddTextureFrameToInputStream(_InputStreamName, textureFrame); + AddTextureFrameToInputStream(_InputStreamName, textureFrame, glContext); } public async Task WaitNextAsync() diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingSolution.cs b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingSolution.cs index b61151794..0af499ed8 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingSolution.cs +++ b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/Holistic/HolisticTrackingSolution.cs @@ -6,10 +6,11 @@ using System.Collections; using UnityEngine; +using UnityEngine.Rendering; namespace Mediapipe.Unity.Sample.Holistic { - public class HolisticTrackingSolution : ImageSourceSolution + public class HolisticTrackingSolution : LegacySolutionRunner { [SerializeField] private RectTransform _worldAnnotationArea; [SerializeField] private DetectionAnnotationController _poseDetectionAnnotationController; @@ -18,6 +19,8 @@ public class HolisticTrackingSolution : ImageSourceSolution graphRunner.modelComplexity; @@ -60,14 +63,34 @@ public float minTrackingConfidence set => graphRunner.minTrackingConfidence = value; } - protected override void SetupScreen(ImageSource imageSource) + protected override IEnumerator Run() { - base.SetupScreen(imageSource); + var graphInitRequest = graphRunner.WaitForInit(runningMode); + var imageSource = ImageSourceProvider.ImageSource; + + yield return imageSource.Play(); + + if (!imageSource.isPrepared) + { + Debug.LogError("Failed to start ImageSource, exiting..."); + yield break; + } + + // Use RGBA32 as the input format. + // TODO: When using GpuBuffer, MediaPipe assumes that the input format is BGRA, so the following code must be fixed. + _textureFramePool = new Experimental.TextureFramePool(imageSource.textureWidth, imageSource.textureHeight, TextureFormat.RGBA32, 10); + + // NOTE: The screen will be resized later, keeping the aspect ratio. + screen.Initialize(imageSource); _worldAnnotationArea.localEulerAngles = imageSource.rotation.Reverse().GetEulerAngles(); - } - protected override void OnStartRun() - { + yield return graphInitRequest; + if (graphInitRequest.isError) + { + Debug.LogError(graphInitRequest.error); + yield break; + } + if (!runningMode.IsSynchronous()) { graphRunner.OnPoseDetectionOutput += OnPoseDetectionOutput; @@ -80,33 +103,72 @@ protected override void OnStartRun() graphRunner.OnPoseRoiOutput += OnPoseRoiOutput; } - var imageSource = ImageSourceProvider.ImageSource; SetupAnnotationController(_poseDetectionAnnotationController, imageSource); SetupAnnotationController(_holisticAnnotationController, imageSource); SetupAnnotationController(_poseWorldLandmarksAnnotationController, imageSource); SetupAnnotationController(_segmentationMaskAnnotationController, imageSource); _segmentationMaskAnnotationController.InitScreen(imageSource.textureWidth, imageSource.textureHeight); SetupAnnotationController(_poseRoiAnnotationController, imageSource); - } - protected override void AddTextureFrameToInputStream(TextureFrame textureFrame) - { - graphRunner.AddTextureFrameToInputStream(textureFrame); - } + graphRunner.StartRun(imageSource); - protected override IEnumerator WaitForNextValue() - { - var task = graphRunner.WaitNextAsync(); - yield return new WaitUntil(() => task.IsCompleted); + AsyncGPUReadbackRequest req = default; + var waitUntilReqDone = new WaitUntil(() => req.done); - var result = task.Result; - _poseDetectionAnnotationController.DrawNow(result.poseDetection); - _holisticAnnotationController.DrawNow(result.faceLandmarks, result.poseLandmarks, result.leftHandLandmarks, result.rightHandLandmarks); - _poseWorldLandmarksAnnotationController.DrawNow(result.poseWorldLandmarks); - _segmentationMaskAnnotationController.DrawNow(result.segmentationMask); - _poseRoiAnnotationController.DrawNow(result.poseRoi); + // NOTE: we can share the GL context of the render thread with MediaPipe (for now, only on Android) + var canUseGpuImage = graphRunner.configType == GraphRunner.ConfigType.OpenGLES && GpuManager.GpuResources != null; + using var glContext = canUseGpuImage ? GpuManager.GetGlContext() : null; - result.segmentationMask?.Dispose(); + while (true) + { + if (isPaused) + { + yield return new WaitWhile(() => isPaused); + } + + if (!_textureFramePool.TryGetTextureFrame(out var textureFrame)) + { + yield return new WaitForEndOfFrame(); + continue; + } + + // Copy current image to TextureFrame + if (canUseGpuImage) + { + yield return new WaitForEndOfFrame(); + textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture()); + } + else + { + req = textureFrame.ReadTextureAsync(imageSource.GetCurrentTexture()); + yield return waitUntilReqDone; + + if (req.hasError) + { + Debug.LogError($"Failed to read texture from the image source, exiting..."); + break; + } + } + + graphRunner.AddTextureFrameToInputStream(textureFrame, glContext); + + if (runningMode.IsSynchronous()) + { + screen.ReadSync(textureFrame); + + var task = graphRunner.WaitNextAsync(); + yield return new WaitUntil(() => task.IsCompleted); + + var result = task.Result; + _poseDetectionAnnotationController.DrawNow(result.poseDetection); + _holisticAnnotationController.DrawNow(result.faceLandmarks, result.poseLandmarks, result.leftHandLandmarks, result.rightHandLandmarks); + _poseWorldLandmarksAnnotationController.DrawNow(result.poseWorldLandmarks); + _segmentationMaskAnnotationController.DrawNow(result.segmentationMask); + _poseRoiAnnotationController.DrawNow(result.poseRoi); + + result.segmentationMask?.Dispose(); + } + } } private void OnPoseDetectionOutput(object stream, OutputStream.OutputEventArgs eventArgs) diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipe Video.unity b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipe Video.unity index d16ba4391..d4e3cc5fc 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipe Video.unity +++ b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipe Video.unity @@ -38,7 +38,6 @@ RenderSettings: m_ReflectionIntensity: 1 m_CustomReflection: {fileID: 0} m_Sun: {fileID: 0} - m_IndirectSpecularColor: {r: 0.44402248, g: 0.49316555, b: 0.5722324, a: 1} m_UseRadianceAmbientProbe: 0 --- !u!157 &3 LightmapSettings: @@ -258,7 +257,6 @@ MonoBehaviour: type: 3} screen: {fileID: 1567625569} graphRunner: {fileID: 1064799460} - textureFramePool: {fileID: 1064799463} runningMode: 1 --- !u!114 &1064799463 MonoBehaviour: @@ -772,6 +770,11 @@ PrefabInstance: propertyPath: _solution value: objectReference: {fileID: 1064799462} + - target: {fileID: 8571076843237194833, guid: 140d1d2c406167c50819d89f86d9092e, + type: 3} + propertyPath: _taskApiRunner + value: + objectReference: {fileID: 1064799462} - target: {fileID: 8571076843980202029, guid: 140d1d2c406167c50819d89f86d9092e, type: 3} propertyPath: contents diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoGraph.cs b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoGraph.cs index aed02d34b..800a0e337 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoGraph.cs +++ b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoGraph.cs @@ -20,7 +20,7 @@ public class MediaPipeVideoGraph : GraphRunner private Packet _outputGpuBufferPacket; private string _destinationBufferName; - private TextureFrame _destinationTexture; + private Experimental.TextureFrame _destinationTexture; private const string _OutputVideoStreamName = "output_video"; private OutputStream _outputVideoStream; @@ -51,19 +51,19 @@ public override IEnumerator Initialize(RunningMode runningMode) return base.Initialize(runningMode); } - public void SetupOutputPacket(TextureFrame textureFrame) + public void SetupOutputPacket(Experimental.TextureFrame textureFrame, GlContext glContext) { if (configType != ConfigType.OpenGLES) { throw new InvalidOperationException("This method is only supported for OpenGL ES"); } _destinationTexture = textureFrame; - _outputGpuBufferPacket = Packet.CreateGpuBuffer(_destinationTexture.BuildGpuBuffer(GpuManager.GlCalculatorHelper.GetGlContext())); + _outputGpuBufferPacket = Packet.CreateGpuBuffer(_destinationTexture.BuildGpuBuffer(glContext)); } - public void AddTextureFrameToInputStream(TextureFrame textureFrame) + public void AddTextureFrameToInputStream(Experimental.TextureFrame textureFrame, GlContext glContext = null) { - AddTextureFrameToInputStream(_InputStreamName, textureFrame); + AddTextureFrameToInputStream(_InputStreamName, textureFrame, glContext); } public async Task WaitNextAsync() diff --git a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoSolution.cs b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoSolution.cs index 922d1c482..2422977f1 100644 --- a/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoSolution.cs +++ b/Assets/MediaPipeUnity/Samples/Scenes/Legacy/MediaPipe Video/MediaPipeVideoSolution.cs @@ -6,27 +6,49 @@ using System.Collections; using UnityEngine; +using UnityEngine.Rendering; namespace Mediapipe.Unity.Sample.MediaPipeVideo { - public class MediaPipeVideoSolution : ImageSourceSolution + public class MediaPipeVideoSolution : LegacySolutionRunner { private Texture2D _outputTexture; + private Experimental.TextureFramePool _textureFramePool; - protected override void SetupScreen(ImageSource imageSource) + protected override IEnumerator Run() { + var graphInitRequest = graphRunner.WaitForInit(runningMode); + var imageSource = ImageSourceProvider.ImageSource; + + yield return imageSource.Play(); + + if (!imageSource.isPrepared) + { + Debug.LogError("Failed to start ImageSource, exiting..."); + yield break; + } + + // Use RGBA32 as the input format. + // TODO: When using GpuBuffer, MediaPipe assumes that the input format is BGRA, so the following code must be fixed. + _textureFramePool = new Experimental.TextureFramePool(imageSource.textureWidth, imageSource.textureHeight, TextureFormat.RGBA32, 10); + // NOTE: The screen will be resized later, keeping the aspect ratio. screen.Resize(imageSource.textureWidth, imageSource.textureHeight); screen.Rotate(imageSource.rotation.Reverse()); + // NOTE: we can share the GL context of the render thread with MediaPipe (for now, only on Android) + var canUseGpuImage = graphRunner.configType == GraphRunner.ConfigType.OpenGLES && GpuManager.GpuResources != null; + using var glContext = canUseGpuImage ? GpuManager.GetGlContext() : null; + // Setup output texture - if (graphRunner.configType == GraphRunner.ConfigType.OpenGLES) + if (canUseGpuImage) { - if (textureFramePool.TryGetTextureFrame(out var textureFrame)) + if (_textureFramePool.TryGetTextureFrame(out var textureFrame)) { textureFrame.RemoveAllReleaseListeners(); - graphRunner.SetupOutputPacket(textureFrame); + graphRunner.SetupOutputPacket(textureFrame, glContext); + // MediaPipe will write the result to the textureFrame screen.texture = Texture2D.CreateExternalTexture(textureFrame.width, textureFrame.height, textureFrame.format, false, false, textureFrame.GetNativeTexturePtr()); } else @@ -39,43 +61,70 @@ protected override void SetupScreen(ImageSource imageSource) _outputTexture = new Texture2D(imageSource.textureWidth, imageSource.textureHeight, TextureFormat.RGBA32, false); screen.texture = _outputTexture; } - } - protected override void OnStartRun() - { - // Do nothing - } - - protected override void AddTextureFrameToInputStream(TextureFrame textureFrame) - { - graphRunner.AddTextureFrameToInputStream(textureFrame); - } - - protected override void RenderCurrentFrame(TextureFrame textureFrame) - { - // Do nothing because the screen will be updated later in `DrawNow`. - } - - protected override IEnumerator WaitForNextValue() - { - if (graphRunner.configType == GraphRunner.ConfigType.OpenGLES) + yield return graphInitRequest; + if (graphInitRequest.isError) { + Debug.LogError(graphInitRequest.error); yield break; } - var task = graphRunner.WaitNextAsync(); - yield return new WaitUntil(() => task.IsCompleted); + graphRunner.StartRun(imageSource); - DrawNow(task.Result); - task.Result?.Dispose(); - } + AsyncGPUReadbackRequest req = default; + var waitUntilReqDone = new WaitUntil(() => req.done); - private void DrawNow(ImageFrame imageFrame) - { - if (imageFrame != null) + while (true) { - _outputTexture.LoadRawTextureData(imageFrame.MutablePixelData(), imageFrame.PixelDataSize()); - _outputTexture.Apply(); + if (isPaused) + { + yield return new WaitWhile(() => isPaused); + } + + if (!_textureFramePool.TryGetTextureFrame(out var textureFrame)) + { + yield return new WaitForEndOfFrame(); + continue; + } + + // Copy current image to TextureFrame + if (canUseGpuImage) + { + yield return new WaitForEndOfFrame(); + textureFrame.ReadTextureOnGPU(imageSource.GetCurrentTexture()); + } + else + { + req = textureFrame.ReadTextureAsync(imageSource.GetCurrentTexture()); + yield return waitUntilReqDone; + + if (req.hasError) + { + Debug.LogError($"Failed to read texture from the image source, exiting..."); + break; + } + } + + graphRunner.AddTextureFrameToInputStream(textureFrame, glContext); + + if (graphRunner.configType == GraphRunner.ConfigType.OpenGLES) + { + continue; + } + + if (runningMode.IsSynchronous()) + { + var task = graphRunner.WaitNextAsync(); + yield return new WaitUntil(() => task.IsCompleted); + + var imageFrame = task.Result; + if (imageFrame != null) + { + _outputTexture.LoadRawTextureData(imageFrame.MutablePixelData(), imageFrame.PixelDataSize()); + _outputTexture.Apply(); + imageFrame.Dispose(); + } + } } } } diff --git a/Assets/MediaPipeUnity/Samples/UI/Scripts/ImageSourceConfig.cs b/Assets/MediaPipeUnity/Samples/UI/Scripts/ImageSourceConfig.cs index 335ecd2e8..4ef264e28 100644 --- a/Assets/MediaPipeUnity/Samples/UI/Scripts/ImageSourceConfig.cs +++ b/Assets/MediaPipeUnity/Samples/UI/Scripts/ImageSourceConfig.cs @@ -7,7 +7,6 @@ using System; using System.Collections.Generic; using System.Linq; -using UnityEngine; using UnityEngine.UI; namespace Mediapipe.Unity.Sample.UI @@ -19,7 +18,6 @@ public class ImageSourceConfig : ModalContents private const string _ResolutionPath = "Scroll View/Viewport/Contents/Resolution/Dropdown"; private const string _IsHorizontallyFlippedPath = "Scroll View/Viewport/Contents/IsHorizontallyFlipped/Toggle"; - private Solution _solution; private Dropdown _sourceTypeInput; private Dropdown _sourceInput; private Dropdown _resolutionInput; @@ -29,7 +27,6 @@ public class ImageSourceConfig : ModalContents private void Start() { - _solution = GameObject.Find("Solution").GetComponent(); InitializeContents(); } diff --git a/Assets/MediaPipeUnity/Samples/UI/Scripts/Modal.cs b/Assets/MediaPipeUnity/Samples/UI/Scripts/Modal.cs index 5d5013b39..e76af8ff2 100644 --- a/Assets/MediaPipeUnity/Samples/UI/Scripts/Modal.cs +++ b/Assets/MediaPipeUnity/Samples/UI/Scripts/Modal.cs @@ -10,7 +10,6 @@ namespace Mediapipe.Unity.Sample.UI { public class Modal : MonoBehaviour { - [SerializeField] private Solution _solution; // TODO: remove this field [SerializeField] private TaskApiRunner _taskApiRunner; private GameObject _contents; @@ -25,10 +24,6 @@ public void Open(GameObject contents) public void OpenAndPause(GameObject contents) { Open(contents); - if (_solution != null) - { - _solution.Pause(); - } if (_taskApiRunner != null) { _taskApiRunner.Pause(); @@ -49,17 +44,13 @@ public void CloseAndResume(bool forceRestart = false) { Close(); - if (_solution == null && _taskApiRunner == null) + if (_taskApiRunner == null) { return; } if (forceRestart) { - if (_solution != null) - { - _solution.Play(); - } if (_taskApiRunner != null) { _taskApiRunner.Play(); @@ -67,10 +58,6 @@ public void CloseAndResume(bool forceRestart = false) } else { - if (_solution != null) - { - _solution.Resume(); - } if (_taskApiRunner != null) { _taskApiRunner.Resume();