face blendhapes support

Author: newnon

Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/Face Blendshapes.unity

@@ -17,6 +17,7 @@ public class FaceMeshGraph : GraphRunner
   {
     public int maxNumFaces = 1;
     public bool refineLandmarks = true;
+    public bool blenshapes = false;
 
     private float _minDetectionConfidence = 0.5f;
     public float minDetectionConfidence
@@ -56,17 +57,26 @@ public event EventHandler<OutputEventArgs<List<NormalizedRect>>> OnFaceRectsFrom
       remove => _faceRectsFromDetectionsStream.RemoveListener(value);
     }
 
+    public event EventHandler<OutputEventArgs<ClassificationList>> OnFaceClassificationsFromBlendShapesOutput
+    {
+      add => _faceBlendShapesStream?.AddListener(value);
+      remove => _faceBlendShapesStream?.RemoveListener(value);
+    }
+
     private const string _InputStreamName = "input_video";
 
     private const string _FaceDetectionsStreamName = "face_detections";
     private const string _MultiFaceLandmarksStreamName = "multi_face_landmarks";
     private const string _FaceRectsFromLandmarksStreamName = "face_rects_from_landmarks";
     private const string _FaceRectsFromDetectionsStreamName = "face_rects_from_detections";
+    private const string _FaceBlendShapesStreamName = "blendshapes";
 
     private OutputStream<DetectionVectorPacket, List<Detection>> _faceDetectionsStream;
     private OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>> _multiFaceLandmarksStream;
     private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromLandmarksStream;
     private OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>> _faceRectsFromDetectionsStream;
+    private OutputStream<ClassificationListPacket, ClassificationList> _faceBlendShapesStream;
+  
 
     public override void StartRun(ImageSource imageSource)
     {
@@ -76,6 +86,8 @@ public override void StartRun(ImageSource imageSource)
         _multiFaceLandmarksStream.StartPolling().AssertOk();
         _faceRectsFromLandmarksStream.StartPolling().AssertOk();
         _faceRectsFromDetectionsStream.StartPolling().AssertOk();
+        if (blenshapes)
+          _faceBlendShapesStream.StartPolling().AssertOk();
       }
       StartRun(BuildSidePacket(imageSource));
     }
@@ -90,6 +102,8 @@ public override void Stop()
       _faceRectsFromLandmarksStream = null;
       _faceRectsFromDetectionsStream?.Close();
       _faceRectsFromDetectionsStream = null;
+      _faceBlendShapesStream?.Close();
+      _faceBlendShapesStream = null;
       base.Stop();
     }
 
@@ -99,15 +113,23 @@ public void AddTextureFrameToInputStream(TextureFrame textureFrame)
     }
 
     public bool TryGetNext(out List<Detection> faceDetections, out List<NormalizedLandmarkList> multiFaceLandmarks,
-                           out List<NormalizedRect> faceRectsFromLandmarks, out List<NormalizedRect> faceRectsFromDetections, bool allowBlock = true)
+                           out List<NormalizedRect> faceRectsFromLandmarks, out List<NormalizedRect> faceRectsFromDetections, out ClassificationList faceBlendShapes, bool allowBlock = true)
     {
       var currentTimestampMicrosec = GetCurrentTimestampMicrosec();
       var r1 = TryGetNext(_faceDetectionsStream, out faceDetections, allowBlock, currentTimestampMicrosec);
       var r2 = TryGetNext(_multiFaceLandmarksStream, out multiFaceLandmarks, allowBlock, currentTimestampMicrosec);
       var r3 = TryGetNext(_faceRectsFromLandmarksStream, out faceRectsFromLandmarks, allowBlock, currentTimestampMicrosec);
       var r4 = TryGetNext(_faceRectsFromDetectionsStream, out faceRectsFromDetections, allowBlock, currentTimestampMicrosec);
-
-      return r1 || r2 || r3 || r4;
+      if (blenshapes)
+      {
+        var r5 = TryGetNext(_faceBlendShapesStream, out faceBlendShapes, allowBlock, currentTimestampMicrosec);
+        return r1 || r2 || r3 || r4 || r5;
+      }
+      else
+      {
+        faceBlendShapes = null;
+        return r1 || r2 || r3 || r4;
+      }
     }
 
     protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
@@ -122,13 +144,18 @@ protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
             calculatorGraph, _FaceRectsFromLandmarksStreamName, config.AddPacketPresenceCalculator(_FaceRectsFromLandmarksStreamName), timeoutMicrosec);
         _faceRectsFromDetectionsStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(
             calculatorGraph, _FaceRectsFromDetectionsStreamName, config.AddPacketPresenceCalculator(_FaceDetectionsStreamName), timeoutMicrosec);
+        if (blenshapes)
+          _faceBlendShapesStream = new OutputStream<ClassificationListPacket, ClassificationList>(
+            calculatorGraph, _FaceBlendShapesStreamName, config.AddPacketPresenceCalculator(_FaceBlendShapesStreamName), timeoutMicrosec);
       }
       else
       {
         _faceDetectionsStream = new OutputStream<DetectionVectorPacket, List<Detection>>(calculatorGraph, _FaceDetectionsStreamName, true, timeoutMicrosec);
         _multiFaceLandmarksStream = new OutputStream<NormalizedLandmarkListVectorPacket, List<NormalizedLandmarkList>>(calculatorGraph, _MultiFaceLandmarksStreamName, true, timeoutMicrosec);
         _faceRectsFromLandmarksStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromLandmarksStreamName, true, timeoutMicrosec);
         _faceRectsFromDetectionsStream = new OutputStream<NormalizedRectVectorPacket, List<NormalizedRect>>(calculatorGraph, _FaceRectsFromDetectionsStreamName, true, timeoutMicrosec);
+        if (blenshapes)
+          _faceBlendShapesStream = new OutputStream<ClassificationListPacket, ClassificationList>(calculatorGraph, _FaceBlendShapesStreamName, true, timeoutMicrosec);
       }
 
       using (var validatedGraphConfig = new ValidatedGraphConfig())
@@ -174,10 +201,13 @@ protected override Status ConfigureCalculatorGraph(CalculatorGraphConfig config)
 
     protected override IList<WaitForResult> RequestDependentAssets()
     {
-      return new List<WaitForResult> {
+      var ret = new List<WaitForResult> {
         WaitForAsset("face_detection_short_range.bytes"),
         WaitForAsset(refineLandmarks ? "face_landmark_with_attention.bytes" : "face_landmark.bytes"),
       };
+      if (blenshapes)
+        ret.Add(WaitForAsset("face_blendshapes.bytes"));
+      return ret;
     }
 
     private SidePacket BuildSidePacket(ImageSource imageSource)

Assets/MediaPipeUnity/Samples/Scenes/Face Mesh/Face Blendshapes.unity.meta

@@ -49,6 +49,7 @@ protected override void OnStartRun()
         graphRunner.OnMultiFaceLandmarksOutput += OnMultiFaceLandmarksOutput;
         graphRunner.OnFaceRectsFromLandmarksOutput += OnFaceRectsFromLandmarksOutput;
         graphRunner.OnFaceRectsFromDetectionsOutput += OnFaceRectsFromDetectionsOutput;
+        graphRunner.OnFaceClassificationsFromBlendShapesOutput += OnFaceClassificationsFromBlendShapesOutput;
       }
 
       var imageSource = ImageSourceProvider.ImageSource;
@@ -69,20 +70,23 @@ protected override IEnumerator WaitForNextValue()
       List<NormalizedLandmarkList> multiFaceLandmarks = null;
       List<NormalizedRect> faceRectsFromLandmarks = null;
       List<NormalizedRect> faceRectsFromDetections = null;
+      ClassificationList faceBlendShapes = null;
 
       if (runningMode == RunningMode.Sync)
       {
-        var _ = graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, true);
+        var _ = graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, out faceBlendShapes, true);
       }
       else if (runningMode == RunningMode.NonBlockingSync)
       {
-        yield return new WaitUntil(() => graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, false));
+        yield return new WaitUntil(() => graphRunner.TryGetNext(out faceDetections, out multiFaceLandmarks, out faceRectsFromLandmarks, out faceRectsFromDetections, out faceBlendShapes, false));
       }
 
       _faceDetectionsAnnotationController.DrawNow(faceDetections);
       _multiFaceLandmarksAnnotationController.DrawNow(multiFaceLandmarks);
       _faceRectsFromLandmarksAnnotationController.DrawNow(faceRectsFromLandmarks);
       _faceRectsFromDetectionsAnnotationController.DrawNow(faceRectsFromDetections);
+      if (faceBlendShapes != null)
+        Debug.Log($"Blendshapes count {faceBlendShapes.Classification.Count}");
     }
 
     private void OnFaceDetectionsOutput(object stream, OutputEventArgs<List<Detection>> eventArgs)
@@ -104,5 +108,10 @@ private void OnFaceRectsFromDetectionsOutput(object stream, OutputEventArgs<List
     {
       _faceRectsFromDetectionsAnnotationController.DrawLater(eventArgs.value);
     }
+
+    private void OnFaceClassificationsFromBlendShapesOutput(object stream, OutputEventArgs<ClassificationList> eventArgs)
+    {
+      Debug.Log($"Blendshapes count {eventArgs.value?.Classification?.Count}");
+    }
   }
 }