fix: FIT-31: Ensure media buffers are syncable events that keep audio and video in sync

web/libs/core/src/lib/utils/feature-flags/flags.ts

@@ -82,3 +82,8 @@ export const FF_ADJUSTABLE_SPANS = "fflag_feat_front_leap_1973_adjustable_spans_
  * Enables the theme toggle in the UI
  */
 export const FF_THEME_TOGGLE = "fflag_feat_front_optic_1217_theme_toggle_short";
+
+/**
+ * Fixes synced audio/video buffering
+ */
+export const FF_SYNCED_BUFFERING = "fflag_fix_front_fit_31_synced_media_buffering";

web/libs/editor/src/components/Timeline/Controls.scss

@@ -1,7 +1,30 @@
 .timeline-controls {
+  position: relative;
   background: var(--color-neutral-background-bold);
   padding: 4px;
 
+  &__buffering {
+    left: 0;
+    right: 0;
+    top: 0;
+    height: 4px;
+    position: absolute;
+    background-color: var(--color-primary-content);
+    overflow: hidden;
+
+    &::after {
+      content: "";
+      position: absolute;
+      top: 0;
+      left: -100%;
+      width: 50%;
+      height: 100%;
+      will-change: left;
+      background-color: var(--color-primary-content-subtle);
+      animation: buffering-media 2.4s ease-out infinite;
+    }
+  }
+
   &__counter {
     height: 36px;
     min-width: 115px;
@@ -85,7 +108,7 @@
         }
       }
 
-      & + & {
+      &+& {
         border-left: 1px solid var(--color-neutral-border);
       }
     }
@@ -110,3 +133,13 @@
     background: var(--color-neutral-surface);
   }
 }
+
+@keyframes buffering-media {
+  0% {
+    left: -100%;
+  }
+
+  100% {
+    left: 200%;
+  }
+}

web/libs/editor/src/components/Timeline/Controls.tsx

@@ -51,6 +51,7 @@ export const Controls: FC<TimelineControlsProps> = memo(
     position,
     frameRate = 1024,
     playing,
+    buffering = false,
     collapsed,
     duration,
     extraControls,
@@ -171,6 +172,7 @@ export const Controls: FC<TimelineControlsProps> = memo(
 
     return (
       <Block name="timeline-controls" tag={Space} spread style={{ gridAutoColumns: "auto" }}>
+        {buffering && <Elem name="buffering" aria-label="Buffering Media Source" />}
         {isFF(FF_DEV_2715) && mediaType === "audio" ? (
           renderControls()
         ) : (

web/libs/editor/src/components/Timeline/Timeline.scss

@@ -3,15 +3,13 @@
   background-color: var(--color-neutral-background);
 
   &__topbar {
+    padding-bottom: var(--spacing-tight);
     min-height: 48px;
-    padding: 6px;
     display: grid;
     grid-row-gap: 8px;
     align-items: center;
     grid-auto-rows: min-content;
     grid-template-rows: 1fr;
-    border-top: 1px solid var(--color-neutral-border);
-    border-bottom: 1px solid var(--color-neutral-border);
   }
 }
 

web/libs/editor/src/components/Timeline/Timeline.tsx

@@ -20,6 +20,7 @@ const TimelineComponent: FC<TimelineProps> = ({
   framerate = 24,
   hopSize = 1,
   playing = false,
+  buffering = false,
   fullscreen = false,
   disableView = false,
   defaultStepSize = 10,
@@ -126,6 +127,7 @@ const TimelineComponent: FC<TimelineProps> = ({
         position={currentPosition}
         frameRate={framerate}
         playing={playing}
+        buffering={buffering}
         volume={props.volume}
         controls={props.controls}
         altHopSize={props.altHopSize}

web/libs/editor/src/components/Timeline/Types.ts

@@ -15,6 +15,7 @@ export interface TimelineProps<D extends ViewTypes = "frames"> {
   mode: D;
   framerate: number;
   playing: boolean;
+  buffering?: boolean;
   zoom?: number;
   volume?: number;
   speed?: number;
@@ -172,6 +173,7 @@ export interface TimelineControlsProps {
   playing: boolean;
   collapsed: boolean;
   fullscreen: boolean;
+  buffering?: boolean;
   volume?: number;
   speed?: number;
   zoom?: number;

web/libs/editor/src/components/VideoCanvas/VideoCanvas.scss

@@ -39,7 +39,7 @@
     padding: 2px 7px;
     text-align: center;
     background-color: var(--color-neutral-background);
-    background-image: repeating-linear-gradient(90deg, var(--grape_500) 0, var(--grape_400) 120px, var(--grape_500) 240px);
+    background-image: repeating-linear-gradient(90deg, var(--color-primary-content) 0, var(--color-primary-content-subtle) 120px, var(--color-primary-content) 240px);
     animation: buffering 1.2s linear infinite;
   }
 }
@@ -72,4 +72,4 @@
   100% {
     background-position: 240px 0;
   }
-}
+}

web/libs/editor/src/components/VideoCanvas/VideoCanvas.tsx

@@ -6,6 +6,10 @@ import "./VideoCanvas.scss";
 import { MAX_ZOOM, MIN_ZOOM } from "./VideoConstants";
 import { VirtualCanvas } from "./VirtualCanvas";
 import { VirtualVideo } from "./VirtualVideo";
+import { ff } from "@humansignal/core";
+import { useSyncedBuffering } from "../../hooks/useSyncedBuffering";
+
+const isSyncedBuffering = ff.isActive(ff.FF_SYNCED_BUFFERING);
 
 type VideoProps = {
   src: string;
@@ -14,6 +18,7 @@ type VideoProps = {
   position?: number;
   currentTime?: number;
   playing?: boolean;
+  buffering?: boolean;
   framerate?: number;
   muted?: boolean;
   zoom?: number;
@@ -37,6 +42,7 @@ type VideoProps = {
   onEnded?: () => void;
   onResize?: (dimensions: VideoDimentions) => void;
   onError?: (error: any) => void;
+  onBuffering?: (isBuffering: boolean) => void;
 };
 
 type PanOptions = {
@@ -88,6 +94,12 @@ export interface VideoRef {
   adjustPan: (x: number, y: number) => PanOptions;
 }
 
+const useLocalBuffering = (props: VideoProps) => {
+  return useState(false);
+};
+
+const useBuffering = isSyncedBuffering ? useSyncedBuffering : useLocalBuffering;
+
 export const VideoCanvas = memo(
   forwardRef<VideoRef, VideoProps>((props, ref) => {
     const raf = useRef<number>();
@@ -106,7 +118,7 @@ export const VideoCanvas = memo(
     const [length, setLength] = useState(0);
     const [currentFrame, setCurrentFrame] = useState(props.position ?? 1);
     const [playing, setPlaying] = useState(false);
-    const [buffering, setBuffering] = useState(false);
+    const [buffering, setBuffering] = useBuffering(props);
     const [zoom, setZoom] = useState(props.zoom ?? 1);
     const [pan, setPan] = useState<PanOptions>(props.pan ?? { x: 0, y: 0 });
 
@@ -115,6 +127,8 @@ export const VideoCanvas = memo(
     const [contrast, setContrast] = useState(1);
     const [brightness, setBrightness] = useState(1);
     const [saturation, setSaturation] = useState(1);
+    const bufferingRef = useRef(buffering);
+    bufferingRef.current = buffering;
 
     const filters = useMemo(() => {
       const result: string[] = [];
@@ -170,6 +184,7 @@ export const VideoCanvas = memo(
     const updateFrame = useCallback(
       (force = false) => {
         if (!contextRef.current) return;
+        if (isSyncedBuffering && bufferingRef.current) return;
 
         const currentTime = videoRef.current?.currentTime ?? 0;
         const frameNumber = isFF(FF_VIDEO_FRAME_SEEK_PRECISION)
@@ -187,6 +202,19 @@ export const VideoCanvas = memo(
       [framerate, currentFrame, drawVideo, props.onFrameChange, length],
     );
 
+    const updateBuffering = useCallback(() => {
+      if (!videoRef.current) return;
+      if (!contextRef.current) return;
+
+      const video = videoRef.current;
+      if (video.networkState === video.NETWORK_IDLE) {
+        hasLoadedRef.current = true;
+        setBuffering(false);
+      } else {
+        setBuffering(true);
+      }
+    }, []);
+
     const delayedUpdate = useCallback(() => {
       if (!videoRef.current) return;
       if (!contextRef.current) return;
@@ -196,40 +224,51 @@ export const VideoCanvas = memo(
       if (video) {
         if (!playing) updateFrame(true);
 
-        if (video.networkState === video.NETWORK_IDLE) {
-          hasLoadedRef.current = true;
-          setBuffering(false);
-        } else {
-          setBuffering(true);
-        }
+        updateBuffering();
       }
     }, [playing, updateFrame]);
 
     // VIDEO EVENTS'
     const handleVideoPlay = useCallback(() => {
       setPlaying(true);
-      setBuffering(false);
+      if (!isSyncedBuffering) {
+        setBuffering(false);
+      } else {
+        updateBuffering();
+      }
       props.onPlay?.();
     }, [props.onPlay]);
 
     const handleVideoPause = useCallback(() => {
       setPlaying(false);
-      setBuffering(false);
+      if (!isSyncedBuffering) {
+        setBuffering(false);
+      } else {
+        updateBuffering();
+      }
       props.onPause?.();
     }, [props.onPause]);
 
     const handleVideoPlaying = useCallback(() => {
-      setBuffering(false);
+      if (!isSyncedBuffering) {
+        setBuffering(false);
+      }
       delayedUpdate();
     }, [delayedUpdate]);
 
     const handleVideoWaiting = useCallback(() => {
-      setBuffering(true);
+      if (!isSyncedBuffering) {
+        setBuffering(true);
+      } else {
+        updateBuffering();
+      }
     }, []);
 
     const handleVideoEnded = useCallback(() => {
       setPlaying(false);
-      setBuffering(false);
+      if (!isSyncedBuffering) {
+        setBuffering(false);
+      }
       props.onSeeked?.();
       props.onEnded?.();
       props.onPause?.();
@@ -290,10 +329,13 @@ export const VideoCanvas = memo(
 
     // Handle extrnal state change [current time]
     useEffect(() => {
-      if (videoRef.current && props.currentTime) {
-        videoRef.current.currentTime = props.currentTime;
-      }
-    }, [props.currentTime]);
+      const updateId = requestAnimationFrame(() => {
+        if (isSyncedBuffering && bufferingRef.current) return;
+        if (videoRef.current && props.currentTime) videoRef.current.currentTime = props.currentTime;
+      });
+
+      return () => cancelAnimationFrame(updateId);
+    }, [props.currentTime, bufferingRef]);
 
     // Handle extrnal state change [play/pause]
     useEffect(() => {
@@ -469,8 +511,8 @@ export const VideoCanvas = memo(
 
     useEffect(() => {
       let isLoaded = false;
-      let loadTimeout: NodeJS.Timeout | undefined = undefined;
-      let timeout: NodeJS.Timeout | undefined = undefined;
+      let loadTimeout: ReturnType<typeof setTimeout> | undefined = undefined;
+      let timeout: ReturnType<typeof setTimeout> | undefined = undefined;
 
       const checkVideoLoaded = () => {
         if (isLoaded) return;
@@ -566,7 +608,7 @@ export const VideoCanvas = memo(
             width={canvasWidth}
             height={canvasHeight}
           />
-          {!loading && buffering && <Elem name="buffering" />}
+          {!loading && buffering && !isSyncedBuffering && <Elem name="buffering" aria-label="Buffering Media Source" />}
         </Elem>
 
         <VirtualVideo
@@ -581,15 +623,21 @@ export const VideoCanvas = memo(
           onLoadedData={delayedUpdate}
           onCanPlay={delayedUpdate}
           onSeeked={(event) => {
-            delayedUpdate();
+            if (!isSyncedBuffering) {
+              delayedUpdate();
+            }
             props.onSeeked?.(event);
           }}
           onSeeking={(event) => {
-            delayedUpdate();
+            if (!isSyncedBuffering) {
+              delayedUpdate();
+            }
             props.onSeeked?.(event);
           }}
           onTimeUpdate={(event) => {
-            delayedUpdate();
+            if (!isSyncedBuffering) {
+              delayedUpdate();
+            }
             props.onTimeUpdate?.(event);
           }}
           onProgress={delayedUpdate}