Temporary fix for video latency (#111)

* Revert default sim image to match with depth

* Lower publication rate to prevent lag on VideoFeed

* Aesthetic changes on DetectingFood

* Change updateHz in app to 3Hz

Author: amalnanavati

feeding_web_app_ros2_test/launch/feeding_web_app_dummy_nodes_launch.xml

@@ -23,7 +23,7 @@
       <remap from="~/camera_info" to="/camera/color/camera_info"/>
       <remap from="~/aligned_depth/camera_info" to="/camera/aligned_depth_to_color/camera_info"/>
       <param name="fps" value="30"/>
-      <param name="rgb_path" value="$(find-pkg-share feeding_web_app_ros2_test)/../data/2022_11_01_ada_picks_up_carrots_camera_compressed_ft_tf.mp4"/>
+      <param name="rgb_path" value="$(find-pkg-share feeding_web_app_ros2_test)/../data/above_plate_1_rgb.jpg"/>
       <param name="depth_path" value="$(find-pkg-share feeding_web_app_ros2_test)/../data/above_plate_1_depth.png"/>
     </node>
   </group>

feedingwebapp/src/Pages/Constants.js

@@ -48,10 +48,10 @@ NON_MOVING_STATES.add(MEAL_STATE.U_PostMeal)
 export { NON_MOVING_STATES }
 
 // The names of the ROS topic(s)
-export const CAMERA_FEED_TOPIC = '/local/camera/color/image_raw/compressed'
+export const CAMERA_FEED_TOPIC = '/local/camera/color/image_raw/compressed/low_hz'
 export const FACE_DETECTION_TOPIC = '/face_detection'
 export const FACE_DETECTION_TOPIC_MSG = 'ada_feeding_msgs/FaceDetection'
-export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img/compressed'
+export const FACE_DETECTION_IMG_TOPIC = '/face_detection_img/compressed/low_hz'
 
 // States from which, if they fail, it is NOT okay for the user to retry the
 // same action.

feedingwebapp/src/Pages/Home/MealStates/DetectingFace.jsx

@@ -43,9 +43,9 @@ const DetectingFace = () => {
   // Font size for text
   let textFontSize = 3
   let buttonWidth = 30
-  let buttonHeight = 20
+  let buttonHeight = 18
   let iconWidth = 28
-  let iconHeight = 18
+  let iconHeight = 16
   let sizeSuffix = isPortrait ? 'vh' : 'vw'
   // The min and max distance from the camera to the face for the face to be
   // conidered valid. NOTE: This must match the values in the MoveToMouth tree.

feedingwebapp/src/Pages/Home/VideoFeed.jsx

@@ -85,30 +85,39 @@ const VideoFeed = (props) => {
    * re-connecting upon re-renders.
    */
   const ros = useRef(useROS().ros)
-  // Store the latest image timestamp in a ref to avoid re-generating cameraCallback
-  const latestImageTimestamp = useRef(null)
+  // Store the latest image message in a ref to avoid re-generating cameraCallback
+  const latestImageMessage = useRef(null)
 
   /**
    * Subscribe to the image topic.
    */
   const cameraCallback = useCallback(
     (message) => {
-      // console.log('Got camera message', message)
-      if (!latestImageTimestamp.current || props.updateRateHz <= 0) {
-        setLatestRenderedImg(message)
-      } else {
-        let currTime = message.header.stamp.sec + message.header.stamp.nanosec * 1e-9
-        if (currTime - latestImageTimestamp.current >= 1.0 / props.updateRateHz) {
-          setLatestRenderedImg(message)
-          latestImageTimestamp.current = currTime
-        }
-      }
+      latestImageMessage.current = message
     },
-    [latestImageTimestamp, setLatestRenderedImg, props.updateRateHz]
+    [latestImageMessage]
   )
+
+  /**
+   * Create a timer to re-render the latest image every props.updateRateHz
+   */
+  const [updateHzCurrentDate, setUpdateHzCurrentDate] = useState(new Date())
+  useEffect(() => {
+    setTimeout(() => {
+      setUpdateHzCurrentDate(new Date())
+      setLatestRenderedImg(latestImageMessage.current)
+    }, 1000 / props.updateRateHz)
+  }, [updateHzCurrentDate, setUpdateHzCurrentDate, props.updateRateHz, setLatestRenderedImg, latestImageMessage])
+  /**
+   * Create a timer to re-render the latest image every props.updateRateHz
+   */
+  const [resubscribeRateCurrentDate, setResubscribeRateCurrentDate] = useState(new Date())
   useEffect(() => {
     console.log('subscribing to img topic')
     let topic = subscribeToROSTopic(ros.current, props.topic, 'sensor_msgs/CompressedImage', cameraCallback)
+    setTimeout(() => {
+      setResubscribeRateCurrentDate(new Date())
+    }, 1000 / props.resubscribeRateHz)
     const cleanup = () => {
       console.log('unsubscribing from img topic')
       unsubscribeFromROSTopic(topic, cameraCallback)
@@ -123,7 +132,7 @@ const VideoFeed = (props) => {
       window.removeEventListener('beforeunload', cleanup)
       cleanup()
     }
-  }, [cameraCallback, props.topic])
+  }, [cameraCallback, props.topic, props.resubscribeRateHz, resubscribeRateCurrentDate, setResubscribeRateCurrentDate])
 
   // Callback to resize the image based on the parent width and height
   const resizeImage = useCallback(() => {
@@ -192,18 +201,20 @@ const VideoFeed = (props) => {
 
   // Render the component
   return (
-    <img
-      src={`data:image/jpeg;base64,${latestRenderedImg ? latestRenderedImg.data : ''}`}
-      alt='Live video feed from the robot'
-      style={{
-        width: imgWidth,
-        height: imgHeight,
-        display: 'block',
-        alignItems: 'center',
-        justifyContent: 'center'
-      }}
-      onClick={props.pointClicked ? imageClicked : null}
-    />
+    <>
+      <img
+        src={`data:image/jpeg;base64,${latestRenderedImg ? latestRenderedImg.data : ''}`}
+        alt='Live video feed from the robot'
+        style={{
+          width: imgWidth,
+          height: imgHeight,
+          display: 'block',
+          alignItems: 'center',
+          justifyContent: 'center'
+        }}
+        onClick={props.pointClicked ? imageClicked : null}
+      />
+    </>
   )
 }
 VideoFeed.propTypes = {
@@ -218,6 +229,8 @@ VideoFeed.propTypes = {
   topic: PropTypes.string.isRequired,
   // The rate at which to update the video feed, in Hz
   updateRateHz: PropTypes.number.isRequired,
+  // The rate at which to resubscribe to the image topic
+  resubscribeRateHz: PropTypes.number.isRequired,
   /**
    * An optional callback function for when the user clicks on the video feed.
    * This function should take in two parameters, `x` and `y`, which are the
@@ -228,7 +241,8 @@ VideoFeed.propTypes = {
 }
 VideoFeed.defaultProps = {
   topic: CAMERA_FEED_TOPIC,
-  updateRateHz: 10
+  updateRateHz: 3,
+  resubscribeRateHz: 0.1
 }
 
 export default VideoFeed