From 623c9b65695bbd4a1f4c820b656a53ef9bbe8285 Mon Sep 17 00:00:00 2001 From: Paul Tunison Date: Wed, 8 May 2024 11:40:32 -0400 Subject: [PATCH 1/6] Rename "image_0" topic back to `PVFramesBGR`, misc sync in R18 configs --- tmux/demos/medical/BBN-M2-Tourniquet.yml | 2 +- ...3-BBN-integrate-Kitware.yml => BBN-M3.yml} | 44 ++-- tmux/demos/medical/BBN-R18.yml | 73 ++++--- .../{M3-Kitware.yml => Kitware-M3.yml} | 40 ++-- tmux/demos/medical/Kitware-R18.yml | 196 ++++++++++-------- 5 files changed, 189 insertions(+), 166 deletions(-) rename tmux/demos/medical/{M3-BBN-integrate-Kitware.yml => BBN-M3.yml} (87%) rename tmux/demos/medical/{M3-Kitware.yml => Kitware-M3.yml} (88%) diff --git a/tmux/demos/medical/BBN-M2-Tourniquet.yml b/tmux/demos/medical/BBN-M2-Tourniquet.yml index c8fcf44ab..32655f402 100644 --- a/tmux/demos/medical/BBN-M2-Tourniquet.yml +++ b/tmux/demos/medical/BBN-M2-Tourniquet.yml @@ -80,7 +80,7 @@ windows: - BBN Interface: layout: even-vertical panes: - - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args + - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args -r __ns:=${ROS_NAMESPACE} -p task_update_topic:=TaskUpdates -p bbn_update_topic:=BBNUpdates diff --git a/tmux/demos/medical/M3-BBN-integrate-Kitware.yml b/tmux/demos/medical/BBN-M3.yml similarity index 87% rename from tmux/demos/medical/M3-BBN-integrate-Kitware.yml rename to tmux/demos/medical/BBN-M3.yml index e06b4101b..3339531c2 100644 --- a/tmux/demos/medical/M3-BBN-integrate-Kitware.yml +++ b/tmux/demos/medical/BBN-M3.yml @@ -49,40 +49,40 @@ windows: - sensor_input: ros2 run angel_system_nodes redis_ros_bridge --ros-args -r __ns:=${ROS_NAMESPACE} -p url:=${HL2SS_API_URL} - -p image_topic:=\image_0 + -p image_topic:=PVFramesBGR -p hand_pose_topic:=HandJointPoseData - - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args + - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=\image_0 - -p output_topic:=\image_0_ts + -p image_topic:=PVFramesBGR + -p output_topic:=PVFramesBGR_ts - - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args + - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=\image_0 - -p det_topic:=pose_dets - -p pose_topic:=PatientPose - -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth - -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth - -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml + -p image_topic:=PVFramesBGR + -p det_topic:=pose_dets + -p pose_topic:=PatientPose + -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth + -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth + -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml -p pose_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py -p cuda_device_id:=0 - - objects_hands_detection: ros2 run angel_system_nodes object_hand_detector --ros-args + - objects_hands_detection: ros2 run angel_system_nodes object_hand_detector --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=\image_0 - -p det_topic:=ObjectDetections2d + -p image_topic:=PVFramesBGR + -p det_topic:=ObjectDetections2d -p net_checkpoint:=${MODEL_DIR}/object_detector/m3_det.pt -p hand_net_checkpoint:=${MODEL_DIR}/object_detector/hands_model.pt - - run_tcn: ros2 run angel_system_nodes activity_classifier_tcn --ros-args + - run_tcn: ros2 run angel_system_nodes activity_classifier_tcn --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_ts_topic:=\image_0_ts - -p det_topic:=ObjectDetections2d - -p pose_topic:=PatientPose - -p model_weights:=${MODEL_DIR}/activity_classifier/m3_tcn.ckpt - -p model_mapping:=${MODEL_DIR}/activity_classifier/m3_mapping.txt - -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/m3.json + -p image_ts_topic:=PVFramesBGR_ts + -p det_topic:=ObjectDetections2d + -p pose_topic:=PatientPose + -p model_weights:=${MODEL_DIR}/activity_classifier/m3_tcn.ckpt + -p model_mapping:=${MODEL_DIR}/activity_classifier/m3_mapping.txt + -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/m3.json -p act_topic:=activity_topic - task_monitor: ros2 run angel_system_nodes global_step_predictor --ros-args @@ -105,7 +105,7 @@ windows: - BBN Interface: layout: even-vertical panes: - - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args + - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args -r __ns:=${ROS_NAMESPACE} -p task_update_topic:=TaskUpdates -p bbn_update_topic:=BBNUpdates diff --git a/tmux/demos/medical/BBN-R18.yml b/tmux/demos/medical/BBN-R18.yml index 35397161a..f4112c058 100644 --- a/tmux/demos/medical/BBN-R18.yml +++ b/tmux/demos/medical/BBN-R18.yml @@ -3,8 +3,7 @@ # # This configuration is for the M2 tourniquet task. # - -name: R18-Chest-Seal +name: BBN-R18-Chest-Seal root: <%= ENV["ANGEL_WORKSPACE_DIR"] %> # Optional tmux socket @@ -51,40 +50,40 @@ windows: - sensor_input: ros2 run angel_system_nodes redis_ros_bridge --ros-args -r __ns:=${ROS_NAMESPACE} -p url:=${HL2SS_API_URL} - -p image_topic:=image_0 + -p image_topic:=PVFramesBGR -p hand_pose_topic:=HandJointPoseData - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p output_topic:=image_0_ts + -p image_topic:=PVFramesBGR + -p output_topic:=PVFramesBGR_ts - - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args + - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p det_topic:=pose_dets - -p pose_topic:=PatientPose - -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth - -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth - -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml + -p image_topic:=PVFramesBGR + -p det_topic:=pose_dets + -p pose_topic:=PatientPose + -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth + -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth + -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml -p pose_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py -p cuda_device_id:=0 - - object_and_hand_detection: ros2 run angel_system_nodes object_and_hand_detector --ros-args + - object_and_hand_detection: ros2 run angel_system_nodes object_and_hand_detector --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p det_topic:=ObjectDetections2d + -p image_topic:=PVFramesBGR + -p det_topic:=ObjectDetections2d -p object_net_checkpoint:=${MODEL_DIR}/object_detector/r18_det.pt -p hand_net_checkpoint:=${MODEL_DIR}/object_detector/hands_model.pt - - activity_classifier: ros2 run angel_system_nodes activity_classifier_tcn --ros-args + - activity_classifier: ros2 run angel_system_nodes activity_classifier_tcn --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_ts_topic:=image_0_ts - -p det_topic:=ObjectDetections2d - -p pose_topic:=PatientPose - -p model_weights:=${MODEL_DIR}/activity_classifier/r18_tcn.ckpt - -p model_mapping:=${MODEL_DIR}/activity_classifier/r18_mapping.txt - -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/r18.json + -p image_ts_topic:=PVFramesBGR_ts + -p det_topic:=ObjectDetections2d + -p pose_topic:=PatientPose + -p model_weights:=${MODEL_DIR}/activity_classifier/r18_tcn.ckpt + -p model_mapping:=${MODEL_DIR}/activity_classifier/r18_mapping.txt + -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/r18.json -p act_topic:=activity_topic - task_monitor: @@ -107,26 +106,12 @@ windows: -p query_task_graph_topic:=query_task_graph - echo: sleep 0.5 && ros2 topic echo --no-arr "${ROS_NAMESPACE}/TaskUpdates" - - BBN Interface: - layout: even-vertical - panes: - - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args - -r __ns:=${ROS_NAMESPACE} - -p task_update_topic:=TaskUpdates - -p bbn_update_topic:=BBNUpdates - -p task_graph_srv_topic:=query_task_graph - - echo: sleep 0.5 && ros2 topic echo "${ROS_NAMESPACE}/BBNUpdates" - - zmq-publisher: ros2 run bbn_integration ZmqIntegrationClient --ros-args - -r __ns:=${ROS_NAMESPACE} - -p topic_update_msg:=BBNUpdates - -p server_address:=${BBN_OUTPUT_URL} - - engineering-ui: layout: even-vertical panes: - simple_2d_overlay: ros2 run angel_utils Simple2dDetectionOverlay --ros-args -r __ns:=${ROS_NAMESPACE} - -p topic_input_images:=image_0 + -p topic_input_images:=PVFramesBGR -p topic_input_det_2d:=ObjectDetections2d -p topic_input_joints:=PatientPose -p topic_output_images:=pv_image_detections_2d @@ -139,3 +124,17 @@ windows: --task_updates_topic=TaskUpdates --activity_detections_topic=activity_topic --task_errors_topic=TaskErrors + + - BBN Interface: + layout: even-vertical + panes: + - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args + -r __ns:=${ROS_NAMESPACE} + -p task_update_topic:=TaskUpdates + -p bbn_update_topic:=BBNUpdates + -p task_graph_srv_topic:=query_task_graph + - echo: sleep 0.5 && ros2 topic echo "${ROS_NAMESPACE}/BBNUpdates" + - zmq-publisher: ros2 run bbn_integration ZmqIntegrationClient --ros-args + -r __ns:=${ROS_NAMESPACE} + -p topic_update_msg:=BBNUpdates + -p server_address:=${BBN_OUTPUT_URL} diff --git a/tmux/demos/medical/M3-Kitware.yml b/tmux/demos/medical/Kitware-M3.yml similarity index 88% rename from tmux/demos/medical/M3-Kitware.yml rename to tmux/demos/medical/Kitware-M3.yml index 77c360681..8f68e432e 100644 --- a/tmux/demos/medical/M3-Kitware.yml +++ b/tmux/demos/medical/Kitware-M3.yml @@ -51,35 +51,35 @@ windows: - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p output_topic:=image_0_ts + -p image_topic:=PVFramesBGR + -p output_topic:=PVFramesBGR_ts - - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args + - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p det_topic:=pose_dets - -p pose_topic:=PatientPose - -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth - -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth - -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml + -p image_topic:=PVFramesBGR + -p det_topic:=pose_dets + -p pose_topic:=PatientPose + -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth + -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth + -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml -p pose_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py -p cuda_device_id:=0 - - objects_hands_detection: ros2 run angel_system_nodes object_hand_detector --ros-args + - objects_hands_detection: ros2 run angel_system_nodes object_hand_detector --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p det_topic:=ObjectDetections2d + -p image_topic:=PVFramesBGR + -p det_topic:=ObjectDetections2d -p net_checkpoint:=${MODEL_DIR}/object_detector/m3_det.pt -p hand_net_checkpoint:=${MODEL_DIR}/object_detector/hands_model.pt - - run_tcn: ros2 run angel_system_nodes activity_classifier_tcn --ros-args + - run_tcn: ros2 run angel_system_nodes activity_classifier_tcn --ros-args -r __ns:=${ROS_NAMESPACE} - -p image_ts_topic:=image_0_ts - -p det_topic:=ObjectDetections2d - -p pose_topic:=PatientPose - -p model_weights:=${MODEL_DIR}/activity_classifier/m3_tcn.ckpt - -p model_mapping:=${MODEL_DIR}/activity_classifier/m3_mapping.txt - -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/m3.json + -p image_ts_topic:=PVFramesBGR_ts + -p det_topic:=ObjectDetections2d + -p pose_topic:=PatientPose + -p model_weights:=${MODEL_DIR}/activity_classifier/m3_tcn.ckpt + -p model_mapping:=${MODEL_DIR}/activity_classifier/m3_mapping.txt + -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/m3.json -p act_topic:=activity_topic - task_monitor: ros2 run angel_system_nodes global_step_predictor --ros-args @@ -102,7 +102,7 @@ windows: - BBN Interface: layout: even-vertical panes: - - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args + - task-converter: ros2 run bbn_integration_py task_to_bbn_update --ros-args -r __ns:=${ROS_NAMESPACE} -p task_update_topic:=TaskUpdates -p bbn_update_topic:=BBNUpdates diff --git a/tmux/demos/medical/Kitware-R18.yml b/tmux/demos/medical/Kitware-R18.yml index 3209fdd80..742c432e2 100644 --- a/tmux/demos/medical/Kitware-R18.yml +++ b/tmux/demos/medical/Kitware-R18.yml @@ -18,12 +18,21 @@ root: <%= ENV["ANGEL_WORKSPACE_DIR"] %> # Runs on project start, always # on_project_start: command on_project_start: | - export ROS_NAMESPACE=${ROS_NAMESPACE:-/kitware} - export HL2SS_API_URL=${HL2SS_API_URL:-128.33.193.178:8000} - export BBN_OUTPUT_URL=${BBN_OUTPUT_URL:-tcp://128.33.193.178:6669} - export CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/config - export NODE_CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/src/angel_system_nodes/configs - export MODEL_DIR=${ANGEL_WORKSPACE_DIR}/model_files + export ROS_NAMESPACE=${ROS_NAMESPACE:-/kitware} + export HL2_IP=${HL2_IP:-192.168.1.4} + export BBN_OUTPUT_URL=${BBN_OUTPUT_URL:-tcp://128.33.193.178:6669} + export CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/config + export NODE_CONFIG_DIR=${ANGEL_WORKSPACE_DIR}/src/angel_system_nodes/configs + export MODEL_DIR=${ANGEL_WORKSPACE_DIR}/model_files + + # Changing the domain ID was important at KHQ to unblock perceived network + # congestion slowdowns to message sending. + export ROS_DOMAIN_ID=77 + + # Set the frame-rate to be used by multiple sources. This should be in frames + # per second (Hz). + export FRAME_RATE=15 + # Run on project start, the first time # on_project_first_start: command @@ -44,84 +53,99 @@ on_project_start: | tmux_options: -f <%= ENV["ANGEL_WORKSPACE_DIR"] %>/tmux/tmux.conf windows: - - - sensor_input: - layout: even-vertical - panes: - # Read sensor input from bag file. - - sensor_input: echo ros2 bag play ${ANGEL_WORKSPACE_DIR}/ros_bags/rosbag2_2024_04_10-18_57_36-R18_Chris_live/ - - - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args - -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p output_topic:=image_0_ts - - - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args - -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p det_topic:=pose_dets - -p pose_topic:=PatientPose - -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth - -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth - -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml - -p pose_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py - -p cuda_device_id:=0 - - - object_detector: - layout: even-vertical - panes: - - object_and_hand_detection: ros2 run angel_system_nodes object_and_hand_detector --ros-args - -r __ns:=${ROS_NAMESPACE} - -p image_topic:=image_0 - -p det_topic:=ObjectDetections2d - -p object_net_checkpoint:=${MODEL_DIR}/object_detector/r18_det.pt - -p hand_net_checkpoint:=${MODEL_DIR}/object_detector/hands_model.pt - - - activity_classifier: ros2 run angel_system_nodes activity_classifier_tcn --ros-args - -r __ns:=${ROS_NAMESPACE} - -p image_ts_topic:=image_0_ts - -p det_topic:=ObjectDetections2d - -p pose_topic:=PatientPose - -p model_weights:=${MODEL_DIR}/activity_classifier/r18_tcn.ckpt - -p model_mapping:=${MODEL_DIR}/activity_classifier/r18_mapping.txt - -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/r18.json - -p act_topic:=activity_topic - - - task_monitor: - layout: even-vertical - panes: - - gsp: ros2 run angel_system_nodes global_step_predictor --ros-args + - sensor_input: + layout: even-vertical + panes: +# # Read sensor input from bag file. +# - sensor_input: echo ros2 bag play ${ANGEL_WORKSPACE_DIR}/ros_bags/rosbag2_2024_04_10-18_57_36-R18_Chris_live/ +# - run_image_timestamp: ros2 run angel_system_nodes image_timestamp_relay --ros-args +# -r __ns:=${ROS_NAMESPACE} +# -p image_topic:=PVFramesBGR +# -p output_topic:=PVFramesBGR_ts + + # Read sensor input from a HoloLens2 unit using HL2SS. + - hl2ss_bridge: ros2 run angel_system_nodes hl2ss_ros_bridge --ros-args + -r __ns:=${ROS_NAMESPACE} + -p ip_addr:=${HL2_IP} + -p image_topic:=PVFramesBGR + -p image_ts_topic:=PVFramesBGR_TS + -p hand_pose_topic:=disable + -p audio_topic:=disable + -p sm_topic:=disable + -p head_pose_topic:=disable + -p pv_width:=1280 + -p pv_height:=720 + -p pv_framerate:=${FRAME_RATE} + -p sm_freq:=5 + -p rm_depth_AHAT:=disable + # Enable sending ROS2 messages to receiving agents in an HoloLens2 app. + - datahub: ros2 run ros_tcp_endpoint default_server_endpoint --ros-args + -r __ns:=${ROS_NAMESPACE} + -p ROS_IP:=0.0.0.0 + + - pose_estimation: ros2 run angel_system_nodes pose_estimator --ros-args + -r __ns:=${ROS_NAMESPACE} + -p image_topic:=PVFramesBGR + -p det_topic:=pose_dets + -p pose_topic:=PatientPose + -p det_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_det_model.pth + -p pose_net_checkpoint:=${MODEL_DIR}/pose_estimation/pose_model.pth + -p det_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/medic_pose.yaml + -p pose_config:=${ANGEL_WORKSPACE_DIR}/python-tpl/TCN_HPL/tcn_hpl/data/utils/pose_generation/configs/ViTPose_base_medic_casualty_256x192.py + -p cuda_device_id:=0 + + - object_and_hand_detection: ros2 run angel_system_nodes object_and_hand_detector --ros-args + -r __ns:=${ROS_NAMESPACE} + -p image_topic:=PVFramesBGR + -p det_topic:=ObjectDetections2d + -p object_net_checkpoint:=${MODEL_DIR}/object_detector/r18_det.pt + -p hand_net_checkpoint:=${MODEL_DIR}/object_detector/hands_model.pt + + - activity_classifier: ros2 run angel_system_nodes activity_classifier_tcn --ros-args + -r __ns:=${ROS_NAMESPACE} + -p image_ts_topic:=PVFramesBGR_ts + -p det_topic:=ObjectDetections2d + -p pose_topic:=PatientPose + -p model_weights:=${MODEL_DIR}/activity_classifier/r18_tcn.ckpt + -p model_mapping:=${MODEL_DIR}/activity_classifier/r18_mapping.txt + -p model_det_label_mapping:=${ANGEL_WORKSPACE_DIR}/config/object_labels/medical/r18.json + -p act_topic:=activity_topic + + - task_monitor: + layout: even-vertical + panes: + - gsp: ros2 run angel_system_nodes global_step_predictor --ros-args + -r __ns:=${ROS_NAMESPACE} + -p config_file:=${CONFIG_DIR}/tasks/medical/multi-task-config-medical-r18.yaml + -p activity_config_file:=${CONFIG_DIR}/activity_labels/medical/r18.yaml + -p task_state_topic:=TaskUpdates + -p task_error_topic:=TaskErrors + -p system_command_topic:=SystemCommands + -p det_topic:=activity_topic + -p model_file:=${MODEL_DIR}/task_monitor/r18_test_activity_preds.mscoco.json + -p thresh_frame_count:=3 + -p deactivate_thresh_frame_count:=10 + -p threshold_multiplier_weak:=0.00 + -p threshold_frame_count_weak:=3 + -p step_mode:=granular + -p query_task_graph_topic:=query_task_graph + - echo: sleep 0.5 && ros2 topic echo --no-arr "${ROS_NAMESPACE}/TaskUpdates" + + - engineering-ui: + layout: even-vertical + panes: + - simple_2d_overlay: ros2 run angel_utils Simple2dDetectionOverlay --ros-args -r __ns:=${ROS_NAMESPACE} - -p config_file:=${CONFIG_DIR}/tasks/medical/multi-task-config-medical-r18.yaml - -p activity_config_file:=${CONFIG_DIR}/activity_labels/medical/r18.yaml - -p task_state_topic:=TaskUpdates - -p task_error_topic:=TaskErrors - -p system_command_topic:=SystemCommands - -p det_topic:=activity_topic - -p model_file:=${MODEL_DIR}/task_monitor/r18_test_activity_preds.mscoco.json - -p thresh_frame_count:=3 - -p deactivate_thresh_frame_count:=10 - -p threshold_multiplier_weak:=0.00 - -p threshold_frame_count_weak:=3 - -p step_mode:=granular - -p query_task_graph_topic:=query_task_graph - - echo: sleep 0.5 && ros2 topic echo --no-arr "${ROS_NAMESPACE}/TaskUpdates" - - - engineering-ui: - layout: even-vertical - panes: - - simple_2d_overlay: ros2 run angel_utils Simple2dDetectionOverlay --ros-args - -r __ns:=${ROS_NAMESPACE} - -p topic_input_images:=image_0 - -p topic_input_det_2d:=ObjectDetections2d - -p topic_input_joints:=PatientPose - -p topic_output_images:=pv_image_detections_2d - -p filter_top_k:=5 - - websocket: ros2 launch rosbridge_server rosbridge_websocket_launch.xml port:=9090 - - engineering_ui_server: node ros/angel_utils/multi_task_demo_ui/index.js - --namespace=${ROS_NAMESPACE} - --image_topic=pv_image_detections_2d/compressed - --query_task_graph_topic=query_task_graph - --task_updates_topic=TaskUpdates - --activity_detections_topic=activity_topic - --task_errors_topic=TaskErrors + -p topic_input_images:=PVFramesBGR + -p topic_input_det_2d:=ObjectDetections2d + -p topic_input_joints:=PatientPose + -p topic_output_images:=pv_image_detections_2d + -p filter_top_k:=5 + - websocket: ros2 launch rosbridge_server rosbridge_websocket_launch.xml port:=9090 + - engineering_ui_server: node ros/angel_utils/multi_task_demo_ui/index.js + --namespace=${ROS_NAMESPACE} + --image_topic=pv_image_detections_2d/compressed + --query_task_graph_topic=query_task_graph + --task_updates_topic=TaskUpdates + --activity_detections_topic=activity_topic + --task_errors_topic=TaskErrors From 91042a02cf8f31b361cf9772376bf6ec0c751d03 Mon Sep 17 00:00:00 2001 From: Paul Tunison Date: Tue, 7 May 2024 13:07:43 -0400 Subject: [PATCH 2/6] Add cache mounting for pip/poetry --- docker/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a9c851c5f..92a9ff62c 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -81,6 +81,8 @@ services: - ${WORKSPACE_SHELL_HOST_DIR}/stuff:${ANGEL_WORKSPACE_DIR}/stuff - ${WORKSPACE_SHELL_HOST_DIR}/ros_home:/root/.ros - ${WORKSPACE_SHELL_HOST_DIR}/cache/torch:/root/.cache/torch + - ${WORKSPACE_SHELL_HOST_DIR}/cache/pip:/root/.cache/pip + - ${WORKSPACE_SHELL_HOST_DIR}/cache/poetry:/root/.cache/pypoetry # X11 things - /tmp/.X11-unix:/tmp/.X11-unix # assume this file exists, should be created before running. From 1b480765db35f1deed4611224391125640c6457f Mon Sep 17 00:00:00 2001 From: Paul Tunison Date: Tue, 7 May 2024 12:32:18 -0400 Subject: [PATCH 3/6] Add ROS2 message for changing tasks --- ros/angel_msgs/msg/SystemCommands.msg | 31 +++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/ros/angel_msgs/msg/SystemCommands.msg b/ros/angel_msgs/msg/SystemCommands.msg index 77ab0b65d..dc204af31 100644 --- a/ros/angel_msgs/msg/SystemCommands.msg +++ b/ros/angel_msgs/msg/SystemCommands.msg @@ -8,8 +8,39 @@ # received message instance. # + +# ===================== # Task Monitor commands +# ===================== + +# Index of GSP tasks being concurrently handled to indicate step progression/regression for. int8 task_index + +# If the current task progress should be reset. bool reset_current_task + +# If the current task progress should be regressed a step, bool previous_step + +# If the current task progress should be progressed a step. bool next_step + + +# ============= +# Task Changing +# ============= + +# Enumeration of sorts for the tasks that may be switched to. +# +# This is an enumeration here in order to be a source of coordination between +# nodes that support this functionality as just simply passing a string can +# easily be subject to spelling errors or misalignment between nodes. +# +uint32 TASK_NO_CHANGE = 0 +uint32 TASK_COOKING = 1 # Cooking is "meta" atm because it is multi-task under the hood. +uint32 TASK_MEDICAL_M2 = 2 +uint32 TASK_MEDICAL_M3 = 3 +uint32 TASK_MEDICAL_R18 = 4 + +# The task that is being changed to. +uint32 change_task From 1ddf8854e4e9d472f7cb2761e2fbc7c691d5436b Mon Sep 17 00:00:00 2001 From: Paul Tunison Date: Wed, 8 May 2024 12:55:14 -0400 Subject: [PATCH 4/6] Let me pass options to docker-compose via build helper --- angel-docker-build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/angel-docker-build.sh b/angel-docker-build.sh index 43018d243..5b7f56a08 100755 --- a/angel-docker-build.sh +++ b/angel-docker-build.sh @@ -37,6 +37,12 @@ do shift FORCE_BUILD=1 ;; + --) + # Escape the remainder of args as to be considered passthrough + shift + dc_forward_params+=("${@}") + break + ;; *) # anything else dc_forward_params+=("$1") shift From ecfb804b578f11e520c4d11aa4da04b4af3eb005 Mon Sep 17 00:00:00 2001 From: Paul Tunison Date: Wed, 8 May 2024 13:09:18 -0400 Subject: [PATCH 5/6] fixup! Add ROS2 message for changing tasks --- ros/angel_msgs/msg/SystemCommands.msg | 5 +++++ .../angel_system_nodes/audio/intent/intent_to_command.py | 1 + .../task_monitoring/keyboard_to_sys_cmd.py | 1 + 3 files changed, 7 insertions(+) diff --git a/ros/angel_msgs/msg/SystemCommands.msg b/ros/angel_msgs/msg/SystemCommands.msg index dc204af31..6bb03160a 100644 --- a/ros/angel_msgs/msg/SystemCommands.msg +++ b/ros/angel_msgs/msg/SystemCommands.msg @@ -8,6 +8,11 @@ # received message instance. # +# Time at which this command was emitted. +# -> Not using a `std_msgs/Header` here as the use of frame_id is currently +# undefined. +builtin_interfaces/Time stamp + # ===================== # Task Monitor commands diff --git a/ros/angel_system_nodes/angel_system_nodes/audio/intent/intent_to_command.py b/ros/angel_system_nodes/angel_system_nodes/audio/intent/intent_to_command.py index 6a0e83bbe..e858ee905 100644 --- a/ros/angel_system_nodes/angel_system_nodes/audio/intent/intent_to_command.py +++ b/ros/angel_system_nodes/angel_system_nodes/audio/intent/intent_to_command.py @@ -96,6 +96,7 @@ def intent_callback(self, intent: InterpretedAudioUserIntent) -> None: return sys_cmd_msg = SystemCommands() + sys_cmd_msg.stamp = self.get_clock().now().to_msg() # TODO: This only works currently since we only recognize a few boolean # based commands (next step, previous step). This will break down if diff --git a/ros/angel_system_nodes/angel_system_nodes/task_monitoring/keyboard_to_sys_cmd.py b/ros/angel_system_nodes/angel_system_nodes/task_monitoring/keyboard_to_sys_cmd.py index 0aeba2a69..be13e5991 100644 --- a/ros/angel_system_nodes/angel_system_nodes/task_monitoring/keyboard_to_sys_cmd.py +++ b/ros/angel_system_nodes/angel_system_nodes/task_monitoring/keyboard_to_sys_cmd.py @@ -79,6 +79,7 @@ def publish_sys_cmd(self, task_id: int, forward: bool) -> None: """ log = self.get_logger() msg = SystemCommands() + msg.stamp = self.get_clock().now().to_msg() msg.task_index = task_id if forward: msg.next_step = True From 4f57389087a1b7bc0f44c663c4d74fa68ad172b5 Mon Sep 17 00:00:00 2001 From: Paul Tunison Date: Fri, 10 May 2024 12:15:14 -0400 Subject: [PATCH 6/6] WIP: Adding configuration and model switching for OaHD node --- .../object_and_hand_detection.yml | 19 ++++ poetry.lock | 2 +- pyproject.toml | 1 + .../object_and_hand_detection.py | 107 ++++++++++++++++-- .../task_monitoring/global_step_predictor.py | 4 +- .../python/angel_utils/system_commands.py | 36 ++++++ tmux/demos/medical/Kitware-R18.yml | 3 +- 7 files changed, 161 insertions(+), 11 deletions(-) create mode 100644 config/task_change_support/object_detection/object_and_hand_detection.yml create mode 100644 ros/angel_utils/python/angel_utils/system_commands.py diff --git a/config/task_change_support/object_detection/object_and_hand_detection.yml b/config/task_change_support/object_detection/object_and_hand_detection.yml new file mode 100644 index 000000000..ea6c25deb --- /dev/null +++ b/config/task_change_support/object_detection/object_and_hand_detection.yml @@ -0,0 +1,19 @@ +# +# File paths will be resolved expanding environment variables via +# `os.path.expandvars`. +# If a variable is referenced that does not exist in the environment then it +# will not be expanded and the text will be left as is. +# +# Be sure to consider the environment varaibles set by the tmuxinator +# configuration that will ultimately be running the node that consumes this +# file. +# +task_config: + m2: + object_net_checkpoint: "${MODEL_DIR}/object_detector/m2_det.pt" + m3: + object_net_checkpoint: "${MODEL_DIR}/object_detector/m3_det.pt" + m5: + object_net_checkpoint: "${MODEL_DIR}/object_detector/m5_det.pt" + r18: + object_net_checkpoint: "${MODEL_DIR}/object_detector/r18_det.pt" diff --git a/poetry.lock b/poetry.lock index df756c6f2..f37947ff0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -7412,4 +7412,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "==3.8.10" -content-hash = "a79aa4a2e42563d8b80cd1bc8ee945ae2fd4df2937a68becf37fba226c0edc49" +content-hash = "af475d673a83cedb5346df77fe06e0debb1084c45b1e3aeb82776eec3a5f8c89" diff --git a/pyproject.toml b/pyproject.toml index f873db5cc..032ee8689 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ openpyxl = "^3.0.10" pandas = ">=1.4.3" Pillow = "=9.1.0" pyarrow = ">=9.0.0" # for feature file support +pydantic = "^1" pynput = "^1.7.6" wheel = "<=0.43.0" python-dotenv = ">=1.0.0" diff --git a/ros/angel_system_nodes/angel_system_nodes/object_detection/object_and_hand_detection.py b/ros/angel_system_nodes/angel_system_nodes/object_detection/object_and_hand_detection.py index b8b9c4851..bdea4c9b7 100644 --- a/ros/angel_system_nodes/angel_system_nodes/object_detection/object_and_hand_detection.py +++ b/ros/angel_system_nodes/angel_system_nodes/object_detection/object_and_hand_detection.py @@ -1,13 +1,16 @@ +import os.path from pathlib import Path -from threading import Event, Lock, Thread -from typing import Union +from threading import Event, RLock, Thread +from typing import Dict, Union from cv_bridge import CvBridge import cv2 import numpy as np +from pydantic import BaseModel, validator from rclpy.callback_groups import MutuallyExclusiveCallbackGroup, ReentrantCallbackGroup from rclpy.node import Node, ParameterDescriptor, Parameter from sensor_msgs.msg import Image +import yaml from yolov7.detect_ptg import load_model, predict_image from angel_system.object_detection.yolov8_detect import predict_hands @@ -19,14 +22,44 @@ from angel_system.utils.event import WaitAndClearEvent from angel_system.utils.simple_timer import SimpleTimer -from angel_msgs.msg import ObjectDetection2dSet +from angel_msgs.msg import ObjectDetection2dSet, SystemCommands from angel_utils import declare_and_get_parameters, RateTracker # , DYNAMIC_TYPE from angel_utils import make_default_main +from angel_utils.system_commands import task_int_to_str BRIDGE = CvBridge() +class OaHDTaskConfig(BaseModel): + """ + Structure of the configuration for a single task + for representation and validation. + """ + object_net_checkpoint: Path + + @validator("object_net_checkpoint") + @classmethod + def expand_vars(cls, v: Path) -> Path: + """ + Expand path values with environment variables via os.path.expandvars + """ + p = Path(os.path.expandvars(v)) + if not p.is_file(): + raise ValueError(f"The file \"{p}\" does not exist or was not a file.") + elif not os.access(p, os.R_OK): + raise ValueError(f"The file \"{p}\" is not readable.") + return p + + +class OaHDPerTaskConfig(BaseModel): + """ + Structure of the whole configuration file across multiple tasks + for representation and validation. + """ + task_config: Dict[str, OaHDTaskConfig] + + class ObjectAndHandDetector(Node): """ ROS node that runs the yolov7 object detector model and outputs @@ -45,8 +78,15 @@ def __init__(self): # Required parameter (no defaults) ("image_topic",), ("det_topic",), - ("object_net_checkpoint",), + ("system_commands_topic",), ("hand_net_checkpoint",), + ("per_task_config",), + # The task to load parameters for initially, from our per-task + # config. This of course needs to be a valid "task_config" + # entry in the input configuration file. Intentionally not + # setting a default as this probably depends on the system + # configuration. + ("default_task",), ################################## # Defaulted parameters ("inference_img_size", 1280), # inference size (pixels) @@ -64,9 +104,14 @@ def __init__(self): ) self._image_topic = param_values["image_topic"] self._det_topic = param_values["det_topic"] - - self._object_model_ckpt_fp = Path(param_values["object_net_checkpoint"]) + self._system_commands_topic = param_values["system_commands_topic"] self._hand_model_chpt_fp = Path(param_values["hand_net_checkpoint"]) + with open(param_values["per_task_config"]) as config_file: + # Load and validate config into structure. + self._per_task_config = OaHDPerTaskConfig( + **yaml.load(config_file, yaml.Loader) + ) + self._default_task = param_values["default_task"] self._inference_img_size = param_values["inference_img_size"] self._det_conf_thresh = param_values["det_conf_threshold"] @@ -77,7 +122,23 @@ def __init__(self): self._enable_trace_logging = param_values["enable_time_trace_logging"] + # Lock to protect task-dependent logic. A simple mutex is appropriate + # for this implementation because the critical sections here will be + # the logic that changes task configuration and the object detection + # inference logic. If configuration usage becomes something that + # multiple concurrent agents use, then a reader-writer lock will need + # to be utilized. + self._task_dependent_lock = RLock() + + # Timestamp of the latest configuration change. Input messages before + # this time should not be processed as they are stale relative to the + # configuration change. This should be in nanoseconds format. + self._timestamp_min = 0 + # Object Model + # TODO: handle with task change logic. Initially "task change" to the + # default task configured. + self._object_model_ckpt_fp = Path(param_values["object_net_checkpoint"]) self.object_model: Union[yolov7.models.yolo.Model, TracedModel] if not self._object_model_ckpt_fp.is_file(): raise ValueError( @@ -95,16 +156,23 @@ def __init__(self): # Single slot for latest image message to process detection over. self._cur_image_msg: Image = None - self._cur_image_msg_lock = Lock() + self._cur_image_msg_lock = RLock() # Initialize ROS hooks - self._subscription = self.create_subscription( + self._image_subscription = self.create_subscription( Image, self._image_topic, self.listener_callback, 1, callback_group=MutuallyExclusiveCallbackGroup(), ) + self._sys_cmd_subscription = self.create_subscription( + Image, + self._system_commands_topic, + self.system_commands_callback, + 1, + callback_group=MutuallyExclusiveCallbackGroup(), + ) self._det_publisher = self.create_publisher( ObjectDetection2dSet, self._det_topic, @@ -142,6 +210,18 @@ def __init__(self): self._rt_thread.daemon = True self._rt_thread.start() + def load_task_configuration(self, task_name: str) -> None: + """ + Load task specific configuration parameters for the given task name. + If the task name provided is not one supported, an exception is raised. + + :param task_name: Name of the task to change to. + """ + with self._task_dependent_lock: + self._object_model_ckpt_fp = ( + self._per_task_config.task_config[task_name].object_net_checkpoint + ) + def listener_callback(self, image: Image): """ Callback function for image messages. Runs the berkeley object detector @@ -154,6 +234,17 @@ def listener_callback(self, image: Image): self._cur_image_msg = image self._rt_awake_evt.set() + def system_commands_callback(self, msg: SystemCommands): + """ + Handle receiving a SystemCommands message. + """ + # Handle a request to changing the task + if msg.change_task != SystemCommands.TASK_NO_CHANGE: + raise NotImplementedError( + f"Not yet handling task changes for requested task " + f"{task_int_to_str(msg.change_task)}" + ) + def rt_alive(self) -> bool: """ Check that the prediction runtime is still alive and raise an exception diff --git a/ros/angel_system_nodes/angel_system_nodes/task_monitoring/global_step_predictor.py b/ros/angel_system_nodes/angel_system_nodes/task_monitoring/global_step_predictor.py index 4b184db33..3ab4abe09 100644 --- a/ros/angel_system_nodes/angel_system_nodes/task_monitoring/global_step_predictor.py +++ b/ros/angel_system_nodes/angel_system_nodes/task_monitoring/global_step_predictor.py @@ -214,7 +214,9 @@ def sys_cmd_callback(self, sys_cmd_msg: SystemCommands): elif self._step_mode == "granular" and sys_cmd_msg.previous_step: update_function = self.gsp.decrement_granular_step else: - # This should never happen + # No next/previous step command received, nothing to do. + # TODO: Handle resetting the current task if that command is + # set. return try: diff --git a/ros/angel_utils/python/angel_utils/system_commands.py b/ros/angel_utils/python/angel_utils/system_commands.py new file mode 100644 index 000000000..708f358cb --- /dev/null +++ b/ros/angel_utils/python/angel_utils/system_commands.py @@ -0,0 +1,36 @@ +from functools import lru_cache +from typing import Dict + +from angel_msgs.msg import SystemCommands + + +__all__ = [ + "task_int_to_str" +] + + +@lru_cache() +def _get_sys_cmds_task_map() -> Dict[int, str]: + """ + Generate the TASK enumeration value to string mapping based on the contents + of the SystemCommands structure at the time of the call. + + This should allow this functionality to be dynamic to any changes in the + SystemCommands message. + + :returns: Dictionary mapping integer SystemCommands.TASK_* values into + their enumeration names. + """ + return {v: k for k, v in SystemCommands.__dict__.items() if k.startswith("TASK_")} + + +def task_int_to_str(i: int) -> str: + """ + Convert the integer value intended to represent one of the "TASK_*" + enumeration values in the `angel_msgs.msg.SystemCommands` message type into + the enumeration property string name. + + :param i: Task enumeration value. + :return: String name of the enumeration value. + """ + return _get_sys_cmds_task_map()[i] diff --git a/tmux/demos/medical/Kitware-R18.yml b/tmux/demos/medical/Kitware-R18.yml index 742c432e2..2382400d9 100644 --- a/tmux/demos/medical/Kitware-R18.yml +++ b/tmux/demos/medical/Kitware-R18.yml @@ -98,8 +98,9 @@ windows: -r __ns:=${ROS_NAMESPACE} -p image_topic:=PVFramesBGR -p det_topic:=ObjectDetections2d - -p object_net_checkpoint:=${MODEL_DIR}/object_detector/r18_det.pt -p hand_net_checkpoint:=${MODEL_DIR}/object_detector/hands_model.pt + -p per_task_config:=${CONFIG_DIR}/task_change_support/object_detection/object_and_hand_detection.yml + -p default_task:=r18 - activity_classifier: ros2 run angel_system_nodes activity_classifier_tcn --ros-args -r __ns:=${ROS_NAMESPACE}