From 1412c8aafc11c4ab34efe7b47afca2fc8d8ae67b Mon Sep 17 00:00:00 2001 From: Jakob Meier Date: Thu, 14 Dec 2023 23:09:46 +0100 Subject: [PATCH] simplest step.ron and basic loading --- .../src/lib/instructor/bouncy_instructor.d.ts | 18 +++++ .../instructor/bouncy_instructor_bg.wasm.d.ts | 4 + bouncy_frontend/src/lib/pose.js | 3 +- bouncy_frontend/static/step.ron | 15 ++++ bouncy_instructor/src/intern.rs | 1 + bouncy_instructor/src/intern/geom.rs | 2 +- bouncy_instructor/src/intern/step.rs | 4 + bouncy_instructor/src/lib.rs | 47 +++++++++++- bouncy_instructor/src/public.rs | 62 +++++++++++++--- bouncy_instructor/src/public/pose_file.rs | 3 +- bouncy_instructor/src/public/step_file.rs | 74 +++++++++++++++++++ bouncy_instructor/src/public/step_info.rs | 26 +++++++ bouncy_instructor/src/web_utils.rs | 6 +- bouncy_instructor/tests/common/mod.rs | 11 ++- bouncy_instructor/tests/data/step.ron | 1 + bouncy_instructor/tests/step_tests.rs | 19 +++++ 16 files changed, 278 insertions(+), 18 deletions(-) create mode 100644 bouncy_frontend/static/step.ron create mode 100644 bouncy_instructor/src/intern/step.rs create mode 100644 bouncy_instructor/src/public/step_file.rs create mode 100644 bouncy_instructor/src/public/step_info.rs create mode 120000 bouncy_instructor/tests/data/step.ron create mode 100644 bouncy_instructor/tests/step_tests.rs diff --git a/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts b/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts index c856a8c..e6c96cf 100644 --- a/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts +++ b/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts @@ -6,6 +6,15 @@ */ export function loadPoseFile(url: string): Promise; /** +* @param {string} url +* @returns {Promise} +*/ +export function loadStepFile(url: string): Promise; +/** +* @returns {(StepInfo)[]} +*/ +export function steps(): (StepInfo)[]; +/** * Coordinate for Keypoints * * The coordinate system is growing down (y-axis), right (x-axis), and away @@ -220,6 +229,15 @@ export class Skeletons { side: Skeleton; } /** +* Information about a step for display in the frontend. +*/ +export class StepInfo { + free(): void; +/** +*/ + readonly name: string; +} +/** */ export class Tracker { free(): void; diff --git a/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts b/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts index 63f9be9..c9ebe94 100644 --- a/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts +++ b/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts @@ -8,9 +8,12 @@ export function keypoints_new(a: number, b: number): number; export function keypointsside_new(a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number): number; export function limberror_name(a: number, b: number): void; export function loadPoseFile(a: number, b: number): number; +export function loadStepFile(a: number, b: number): number; export function poseapproximation_limbErrors(a: number, b: number): void; export function poseapproximation_name(a: number, b: number): void; export function poseapproximation_worstLimbs(a: number, b: number, c: number): void; +export function stepinfo_name(a: number, b: number): void; +export function steps(a: number): void; export function tracker_addKeypoints(a: number, b: number, c: number): number; export function tracker_allPoseErrors(a: number, b: number, c: number): void; export function tracker_bestFitPose(a: number, b: number, c: number): number; @@ -82,6 +85,7 @@ export function __wbg_set_skeletons_side(a: number, b: number): void; export function __wbg_skeleton_free(a: number): void; export function __wbg_skeletons_free(a: number): void; export function __wbg_skeletonside_free(a: number): void; +export function __wbg_stepinfo_free(a: number): void; export function __wbg_tracker_free(a: number): void; export function __wbindgen_add_to_stack_pointer(a: number): number; export function __wbindgen_export_0(a: number, b: number): number; diff --git a/bouncy_frontend/src/lib/pose.js b/bouncy_frontend/src/lib/pose.js index 49a635d..a27099a 100644 --- a/bouncy_frontend/src/lib/pose.js +++ b/bouncy_frontend/src/lib/pose.js @@ -13,7 +13,7 @@ */ import { PoseLandmarker, FilesetResolver } from '@mediapipe/tasks-vision'; -import { Cartesian3d, Keypoints, KeypointsSide, loadPoseFile } from './instructor/bouncy_instructor'; +import { Cartesian3d, Keypoints, KeypointsSide, loadPoseFile, loadStepFile } from './instructor/bouncy_instructor'; export function landmarksToKeypoints(landmarks) { @@ -64,6 +64,7 @@ export class PoseDetection { static async new(consumer) { const mp = await initMediaPipeBackend(); await loadPoseFile('/pose.ron').catch((e) => console.error(e)); + await loadStepFile('/step.ron').catch((e) => console.error(e)); return new PoseDetection(consumer, mp); } diff --git a/bouncy_frontend/static/step.ron b/bouncy_frontend/static/step.ron new file mode 100644 index 0000000..fc9fab8 --- /dev/null +++ b/bouncy_frontend/static/step.ron @@ -0,0 +1,15 @@ +#![enable(implicit_some)] +( + version: 0, + steps: [ + ( + name: "Running Man", + keyframes: [ + (pose: "right-forward", orientation: Right), + (pose: "left-up", orientation: Right), + (pose: "left-forward", orientation: Right), + (pose: "right-up", orientation: Right), + ] + ), + ] +) \ No newline at end of file diff --git a/bouncy_instructor/src/intern.rs b/bouncy_instructor/src/intern.rs index af24f44..6f2912c 100644 --- a/bouncy_instructor/src/intern.rs +++ b/bouncy_instructor/src/intern.rs @@ -7,3 +7,4 @@ pub(crate) mod pose; pub(crate) mod pose_db; pub(crate) mod pose_score; pub(crate) mod skeleton_3d; +pub(crate) mod step; diff --git a/bouncy_instructor/src/intern/geom.rs b/bouncy_instructor/src/intern/geom.rs index 0c99c0f..46030c9 100644 --- a/bouncy_instructor/src/intern/geom.rs +++ b/bouncy_instructor/src/intern/geom.rs @@ -4,5 +4,5 @@ pub(crate) use angle3d::Angle3d; pub(crate) use signed_angle::SignedAngle; mod angle3d; +mod cartesian; mod signed_angle; -mod cartesian; \ No newline at end of file diff --git a/bouncy_instructor/src/intern/step.rs b/bouncy_instructor/src/intern/step.rs new file mode 100644 index 0000000..c231a15 --- /dev/null +++ b/bouncy_instructor/src/intern/step.rs @@ -0,0 +1,4 @@ +pub(crate) struct Step { + pub name: String, + // TODO: add other fields +} diff --git a/bouncy_instructor/src/lib.rs b/bouncy_instructor/src/lib.rs index 6ba009f..158c78d 100644 --- a/bouncy_instructor/src/lib.rs +++ b/bouncy_instructor/src/lib.rs @@ -8,13 +8,58 @@ mod test_utils; pub use public::*; use intern::pose_db::LimbPositionDatabase; +use intern::step::Step; use std::cell::RefCell; /// Singleton internal state, shared between `Tracker` instances that run in the /// same JS worker thread. struct State { db: LimbPositionDatabase, + steps: Vec, } thread_local! { - static STATE: RefCell = State { db: Default::default() }.into(); + static STATE: RefCell = + State { + db: Default::default(), + steps: Default::default() + }.into(); +} + +impl State { + fn add_poses( + &mut self, + poses: Vec, + ) -> Result<(), intern::pose_db::AddPoseError> { + self.db.add(poses) + } + + fn add_steps(&mut self, steps: &[step_file::Step]) -> Result<(), AddStepError> { + for def in steps { + for frame in &def.keyframes { + let _pose = self + .db + .pose_by_id(&frame.pose) + .ok_or_else(|| AddStepError::MissingPose(frame.pose.clone()))?; + } + + let new_step = Step { + name: def.name.clone(), + }; + self.steps.push(new_step); + } + Ok(()) + } +} + +#[derive(Debug)] +enum AddStepError { + MissingPose(String), +} + +impl From for pose_file::ParseFileError { + fn from(error: AddStepError) -> Self { + match error { + AddStepError::MissingPose(id) => Self::UnknownPoseReference(id), + } + } } diff --git a/bouncy_instructor/src/public.rs b/bouncy_instructor/src/public.rs index fa9d92d..e201156 100644 --- a/bouncy_instructor/src/public.rs +++ b/bouncy_instructor/src/public.rs @@ -3,12 +3,16 @@ pub(crate) mod keypoints; pub(crate) mod pose_file; pub(crate) mod skeleton; +pub(crate) mod step_file; +pub(crate) mod step_info; pub(crate) mod tracker; pub use keypoints::{Keypoints, Side as KeypointsSide}; +pub use step_info::StepInfo; pub use tracker::Tracker; use self::pose_file::ParseFileError; +use self::step_file::StepFile; use super::STATE; use pose_file::PoseFile; use wasm_bindgen::prelude::wasm_bindgen; @@ -18,6 +22,36 @@ use web_sys::Request; #[wasm_bindgen(js_name = loadPoseFile)] pub async fn load_pose_file(url: &str) -> Result<(), JsValue> { + let text = load_text_file(url).await?; + load_pose_str(&text)?; + Ok(()) +} + +#[wasm_bindgen(js_name = loadStepFile)] +pub async fn load_step_file(url: &str) -> Result<(), JsValue> { + let text = load_text_file(url).await?; + load_step_str(&text)?; + Ok(()) +} + +#[wasm_bindgen] +pub fn steps() -> Vec { + STATE.with_borrow(|state| state.steps.iter().map(StepInfo::from).collect::>()) +} + +pub fn load_pose_str(text: &str) -> Result<(), ParseFileError> { + let parsed = PoseFile::from_str(&text)?; + STATE.with(|state| state.borrow_mut().add_poses(parsed.poses))?; + Ok(()) +} + +pub fn load_step_str(text: &str) -> Result<(), ParseFileError> { + let parsed = StepFile::from_str(&text)?; + STATE.with(|state| state.borrow_mut().add_steps(&parsed.steps))?; + Ok(()) +} + +async fn load_text_file(url: &str) -> Result { let request = Request::new_with_str(&url)?; let window = web_sys::window().unwrap(); @@ -25,22 +59,16 @@ pub async fn load_pose_file(url: &str) -> Result<(), JsValue> { let resp: web_sys::Response = resp_value.dyn_into().unwrap(); let js_value = JsFuture::from(resp.text()?).await?; let text = js_value.as_string().ok_or("Not a string")?; - load_pose_str(&text)?; - Ok(()) + Ok(text) } -pub fn load_pose_str(text: &str) -> Result<(), ParseFileError> { - let parsed = PoseFile::from_str(&text)?; - STATE.with(|state| state.borrow_mut().db.add(parsed.poses))?; - Ok(()) -} #[cfg(test)] mod tests { use super::*; #[test] fn test_valid_pose_reference() { - let input = r#" + let pose_str = r#" #![enable(implicit_some)] ( version: 0, @@ -60,7 +88,23 @@ mod tests { ] ) "#; - load_pose_str(input).unwrap(); + let step_str = r#" + #![enable(implicit_some)] + ( + version: 0, + steps: [ + ( + name: "Running Man", + keyframes: [ + (pose: "test-pose-left", orientation: Right), + (pose: "test-pose-right", orientation: Right), + ] + ), + ] + ) + "#; + load_pose_str(pose_str).unwrap(); + load_step_str(step_str).unwrap(); let num_poses = STATE.with_borrow(|state| state.db.poses().len()); assert_eq!(num_poses, 2); } diff --git a/bouncy_instructor/src/public/pose_file.rs b/bouncy_instructor/src/public/pose_file.rs index 93cf514..a0a710a 100644 --- a/bouncy_instructor/src/public/pose_file.rs +++ b/bouncy_instructor/src/public/pose_file.rs @@ -1,4 +1,5 @@ -//! Defines the external format for defining poses. +//! Defines the external format for defining poses, which are still positions of +//! a body. //! //! Best practice: Don't use any of the type of this file outside of parsing //! logic. Instead, translate to internal types. This allows refactoring diff --git a/bouncy_instructor/src/public/step_file.rs b/bouncy_instructor/src/public/step_file.rs new file mode 100644 index 0000000..1873f22 --- /dev/null +++ b/bouncy_instructor/src/public/step_file.rs @@ -0,0 +1,74 @@ +//! Defines the external format for defining steps, which are a combination of +//! poses. +//! +//! Best practice: Don't use any of the type of this file outside of parsing +//! logic. Instead, translate to internal types. This allows refactoring +//! internal without changing the external formats. + +use crate::pose_file::ParseFileError; +use serde::{Deserialize, Serialize}; + +const CURRENT_VERSION: u16 = 0; + +/// Format for pose definition files. +#[derive(Deserialize)] +pub(crate) struct StepFile { + pub version: u16, + pub steps: Vec, +} + +/// Description of a step. +/// +/// A step is a sequence of poses with timing and orientation information. +/// This is the format for external files and loaded in at runtime. +/// It is converted to a [`crate::step::Step`] for step detection. +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub(crate) struct Step { + pub name: String, + pub keyframes: Vec, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub(crate) struct StepPosition { + /// Reference to the name of a pose + pub pose: String, + /// specify how the pose should be oriented + #[serde(default, skip_serializing_if = "Orientation::any")] + pub orientation: Orientation, +} + +/// Define in which direction a pose should be oriented. +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] +pub(crate) enum Orientation { + ToCamera, + Right, + Away, + Left, + /// It doesn't matter in which direction the pose is done. + Any, +} + +impl StepFile { + pub(crate) fn from_str(text: &str) -> Result { + let parsed: StepFile = ron::from_str(text)?; + if parsed.version != CURRENT_VERSION { + return Err(ParseFileError::VersionMismatch { + expected: CURRENT_VERSION, + found: parsed.version, + }); + } + Ok(parsed) + } +} + +impl Orientation { + fn any(&self) -> bool { + matches!(self, Orientation::Any) + } +} + +impl Default for Orientation { + fn default() -> Self { + Self::Any + } +} diff --git a/bouncy_instructor/src/public/step_info.rs b/bouncy_instructor/src/public/step_info.rs new file mode 100644 index 0000000..e049b87 --- /dev/null +++ b/bouncy_instructor/src/public/step_info.rs @@ -0,0 +1,26 @@ +use crate::intern::step::Step; +use wasm_bindgen::prelude::wasm_bindgen; + +/// Information about a step for display in the frontend. +#[derive(Debug)] +#[wasm_bindgen] +pub struct StepInfo { + name: String, + // TODO: other fields +} + +#[wasm_bindgen] +impl StepInfo { + #[wasm_bindgen(getter)] + pub fn name(&self) -> String { + self.name.clone() + } +} + +impl From<&Step> for StepInfo { + fn from(value: &Step) -> Self { + Self { + name: value.name.clone(), + } + } +} diff --git a/bouncy_instructor/src/web_utils.rs b/bouncy_instructor/src/web_utils.rs index 04ebe00..79dfa1f 100644 --- a/bouncy_instructor/src/web_utils.rs +++ b/bouncy_instructor/src/web_utils.rs @@ -1,5 +1,5 @@ #[macro_export] -#[cfg(target_arch="wasm32")] +#[cfg(target_arch = "wasm32")] macro_rules! println { ( $( $t:tt )* ) => { web_sys::console::log_1(&format!( $( $t )* ).into()); @@ -7,9 +7,9 @@ macro_rules! println { } #[macro_export] -#[cfg(not(target_arch="wasm32"))] +#[cfg(not(target_arch = "wasm32"))] macro_rules! println { ( $( $t:tt )* ) => { println!( $( $t )* ); } -} \ No newline at end of file +} diff --git a/bouncy_instructor/tests/common/mod.rs b/bouncy_instructor/tests/common/mod.rs index 1530264..21750a7 100644 --- a/bouncy_instructor/tests/common/mod.rs +++ b/bouncy_instructor/tests/common/mod.rs @@ -1,9 +1,16 @@ +#![allow(dead_code)] //! Common code for Rust-style integration tests. -use bouncy_instructor::{load_pose_str, Tracker}; +use bouncy_instructor::{load_pose_str, load_step_str, Tracker}; -pub(crate) fn setup_tracker() -> Tracker { +pub(crate) fn load_static_files() { const POSE_STR: &str = include_str!("../data/pose.ron"); + const STEP_STR: &str = include_str!("../data/step.ron"); load_pose_str(POSE_STR).expect("loading poses"); + load_step_str(STEP_STR).expect("loading steps"); +} + +pub(crate) fn setup_tracker() -> Tracker { + load_static_files(); Tracker::new() } diff --git a/bouncy_instructor/tests/data/step.ron b/bouncy_instructor/tests/data/step.ron new file mode 120000 index 0000000..634e557 --- /dev/null +++ b/bouncy_instructor/tests/data/step.ron @@ -0,0 +1 @@ +../../../bouncy_frontend/static/step.ron \ No newline at end of file diff --git a/bouncy_instructor/tests/step_tests.rs b/bouncy_instructor/tests/step_tests.rs new file mode 100644 index 0000000..265ad4d --- /dev/null +++ b/bouncy_instructor/tests/step_tests.rs @@ -0,0 +1,19 @@ +use expect_test::expect; + +mod common; + +/// This test checks that the list of defined steps doesn't change unexpectedly. +/// Anytime a static step is added, this test needs to be updated. +#[test] +fn test_listing_static_steps() { + common::load_static_files(); + let mut step_names: Vec<_> = bouncy_instructor::steps() + .iter() + .map(|s| s.name()) + .collect(); + + step_names.sort(); + + let expect = expect!["Running Man"]; + expect.assert_eq(&step_names.join(",")); +}