simplest step.ron and basic loading

bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts

@@ -6,6 +6,15 @@
 */
 export function loadPoseFile(url: string): Promise<void>;
 /**
+* @param {string} url
+* @returns {Promise<void>}
+*/
+export function loadStepFile(url: string): Promise<void>;
+/**
+* @returns {(StepInfo)[]}
+*/
+export function steps(): (StepInfo)[];
+/**
 * Coordinate for Keypoints
 *
 * The coordinate system is growing down (y-axis), right (x-axis), and away
@@ -220,6 +229,15 @@ export class Skeletons {
   side: Skeleton;
 }
 /**
+* Information about a step for display in the frontend.
+*/
+export class StepInfo {
+  free(): void;
+/**
+*/
+  readonly name: string;
+}
+/**
 */
 export class Tracker {
   free(): void;

bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts

@@ -8,9 +8,12 @@ export function keypoints_new(a: number, b: number): number;
 export function keypointsside_new(a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number): number;
 export function limberror_name(a: number, b: number): void;
 export function loadPoseFile(a: number, b: number): number;
+export function loadStepFile(a: number, b: number): number;
 export function poseapproximation_limbErrors(a: number, b: number): void;
 export function poseapproximation_name(a: number, b: number): void;
 export function poseapproximation_worstLimbs(a: number, b: number, c: number): void;
+export function stepinfo_name(a: number, b: number): void;
+export function steps(a: number): void;
 export function tracker_addKeypoints(a: number, b: number, c: number): number;
 export function tracker_allPoseErrors(a: number, b: number, c: number): void;
 export function tracker_bestFitPose(a: number, b: number, c: number): number;
@@ -82,6 +85,7 @@ export function __wbg_set_skeletons_side(a: number, b: number): void;
 export function __wbg_skeleton_free(a: number): void;
 export function __wbg_skeletons_free(a: number): void;
 export function __wbg_skeletonside_free(a: number): void;
+export function __wbg_stepinfo_free(a: number): void;
 export function __wbg_tracker_free(a: number): void;
 export function __wbindgen_add_to_stack_pointer(a: number): number;
 export function __wbindgen_export_0(a: number, b: number): number;

bouncy_frontend/src/lib/pose.js

@@ -13,7 +13,7 @@
  */
 
 import { PoseLandmarker, FilesetResolver } from '@mediapipe/tasks-vision';
-import { Cartesian3d, Keypoints, KeypointsSide, loadPoseFile } from './instructor/bouncy_instructor';
+import { Cartesian3d, Keypoints, KeypointsSide, loadPoseFile, loadStepFile } from './instructor/bouncy_instructor';
 
 
 export function landmarksToKeypoints(landmarks) {
@@ -64,6 +64,7 @@ export class PoseDetection {
     static async new(consumer) {
         const mp = await initMediaPipeBackend();
         await loadPoseFile('/pose.ron').catch((e) => console.error(e));
+        await loadStepFile('/step.ron').catch((e) => console.error(e));
         return new PoseDetection(consumer, mp);
     }
 

bouncy_frontend/static/step.ron

@@ -0,0 +1,15 @@
+#![enable(implicit_some)]
+(
+  version: 0,
+  steps: [
+    (
+      name: "Running Man",
+      keyframes: [
+        (pose: "right-forward", orientation: Right),
+        (pose: "left-up", orientation: Right),
+        (pose: "left-forward", orientation: Right),
+        (pose: "right-up", orientation: Right),
+      ]
+    ),
+  ]
+)

bouncy_instructor/src/intern.rs

@@ -7,3 +7,4 @@ pub(crate) mod pose;
 pub(crate) mod pose_db;
 pub(crate) mod pose_score;
 pub(crate) mod skeleton_3d;
+pub(crate) mod step;

bouncy_instructor/src/intern/geom.rs

@@ -4,5 +4,5 @@ pub(crate) use angle3d::Angle3d;
 pub(crate) use signed_angle::SignedAngle;
 
 mod angle3d;
+mod cartesian;
 mod signed_angle;
-mod cartesian;

bouncy_instructor/src/intern/step.rs

@@ -0,0 +1,4 @@
+pub(crate) struct Step {
+    pub name: String,
+    // TODO: add other fields
+}

bouncy_instructor/src/lib.rs

@@ -8,13 +8,58 @@ mod test_utils;
 pub use public::*;
 
 use intern::pose_db::LimbPositionDatabase;
+use intern::step::Step;
 use std::cell::RefCell;
 
 /// Singleton internal state, shared between `Tracker` instances that run in the
 /// same JS worker thread.
 struct State {
     db: LimbPositionDatabase,
+    steps: Vec<Step>,
 }
 thread_local! {
-    static STATE: RefCell<State> = State { db: Default::default() }.into();
+    static STATE: RefCell<State> =
+        State {
+            db: Default::default(),
+            steps: Default::default()
+        }.into();
+}
+
+impl State {
+    fn add_poses(
+        &mut self,
+        poses: Vec<pose_file::Pose>,
+    ) -> Result<(), intern::pose_db::AddPoseError> {
+        self.db.add(poses)
+    }
+
+    fn add_steps(&mut self, steps: &[step_file::Step]) -> Result<(), AddStepError> {
+        for def in steps {
+            for frame in &def.keyframes {
+                let _pose = self
+                    .db
+                    .pose_by_id(&frame.pose)
+                    .ok_or_else(|| AddStepError::MissingPose(frame.pose.clone()))?;
+            }
+
+            let new_step = Step {
+                name: def.name.clone(),
+            };
+            self.steps.push(new_step);
+        }
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+enum AddStepError {
+    MissingPose(String),
+}
+
+impl From<AddStepError> for pose_file::ParseFileError {
+    fn from(error: AddStepError) -> Self {
+        match error {
+            AddStepError::MissingPose(id) => Self::UnknownPoseReference(id),
+        }
+    }
 }

bouncy_instructor/src/public.rs

@@ -3,12 +3,16 @@
 pub(crate) mod keypoints;
 pub(crate) mod pose_file;
 pub(crate) mod skeleton;
+pub(crate) mod step_file;
+pub(crate) mod step_info;
 pub(crate) mod tracker;
 
 pub use keypoints::{Keypoints, Side as KeypointsSide};
+pub use step_info::StepInfo;
 pub use tracker::Tracker;
 
 use self::pose_file::ParseFileError;
+use self::step_file::StepFile;
 use super::STATE;
 use pose_file::PoseFile;
 use wasm_bindgen::prelude::wasm_bindgen;
@@ -18,29 +22,53 @@ use web_sys::Request;
 
 #[wasm_bindgen(js_name = loadPoseFile)]
 pub async fn load_pose_file(url: &str) -> Result<(), JsValue> {
+    let text = load_text_file(url).await?;
+    load_pose_str(&text)?;
+    Ok(())
+}
+
+#[wasm_bindgen(js_name = loadStepFile)]
+pub async fn load_step_file(url: &str) -> Result<(), JsValue> {
+    let text = load_text_file(url).await?;
+    load_step_str(&text)?;
+    Ok(())
+}
+
+#[wasm_bindgen]
+pub fn steps() -> Vec<StepInfo> {
+    STATE.with_borrow(|state| state.steps.iter().map(StepInfo::from).collect::<Vec<_>>())
+}
+
+pub fn load_pose_str(text: &str) -> Result<(), ParseFileError> {
+    let parsed = PoseFile::from_str(&text)?;
+    STATE.with(|state| state.borrow_mut().add_poses(parsed.poses))?;
+    Ok(())
+}
+
+pub fn load_step_str(text: &str) -> Result<(), ParseFileError> {
+    let parsed = StepFile::from_str(&text)?;
+    STATE.with(|state| state.borrow_mut().add_steps(&parsed.steps))?;
+    Ok(())
+}
+
+async fn load_text_file(url: &str) -> Result<String, JsValue> {
     let request = Request::new_with_str(&url)?;
 
     let window = web_sys::window().unwrap();
     let resp_value = JsFuture::from(window.fetch_with_request(&request)).await?;
     let resp: web_sys::Response = resp_value.dyn_into().unwrap();
     let js_value = JsFuture::from(resp.text()?).await?;
     let text = js_value.as_string().ok_or("Not a string")?;
-    load_pose_str(&text)?;
-    Ok(())
+    Ok(text)
 }
 
-pub fn load_pose_str(text: &str) -> Result<(), ParseFileError> {
-    let parsed = PoseFile::from_str(&text)?;
-    STATE.with(|state| state.borrow_mut().db.add(parsed.poses))?;
-    Ok(())
-}
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
     fn test_valid_pose_reference() {
-        let input = r#"
+        let pose_str = r#"
         #![enable(implicit_some)]
         (
           version: 0,
@@ -60,7 +88,23 @@ mod tests {
           ]
         )
         "#;
-        load_pose_str(input).unwrap();
+        let step_str = r#"
+        #![enable(implicit_some)]
+        (
+          version: 0,
+          steps: [
+            (
+              name: "Running Man",
+              keyframes: [
+                (pose: "test-pose-left", orientation: Right),
+                (pose: "test-pose-right", orientation: Right),
+              ]
+            ),
+          ]
+        )
+        "#;
+        load_pose_str(pose_str).unwrap();
+        load_step_str(step_str).unwrap();
         let num_poses = STATE.with_borrow(|state| state.db.poses().len());
         assert_eq!(num_poses, 2);
     }

bouncy_instructor/src/public/pose_file.rs

@@ -1,4 +1,5 @@
-//! Defines the external format for defining poses.
+//! Defines the external format for defining poses, which are still positions of
+//! a body.
 //!
 //! Best practice: Don't use any of the type of this file outside of parsing
 //! logic. Instead, translate to internal types. This allows refactoring

bouncy_instructor/src/public/step_file.rs

@@ -0,0 +1,74 @@
+//! Defines the external format for defining steps, which are a combination of
+//! poses.
+//!
+//! Best practice: Don't use any of the type of this file outside of parsing
+//! logic. Instead, translate to internal types. This allows refactoring
+//! internal without changing the external formats.
+
+use crate::pose_file::ParseFileError;
+use serde::{Deserialize, Serialize};
+
+const CURRENT_VERSION: u16 = 0;
+
+/// Format for pose definition files.
+#[derive(Deserialize)]
+pub(crate) struct StepFile {
+    pub version: u16,
+    pub steps: Vec<Step>,
+}
+
+/// Description of a step.
+///
+/// A step is a sequence of poses with timing and orientation information.
+/// This is the format for external files and loaded in at runtime.
+/// It is converted to a [`crate::step::Step`] for step detection.
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+pub(crate) struct Step {
+    pub name: String,
+    pub keyframes: Vec<StepPosition>,
+}
+
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+pub(crate) struct StepPosition {
+    /// Reference to the name of a pose
+    pub pose: String,
+    /// specify how the pose should be oriented
+    #[serde(default, skip_serializing_if = "Orientation::any")]
+    pub orientation: Orientation,
+}
+
+/// Define in which direction a pose should be oriented.
+#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
+pub(crate) enum Orientation {
+    ToCamera,
+    Right,
+    Away,
+    Left,
+    /// It doesn't matter in which direction the pose is done.
+    Any,
+}
+
+impl StepFile {
+    pub(crate) fn from_str(text: &str) -> Result<Self, ParseFileError> {
+        let parsed: StepFile = ron::from_str(text)?;
+        if parsed.version != CURRENT_VERSION {
+            return Err(ParseFileError::VersionMismatch {
+                expected: CURRENT_VERSION,
+                found: parsed.version,
+            });
+        }
+        Ok(parsed)
+    }
+}
+
+impl Orientation {
+    fn any(&self) -> bool {
+        matches!(self, Orientation::Any)
+    }
+}
+
+impl Default for Orientation {
+    fn default() -> Self {
+        Self::Any
+    }
+}

bouncy_instructor/src/public/step_info.rs

@@ -0,0 +1,26 @@
+use crate::intern::step::Step;
+use wasm_bindgen::prelude::wasm_bindgen;
+
+/// Information about a step for display in the frontend.
+#[derive(Debug)]
+#[wasm_bindgen]
+pub struct StepInfo {
+    name: String,
+    // TODO: other fields
+}
+
+#[wasm_bindgen]
+impl StepInfo {
+    #[wasm_bindgen(getter)]
+    pub fn name(&self) -> String {
+        self.name.clone()
+    }
+}
+
+impl From<&Step> for StepInfo {
+    fn from(value: &Step) -> Self {
+        Self {
+            name: value.name.clone(),
+        }
+    }
+}

bouncy_instructor/src/web_utils.rs

@@ -1,15 +1,15 @@
 #[macro_export]
-#[cfg(target_arch="wasm32")]
+#[cfg(target_arch = "wasm32")]
 macro_rules! println {
     ( $( $t:tt )* ) => {
         web_sys::console::log_1(&format!( $( $t )* ).into());
     }
 }
 
 #[macro_export]
-#[cfg(not(target_arch="wasm32"))]
+#[cfg(not(target_arch = "wasm32"))]
 macro_rules! println {
     ( $( $t:tt )* ) => {
         println!( $( $t )* );
     }
-}
+}