interactive pose form

Now we can edit angles and see the change reflected on an avatar. Copying values from the video is done with explicit user input, from the currently visible frame of an uploaded video.
jakmeier · Oct 9, 2024 · d50f036 · d50f036
1 parent e549942
commit d50f036
Show file tree

Hide file tree

Showing 6 changed files with 213 additions and 60 deletions.
diff --git a/bouncy_frontend/src/app.css b/bouncy_frontend/src/app.css
@@ -112,6 +112,15 @@ button.wide {
     max-width: 450px;
 }
 
+button.full-width {
+    margin: 5px auto;
+    width: 100%;
+}
+
+button.short {
+    height: 50px;
+}
+
 button.locked {
     background-color: var(--theme-neutral-gray);
     color: var(--theme-neutral-dark);

diff --git a/bouncy_frontend/src/lib/components/editor/PoseInputForm.svelte b/bouncy_frontend/src/lib/components/editor/PoseInputForm.svelte
@@ -1,5 +1,9 @@
 <script>
+  import { LEFT_RIGHT_COLORING } from '$lib/constants';
   import { Skeleton } from '$lib/instructor/bouncy_instructor';
+  import { SkeletonField } from '$lib/instructor/bouncy_instructor_bg';
+  import Svg from '../avatar/Svg.svelte';
+  import SvgAvatar from '../avatar/SvgAvatar.svelte';
 
   let bodyParts = [
     { name: 'Left Arm', angle: 0, isKey: false, weight: null },
@@ -13,19 +17,23 @@
     { name: 'Left Foot', angle: 0, isKey: false, weight: null },
     { name: 'Right Foot', angle: 0, isKey: false, weight: null },
   ];
+  /** @type{Skeleton | undefined} */
+  let skeleton;
 
-  /** @param {Skeleton} skeleton */
-  export function loadSkeleton(skeleton) {
-    bodyParts[0].angle = formatAngle(skeleton.left.arm.angle);
-    bodyParts[1].angle = formatAngle(skeleton.right.arm.angle);
-    bodyParts[2].angle = formatAngle(skeleton.left.forearm.angle);
-    bodyParts[3].angle = formatAngle(skeleton.right.forearm.angle);
-    bodyParts[4].angle = formatAngle(skeleton.left.thigh.angle);
-    bodyParts[5].angle = formatAngle(skeleton.right.thigh.angle);
-    bodyParts[6].angle = formatAngle(skeleton.left.shin.angle);
-    bodyParts[7].angle = formatAngle(skeleton.right.shin.angle);
-    bodyParts[8].angle = formatAngle(skeleton.left.foot.angle);
-    bodyParts[9].angle = formatAngle(skeleton.right.foot.angle);
+  /** @param {Skeleton} newSkeleton */
+  export function loadSkeleton(newSkeleton) {
+    // TODO(performance): these are a lot of unnecessary allocation in WASM
+    bodyParts[0].angle = formatAngle(newSkeleton.left.arm.angle);
+    bodyParts[1].angle = formatAngle(newSkeleton.right.arm.angle);
+    bodyParts[2].angle = formatAngle(newSkeleton.left.forearm.angle);
+    bodyParts[3].angle = formatAngle(newSkeleton.right.forearm.angle);
+    bodyParts[4].angle = formatAngle(newSkeleton.left.thigh.angle);
+    bodyParts[5].angle = formatAngle(newSkeleton.right.thigh.angle);
+    bodyParts[6].angle = formatAngle(newSkeleton.left.shin.angle);
+    bodyParts[7].angle = formatAngle(newSkeleton.right.shin.angle);
+    bodyParts[8].angle = formatAngle(newSkeleton.left.foot.angle);
+    bodyParts[9].angle = formatAngle(newSkeleton.right.foot.angle);
+    skeleton = newSkeleton;
   }
 
   /**
@@ -36,6 +44,34 @@
     return parseFloat(((180 * number) / Math.PI).toFixed(0));
   }
 
+  /**
+   * @param {number} number
+   * @returns {number}
+   */
+  function toRad(number) {
+    return (number * Math.PI) / 180;
+  }
+
+  function updateAvatar() {
+    if (skeleton) {
+      // use setter on skeleton to avoid copying intermediate objects on the
+      // field getters generated by wasm bindgen
+      skeleton.setAngle(SkeletonField.LeftArm, toRad(bodyParts[0].angle));
+      skeleton.setAngle(SkeletonField.RightArm, toRad(bodyParts[1].angle));
+      skeleton.setAngle(SkeletonField.LeftForearm, toRad(bodyParts[2].angle));
+      skeleton.setAngle(SkeletonField.RightForearm, toRad(bodyParts[3].angle));
+      skeleton.setAngle(SkeletonField.LeftThigh, toRad(bodyParts[4].angle));
+      skeleton.setAngle(SkeletonField.RightThigh, toRad(bodyParts[5].angle));
+      skeleton.setAngle(SkeletonField.LeftShin, toRad(bodyParts[6].angle));
+      skeleton.setAngle(SkeletonField.RightShin, toRad(bodyParts[7].angle));
+      skeleton.setAngle(SkeletonField.LeftFoot, toRad(bodyParts[8].angle));
+      skeleton.setAngle(SkeletonField.RightFoot, toRad(bodyParts[9].angle));
+
+      // trigger reactivity
+      skeleton = skeleton;
+    }
+  }
+
   //   function toggleKeyPart(index) {
   //     bodyParts[index].isKey = !bodyParts[index].isKey;
   //     if (!bodyParts[index].isKey) {
@@ -44,14 +80,19 @@
   //   }
 </script>
 
-<div class="body-part-group">
+<div class="container">
   {#each bodyParts as part, index}
-    <div class="body-part">
+    <div
+      class="body-part"
+      class:left={index % 2 === 0}
+      class:right={index % 2 === 1}
+    >
       <label>{part.name}</label>
       <input
         type="number"
         class="angle-input"
         bind:value={part.angle}
+        on:change={updateAvatar}
         placeholder="Angle"
       />
       <!-- <button class="key-button" on:click={() => toggleKeyPart(index)}>
@@ -66,20 +107,48 @@
       {/if}
     </div>
   {/each}
+
+  <div class="avatar">
+    <Svg width={200} height={300} orderByZ>
+      {#if skeleton}
+        <SvgAvatar
+          {skeleton}
+          width={200}
+          height={300}
+          style={LEFT_RIGHT_COLORING}
+        ></SvgAvatar>
+      {/if}
+    </Svg>
+  </div>
 </div>
 
 <style>
-  .body-part-group {
-    display: flex;
-    flex-wrap: wrap;
+  .container {
+    display: grid;
+    grid-template-areas:
+      'left-values avatar right-values'
+      'left-values avatar right-values'
+      'left-values avatar right-values'
+      'left-values avatar right-values'
+      'left-values avatar right-values';
     gap: 8px;
-    justify-content: space-between;
+    justify-items: center;
+  }
+  /* .left {
+    grid-area: left-values;
+  }
+  .right {
+    grid-area: right-values;
+  } */
+  .avatar {
+    grid-column: 1 / 5;
+    grid-area: avatar;
   }
   .body-part {
     display: flex;
     flex-direction: column;
     align-items: center;
-    width: 45%; /* Adjust for mobile view */
+    width: 90%;
   }
   .angle-input {
     width: 100%;

diff --git a/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts b/bouncy_frontend/src/lib/instructor/bouncy_instructor.d.ts
@@ -66,6 +66,29 @@ export function dances(): (DanceInfo)[];
 export function danceBuilderFromDance(dance_id: string): DanceBuilder;
 /**
 */
+export enum SkeletonField {
+  LeftThigh = 0,
+  LeftShin = 1,
+  LeftArm = 2,
+  LeftForearm = 3,
+  LeftFoot = 4,
+  RightThigh = 5,
+  RightShin = 6,
+  RightArm = 7,
+  RightForearm = 8,
+  RightFoot = 9,
+}
+/**
+* Best guess for what the dancer needs to change to fit the pose.
+*/
+export enum PoseHint {
+  DontKnow = 0,
+  LeftRight = 1,
+  ZOrder = 2,
+  WrongDirection = 3,
+}
+/**
+*/
 export enum DetectionFailureReason {
 /**
 * The last match was too recent to have another match.
@@ -117,15 +140,6 @@ export enum DetectionState {
 */
   TrackingDone = 5,
 }
-/**
-* Best guess for what the dancer needs to change to fit the pose.
-*/
-export enum PoseHint {
-  DontKnow = 0,
-  LeftRight = 1,
-  ZOrder = 2,
-  WrongDirection = 3,
-}
 
 import type { Readable } from "svelte/store";
 
@@ -642,6 +656,17 @@ export class Segment {
 export class Skeleton {
   free(): void;
 /**
+* Compute 2d coordinates for the skeleton for rendering.
+*
+* The skeleton will be rendered assuming hard-coded values for body part
+* proportional lengths, multiplied with the size parameter. The hip
+* segment will have its center at the given position.
+* @param {Cartesian2d} hip_center
+* @param {number} size
+* @returns {SkeletonV2}
+*/
+  render(hip_center: Cartesian2d, size: number): SkeletonV2;
+/**
 * @param {boolean} sideway
 * @returns {Skeleton}
 */
@@ -655,16 +680,10 @@ export class Skeleton {
 */
   debugString(): string;
 /**
-* Compute 2d coordinates for the skeleton for rendering.
-*
-* The skeleton will be rendered assuming hard-coded values for body part
-* proportional lengths, multiplied with the size parameter. The hip
-* segment will have its center at the given position.
-* @param {Cartesian2d} hip_center
-* @param {number} size
-* @returns {SkeletonV2}
+* @param {SkeletonField} field
+* @param {number} value
 */
-  render(hip_center: Cartesian2d, size: number): SkeletonV2;
+  setAngle(field: SkeletonField, value: number): void;
 /**
 * Does the dancer face away more than they face the camera?
 */
@@ -830,15 +849,6 @@ export class StepInfo {
 export class Tracker {
   free(): void;
 /**
-* @param {number} timestamp
-* @returns {ExportedFrame}
-*/
-  exportFrame(timestamp: number): ExportedFrame;
-/**
-* @returns {string}
-*/
-  exportKeypoints(): string;
-/**
 * Create a tracker for all known steps.
 */
   constructor();
@@ -992,6 +1002,15 @@ export class Tracker {
 */
   renderedKeypointsAt(timestamp: number, width: number, height: number): SkeletonV2 | undefined;
 /**
+* @param {number} timestamp
+* @returns {ExportedFrame}
+*/
+  exportFrame(timestamp: number): ExportedFrame;
+/**
+* @returns {string}
+*/
+  exportKeypoints(): string;
+/**
 */
   readonly detectionState: ReadableDetectionState;
 /**

diff --git a/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts b/bouncy_frontend/src/lib/instructor/bouncy_instructor_bg.wasm.d.ts
@@ -76,6 +76,7 @@ export function skeleton_debugString(a: number, b: number): void;
 export function skeleton_render(a: number, b: number, c: number): number;
 export function skeleton_resting(a: number): number;
 export function skeleton_restingPose(a: number): number;
+export function skeleton_setAngle(a: number, b: number, c: number): void;
 export function stepById(a: number, b: number, c: number): number;
 export function stepinfo_beats(a: number): number;
 export function stepinfo_bodyShift(a: number, b: number): number;

diff --git a/bouncy_frontend/src/routes/dev/+page.svelte b/bouncy_frontend/src/routes/dev/+page.svelte
@@ -16,10 +16,13 @@
   /** @type {HTMLVideoElement}  */
   let video;
   let prevTime = -1;
+  let selectedTimestamp = 0;
   let videoSrcWidth = 0;
   let videoSrcHeight = 0;
-  /** @type {import("$lib/instructor/bouncy_instructor").SkeletonV2|undefined} */
+  /** @type {import("$lib/instructor/bouncy_instructor").SkeletonV2 | undefined} */
   let liveSkeleton;
+  /** @type {import("$lib/instructor/bouncy_instructor").Skeleton | undefined} */
+  let poseSkeleton;
 
   let dataListener;
   /** @type {(skeleton: import("$lib/instructor/bouncy_instructor").Skeleton)=>void} */
@@ -74,16 +77,13 @@
         if (result.landmarks && result.landmarks.length >= 1) {
           const kp = landmarksToKeypoints(result.landmarks[0]);
           tracker.addKeypoints(kp, timestamp);
-          recordingEnd = timestamp;
+          recordingEnd = Math.max(timestamp, recordingEnd);
+          selectedTimestamp = timestamp;
           liveSkeleton = tracker.renderedKeypointsAt(
             timestamp,
             videoSrcWidth,
             videoSrcHeight
           );
-          let skeleton = tracker.skeletonAt(timestamp);
-          if (skeleton) {
-            loadSkeleton(skeleton);
-          }
         }
       }
     );
@@ -125,10 +125,30 @@
       console.log(step.name, step.start, step.end);
     });
   }
+
+  function copySkeleton() {
+    poseSkeleton = tracker.skeletonAt(selectedTimestamp);
+    if (poseSkeleton) {
+      loadSkeleton(poseSkeleton);
+    }
+  }
+
+  function copyPose() {
+    // TOOD
+  }
 </script>
 
 <h1>Dev</h1>
 
+<p>
+  <input
+    bind:this={upload}
+    type="file"
+    accept="video/*"
+    on:change={loadVideo}
+  />
+</p>
+
 <!-- svelte-ignore a11y-media-has-caption -->
 <div class="side-by-side">
   <video
@@ -151,16 +171,12 @@
   </div>
 </div>
 
+<button class="light full-width short" on:click={copySkeleton}> ↓ </button>
+
 <PoseInputForm bind:loadSkeleton></PoseInputForm>
 
-<p>
-  <input
-    bind:this={upload}
-    type="file"
-    accept="video/*"
-    on:change={loadVideo}
-  />
-</p>
+<button class="light full-width short" on:click={copyPose}> ↓ </button>
+
 <button on:click={downloadFrame}> Download Keypoints of Frame </button>
 <button on:click={downloadKeypoints}> Download Keypoints of Video </button>
 <h2>Dance Evaluation</h2>