move editor to new "lab" page, hidden as experimental feature

Author: jakmeier

bouncy_frontend/src/lib/components/editor/VideoToStep.svelte

@@ -0,0 +1,141 @@
+<script>
+  import { Tracker } from '$lib/instructor/bouncy_instructor';
+  import { getContext, onMount } from 'svelte';
+  import { landmarksToKeypoints } from '$lib/pose';
+  import { registerTracker } from '$lib/stores/Beat';
+  import Svg from '$lib/components/avatar/Svg.svelte';
+  import SvgAvatar2 from '$lib/components/avatar/SvgAvatar2.svelte';
+  import { LEFT_RIGHT_COLORING_LIGHT } from '$lib/constants';
+  import PoseInputForm from '$lib/components/editor/PoseInputForm.svelte';
+  import PoseEditForm from '$lib/components/editor/PoseEditForm.svelte';
+  import { fileToUrl, waitForVideoMetaLoaded } from '$lib/promise_util';
+  import { PoseDetection } from '$lib/pose';
+
+  const poseCtx = getContext('pose');
+  let tracker = new Tracker();
+  registerTracker(tracker);
+
+  /** @type {PoseDetection} */
+  let dataListener;
+  /** @type {(skeleton: import("$lib/instructor/bouncy_instructor").SkeletonWrapper)=>void} */
+  let loadSkeleton;
+  /** @type {()=>import("$lib/instructor/bouncy_instructor").PoseWrapper} */
+  let poseFromForm;
+  /** @type {(skeleton: import("$lib/instructor/bouncy_instructor").PoseWrapper)=>void} */
+  let loadPose;
+
+  /** @type {import("$lib/instructor/bouncy_instructor").SkeletonV2 | undefined} */
+  let liveSkeleton;
+  /** @type {import("$lib/instructor/bouncy_instructor").SkeletonWrapper | undefined} */
+  let poseSkeleton;
+
+  /** @type {HTMLInputElement}  */
+  let upload;
+  /** @type {HTMLVideoElement}  */
+  let video;
+  let videoSrcWidth = 0;
+  let videoSrcHeight = 0;
+
+  /** @type {undefined | number} */
+  let recordingStart;
+  /** @type {undefined | number} */
+  let recordingEnd;
+  let prevTime = -1;
+  let selectedTimestamp = 0;
+
+  /** @param { Event } event */
+  async function loadVideo(event) {
+    if (event.target && event.target.files && event.target.files[0]) {
+      video.src = await fileToUrl(event.target.files[0]);
+      await waitForVideoMetaLoaded(video);
+      tracker.clear();
+      video.play();
+      loop();
+    }
+  }
+
+  function loop() {
+    if (dataListener && prevTime !== video.currentTime) {
+      prevTime = video.currentTime;
+      dataListener.trackFrame(video, undefined);
+    }
+    requestAnimationFrame(loop);
+  }
+
+  onMount(async () => {
+    dataListener = await poseCtx.newPoseDetection(
+      (
+        /** @type {{ landmarks: string | any[]; }} */ result,
+        /** @type {number} */ timestamp
+      ) => {
+        if (recordingStart === undefined) {
+          recordingStart = timestamp;
+        }
+        if (result.landmarks && result.landmarks.length >= 1) {
+          const kp = landmarksToKeypoints(result.landmarks[0]);
+          tracker.addKeypoints(kp, timestamp);
+          recordingEnd = Math.max(timestamp, recordingEnd || 0);
+          selectedTimestamp = timestamp;
+          liveSkeleton = tracker.renderedKeypointsAt(
+            timestamp,
+            videoSrcWidth,
+            videoSrcHeight
+          );
+        }
+      }
+    );
+  });
+
+  function copySkeleton() {
+    poseSkeleton = tracker.skeletonWrapperAt(selectedTimestamp);
+    if (poseSkeleton) {
+      loadSkeleton(poseSkeleton);
+    }
+  }
+
+  function copyPose() {
+    let pose = poseFromForm();
+    if (pose) {
+      loadPose(pose);
+    }
+  }
+</script>
+
+<p>
+  <input
+    bind:this={upload}
+    type="file"
+    accept="video/*"
+    on:change={loadVideo}
+  />
+</p>
+
+<!-- svelte-ignore a11y-media-has-caption -->
+<div class="side-by-side">
+  <video
+    bind:this={video}
+    bind:videoWidth={videoSrcWidth}
+    bind:videoHeight={videoSrcHeight}
+    playsinline
+    controls
+  ></video>
+  <div>
+    {#if liveSkeleton}
+      <Svg width={videoSrcWidth} height={videoSrcHeight} orderByZ showOverflow>
+        <SvgAvatar2
+          skeleton={liveSkeleton}
+          lineWidth={3}
+          style={LEFT_RIGHT_COLORING_LIGHT}
+        />
+      </Svg>
+    {/if}
+  </div>
+</div>
+
+<button class="light full-width short" on:click={copySkeleton}> ↓ </button>
+
+<PoseInputForm bind:loadSkeleton bind:readPose={poseFromForm}></PoseInputForm>
+
+<button class="light full-width short" on:click={copyPose}> ↓ </button>
+
+<PoseEditForm bind:loadPose></PoseEditForm>

bouncy_frontend/src/lib/i18n/de-CH.json

@@ -73,6 +73,8 @@
         }
     },
     "editor": {
+        "title": "Labor",
+        "nav": "Labor",
         "edit-dance-context-menu": "Bearbeiten",
         "dance-copy-postfix": "Kopie",
         "delete-dance-button": "Tanz löschen",

bouncy_frontend/src/lib/i18n/en-GB.json

@@ -73,6 +73,8 @@
         }
     },
     "editor": {
+        "title": "Laboratory",
+        "nav": "Laboratory",
         "edit-dance-context-menu": "Edit",
         "dance-copy-postfix": "copy",
         "delete-dance-button": "Delete dance",

bouncy_frontend/src/lib/pose.js

@@ -91,7 +91,7 @@ export class PoseDetection {
     /**
      * 
      * @param {import('@mediapipe/tasks-vision').ImageSource} videoElement
-     * @param {number} timestamp
+     * @param {number | undefined} timestamp
     */
     trackFrame(videoElement, timestamp) {
         if (timestamp === undefined || timestamp === null) {

bouncy_frontend/src/lib/stores/FeatureSelection.js

@@ -3,6 +3,7 @@
 import { derived, readable, writable } from "svelte/store";
 import { dev as envDev, browser } from '$app/environment';
 
+let experimental = writable(false);
 let privDev = envDev;
 export const dev = readable(privDev, (set) => {
     if (browser) {
@@ -23,7 +24,7 @@ function versionNumberToString(v) {
 }
 
 /** @type {import("svelte/motion").Readable<Features>} */
-export const features = derived([version, dev], ([$v, $dev]) => {
+export const features = derived([version, dev, experimental], ([$v, $dev, $experimental]) => {
     return {
         /* Fully enabled features for now but might be disabled again*/
         enableDanceCollection: $v >= 0.003,
@@ -32,6 +33,7 @@ export const features = derived([version, dev], ([$v, $dev]) => {
 
         /* Partially enabled features */
         enableStepRecording: (stepName) => STABLE_TRACKING_STEPS.includes(stepName),
+        enableEditorPage: $experimental,
 
         /* Features that are not ready to be released */
         enableAvatarRotation: $v >= 0.999,
@@ -43,6 +45,11 @@ export const features = derived([version, dev], ([$v, $dev]) => {
 }
 );
 
+/** @param {boolean} yes */
+export function showExperimentalFeatures(yes) {
+    experimental.set(yes);
+}
+
 /** Steps that should be possible to track, with passing tests. */
 export const STABLE_TRACKING_STEPS = [
     "Running Man",

bouncy_frontend/src/lib/types.js

@@ -6,6 +6,7 @@
  * @property {(step: string)=>boolean} enableStepRecording
  * @property {boolean} enableDanceCollection
  * @property {boolean} enableDanceCreator
+ * @property {boolean} enableEditorPage
  * @property {boolean} enableCourses
  * @property {boolean} enableDevView
  *

bouncy_frontend/src/routes/TabNavigation.svelte

@@ -31,6 +31,13 @@
       icon: 'book_5',
       route: `${base}/collection`,
     });
+    if (features.enableEditorPage) {
+      tabs.push({
+        label: $t('editor.nav'),
+        icon: 'experiment',
+        route: `${base}/editor`,
+      });
+    }
     tabs.push({
       label: $t('profile.nav'),
       icon: 'account_circle',

bouncy_frontend/src/routes/UserContext.svelte

@@ -7,6 +7,7 @@
   import { generateRandomUsername } from '$lib/username';
   import { setContext } from 'svelte';
   import { writable } from 'svelte/store';
+  import { showExperimentalFeatures } from '$lib/stores/FeatureSelection.js';
 
   function fromLocalStorage() {
     try {
@@ -49,6 +50,9 @@
   );
   if (browser) {
     user.subscribe((value) => (localStorage.user = JSON.stringify(value)));
+    user.subscribe((value) => {
+      showExperimentalFeatures(value.experimentalFeatures);
+    });
   }
 
   /**

bouncy_frontend/src/routes/dev/+page.svelte

@@ -9,8 +9,6 @@
   import Svg from '$lib/components/avatar/Svg.svelte';
   import SvgAvatar2 from '$lib/components/avatar/SvgAvatar2.svelte';
   import { LEFT_RIGHT_COLORING_LIGHT } from '$lib/constants';
-  import PoseInputForm from '$lib/components/editor/PoseInputForm.svelte';
-  import PoseEditForm from '$lib/components/editor/PoseEditForm.svelte';
 
   /** @type {HTMLInputElement}  */
   let upload;
@@ -22,8 +20,6 @@
   let videoSrcHeight = 0;
   /** @type {import("$lib/instructor/bouncy_instructor").SkeletonV2 | undefined} */
   let liveSkeleton;
-  /** @type {import("$lib/instructor/bouncy_instructor").SkeletonWrapper | undefined} */
-  let poseSkeleton;
 
   let dataListener;
   /** @type {(skeleton: import("$lib/instructor/bouncy_instructor").SkeletonWrapper)=>void} */
@@ -130,20 +126,6 @@
       console.log(step.name, step.start, step.end);
     });
   }
-
-  function copySkeleton() {
-    poseSkeleton = tracker.skeletonWrapperAt(selectedTimestamp);
-    if (poseSkeleton) {
-      loadSkeleton(poseSkeleton);
-    }
-  }
-
-  function copyPose() {
-    let pose = poseFromForm();
-    if (pose) {
-      loadPose(pose);
-    }
-  }
 </script>
 
 <h1>Dev</h1>
@@ -179,14 +161,6 @@
   </div>
 </div>
 
-<button class="light full-width short" on:click={copySkeleton}> ↓ </button>
-
-<PoseInputForm bind:loadSkeleton bind:readPose={poseFromForm}></PoseInputForm>
-
-<button class="light full-width short" on:click={copyPose}> ↓ </button>
-
-<PoseEditForm bind:loadPose></PoseEditForm>
-
 <button on:click={downloadFrame}> Download Keypoints of Frame </button>
 <button on:click={downloadKeypoints}> Download Keypoints of Video </button>
 <h2>Dance Evaluation</h2>

bouncy_frontend/src/routes/editor/+page.svelte

@@ -0,0 +1,5 @@
+<script>
+  import VideoToStep from '$lib/components/editor/VideoToStep.svelte';
+</script>
+
+<VideoToStep></VideoToStep>