various fixes; can load any recording now!!

bvngee · bvngee · commit 7631772cd827 · 2025-05-15T16:46:46.000-07:00
diff --git a/backend/src/http_server.py b/backend/src/http_server.py
@@ -17,10 +17,8 @@ def check_subdir(d):
         elif f.endswith(".pq") or f.endswith(".parquet"):
             parquet_files.add(os.path.relpath(path, data_dir))
 
-
 check_subdir(data_dir)
 
-
 routes = web.RouteTableDef()
 
 
diff --git a/frontend/app/components/Navbar.tsx b/frontend/app/components/Navbar.tsx
@@ -22,6 +22,7 @@ import { availableRecordings, getDBCForRecording } from "../data-processing/http
 import { useDataMethods } from "../data-processing/DataMethodsProvider";
 import AutocompleteSearchbar from "./Autocomplete";
 import { File } from "@phosphor-icons/react/dist/ssr";
+import { log } from "console";
 
 //takes in the list of files as an array and outputs it in mantine tree format
 function createFileTree(paths: string[] | undefined) {
@@ -62,7 +63,7 @@ function createFileTree(paths: string[] | undefined) {
             currentLevel = existingNode.children;
         }
     }
-    return tree.children[0];
+    return tree.children;
 }
 
 export default function Navbar() {
@@ -74,7 +75,7 @@ export default function Navbar() {
     const [isProduction, setIsProduction] = useState<boolean>(true);
     const displayedName = fileName.split("/")[fileName.split("/").length - 1];
 
-    const { switchToRecording, switchToLiveData, subscribeNumRows } = useDataMethods();
+    const { switchToRecording, switchToLiveData, subscribeNumRows, reset } = useDataMethods();
     const myRowsPRef = useRef<HTMLParagraphElement | null>(null);
     useEffect(() => {
         return subscribeNumRows((numRows: number) => {
@@ -125,22 +126,20 @@ export default function Navbar() {
     function DataSourcePicker() {
         function onPickerChanged(value: string) {
             if (value === "recording") {
-                //TODO Make an actual function to stop the live view
-                switchToRecording("");
+                reset();
                 setLive(false);
             } else {
                 setLive(true);
                 switchToLiveData();
             }
-            // setLive(value === "live");
         }
 
         function FileTree() {
             if (recordings == null) {
                 return <span>Loading...</span>;
             }
 
-            const data = [createFileTree(recordings)];
+            const data = createFileTree(recordings);
 
             return (
                 <>
@@ -164,10 +163,10 @@ export default function Navbar() {
 
                                 <span
                                     onClick={async () => {
-                                        if (tree.hoveredNode?.includes("/")) {
+                                        if (!hasChildren && tree.hoveredNode) {
                                             setFileName(tree.hoveredNode);
                                             setLoading(true);
-                                            switchToRecording(tree.hoveredNode);
+                                            switchToRecording(tree.hoveredNode, isProduction);
                                             setLoading(false);
                                             close();
 
@@ -233,7 +232,7 @@ export default function Navbar() {
                                 const file = files[0];
                                 setFileName(file.name);
                                 setLoading(true);
-                                switchToRecording(file);
+                                switchToRecording(file, isProduction);
                                 setLoading(false);
                                 close();
                             }}
diff --git a/frontend/app/components/TimelineBar.tsx b/frontend/app/components/TimelineBar.tsx
@@ -1,5 +1,5 @@
-import { Box, Button, RangeSlider, RangeSliderValue } from "@mantine/core";
-import { ArrowFatLinesRight, CaretDoubleRight, Pause, Play } from "@phosphor-icons/react";
+import { Box, RangeSlider, RangeSliderValue } from "@mantine/core";
+import { CaretDoubleRight, Pause, Play } from "@phosphor-icons/react";
 import { useEffect, useMemo, useRef, useState } from "react";
 import { useDataMethods } from "../data-processing/DataMethodsProvider";
 import DataSourceType from "@/models/DataSourceType";
@@ -8,7 +8,6 @@ import { timeColumnName } from "../data-processing/datatypes";
 
 export default function TimelineBar() {
     const [paused, setPaused] = useState(true);
-    const { switchToRecording } = useDataMethods();
 
     return (
         <>
@@ -25,11 +24,6 @@ export default function TimelineBar() {
                     <MainSlider />
                 </div>
                 <SyncButton />
-                <Button
-                    onClick={() =>
-                        switchToRecording("fs-data/FS-2/2025-03-06-BrakingTests1.parquet")
-                    }
-                />
             </div>
         </>
     );
diff --git a/frontend/app/components/visualizations/GPS/GPSInternal.tsx b/frontend/app/components/visualizations/GPS/GPSInternal.tsx
@@ -183,11 +183,14 @@ export default function GPSInternal({
             });
         });
         const unsub2 = subscribeViewInterval(([left, right]) => {
-            console.log("1", performance.now());
+            // console.log("1", performance.now());
+
             // We reset and regenerate the fillStyle lookupValues each time
             // viewInterval changes. This seems to be a fairly fast operation
             visibleSeries.fill({ lookupValue: 0 });
-            console.log("2", performance.now());
+
+            // console.log("2", performance.now());
+
             // We need +1 as left and right from dataProvider are both inclusive
             const length = right - left + 1;
             // Generates a list of values of length `length` that are evenly
@@ -196,14 +199,17 @@ export default function GPSInternal({
                 { length },
                 (_, i) => 0.3 + (i / (length - 1)) * (1 - 0.3),
             );
-            console.log("3", performance.now());
+
+            // console.log("3", performance.now());
+
             // Insert the above gradient array into the lookupValues dataset
             // channel starting at the correct index
             visibleSeries.alterSamples(left, {
                 // todo: this doesn't work with data cleaning
                 lookupValues: gradient,
             });
-            console.log("4", performance.now());
+
+            // console.log("4", performance.now());
         });
         const unsub3 = subscribeCursorRow((cursorRow) => {
             // Always update carSeries to point to where the cursor is hovering,
@@ -230,6 +236,7 @@ export default function GPSInternal({
         const unsub4 = subscribeReset(() => {
             bgSeries.clear();
             visibleSeries.clear();
+            carSeries.clear();
         });
 
         return () => {
diff --git a/frontend/app/components/visualizations/LapCounter.tsx b/frontend/app/components/visualizations/LapCounter.tsx
@@ -3,7 +3,6 @@
 import { useContext, useEffect, useId, useRef } from "react";
 import { LightningChartsContext } from "./lightning-charts/GlobalContext";
 import globalTheme from "./lightning-charts/GlobalTheme";
-import { VisualizationProps } from "../FlexLayoutComponent";
 import { useDataMethods } from "@/app/data-processing/DataMethodsProvider";
 
 // data is represented in row, column (y, x)
@@ -34,7 +33,6 @@ export function LapCounter() {
 
             if (!lapArray || !lapTimeArray) return;
 
-            console.log("hi");
             lapTimes = [];
 
             let previous = lapArray[0];
@@ -72,8 +70,6 @@ export function LapCounter() {
                 dataGrid.setRowContent(i + 1, [i + 1, lapTimes[i]]);
             }
 
-            console.log(lapTime);
-
             dataGrid.setRowContent(dataGrid.getRowMax(), [dataGrid.getRowMax(), lapTime]);
         });
 
diff --git a/frontend/app/data-processing/DataMethodsProvider.tsx b/frontend/app/data-processing/DataMethodsProvider.tsx
@@ -67,7 +67,7 @@ type DataControllers = {
     reset: () => void;
 
     switchToLiveData: (setIsConnected?: Dispatch<SetStateAction<boolean>>) => void;
-    switchToRecording: (filename: string | File) => void;
+    switchToRecording: (filename: string | File, isProduction: boolean) => void;
 };
 type DataRefs = {
     dataArraysRef: RefObject<DataArraysTyped>;
@@ -206,6 +206,7 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
         closeWebSocketConnection();
 
         subscriptionsReset.current.forEach((s) => s());
+        subscriptionsDataSource.current.forEach((s) => s(DataSourceType.NONE));
     };
 
     const switchToLiveData = (setIsConnected?: Dispatch<SetStateAction<boolean>>) => {
@@ -288,29 +289,40 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
         subscriptionsDataSource.current.forEach((s) => s(DataSourceType.LIVE));
         setIsTimelineSynced(true, "dataProvider");
     };
-    const switchToRecording = async (file: string | File) => {
+    const switchToRecording = async (file: string | File, isProduction: boolean) => {
         reset();
-        const table = await getRecording(file);
-        if (!table) return;
+        const table = await getRecording(file, isProduction);
+        if (!table) {
+            console.log("WARN: couldn't load recording!");
+            return;
+        };
+
         let arraysTyped = {} as DataArraysTyped;
         // TODO: Make this actually use the schema of the incoming
-        // websocket stream!!! This assumes all columns are present
+        // parquet recording!!! This assumes all columns are present
         // (non-present should be null)
         for (const key of columnNames) {
             const vector = table.getChild(key);
             if (vector) {
-                const arr = vector.toArray();
+                let arr = vector.toArray();
+
+                // Hack since lcjs/chartjs don't support 64bit ints
+                if (arr instanceof BigInt64Array || arr instanceof BigUint64Array) {
+                    arr = convertToInt32Array(arr);
+                }
+
+                // Store extracted TypedArray in final dictionary
                 arraysTyped[key] = arr;
             }
         }
         // some basic manual data processing. TODO: make somthing extremely
         // generic / extensible for arbitrary data processing
-        arraysTyped.VDM_GPS_Latitude = arraysTyped.VDM_GPS_Latitude!.map((n) =>
+        arraysTyped.VDM_GPS_Latitude = arraysTyped.VDM_GPS_Latitude?.map((n) =>
             n == 0.0 ? NaN : n,
-        );
-        arraysTyped.VDM_GPS_Longitude = arraysTyped.VDM_GPS_Longitude!.map((n) =>
+        ) ?? null;
+        arraysTyped.VDM_GPS_Longitude = arraysTyped.VDM_GPS_Longitude?.map((n) =>
             n == 0.0 ? NaN : n,
-        );
+        ) ?? null;
 
         numRowsRef.current = table.numRows;
         subscriptionsNumRows.current.forEach((s) => s(numRowsRef.current));
@@ -362,11 +374,6 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
         };
     }, []);
 
-    // const methods: DataMethods = useMemo(
-    //     () => (),
-    //     [],
-    // );
-
     return (
         <DataMethodsContext.Provider value={methods}>{children}</DataMethodsContext.Provider>
     );
@@ -379,3 +386,17 @@ export function useDataMethods() {
     }
     return context;
 }
+
+
+// Used for recordings that have some i64 columns (Thanks claude)
+function convertToInt32Array(bigIntArray: BigInt64Array | BigUint64Array) {
+    const int32Array = new Int32Array(bigIntArray.length);
+    const INT32_MIN = -2147483648;
+    const INT32_MAX = 2147483647;
+
+    for (let i = 0; i < bigIntArray.length; i++) {
+        const num = Number(bigIntArray[i]);
+        int32Array[i] = Math.max(INT32_MIN, Math.min(INT32_MAX, num));
+    }
+    return int32Array;
+}
diff --git a/frontend/app/data-processing/http.ts b/frontend/app/data-processing/http.ts
@@ -3,8 +3,9 @@ import wasmInit, { readParquet, wasmMemory } from "parquet-wasm/esm";
 // Enables zero-copy transer of Arrow data from wasm memory to JS
 import * as arrowJSFFI from "arrow-js-ffi";
 
-import { DataRow, schema } from "./datatypes";
-
+// Tries to call one of the HTTP endpoints of the server, either local or remote
+// depending on isProduction. Returns the Response, with some error handling.
+// Internal; should be used by the public-facing functions below
 async function tryFetch(path: string, isProduction: boolean) {
     const url = isProduction ? "https://telemetry.formulaslug.com" : "http://localhost:9000";
     try {
@@ -19,6 +20,8 @@ async function tryFetch(path: string, isProduction: boolean) {
     }
 }
 
+// Fetch the DBC data associated with a given Recording (filepath). Contains a
+// map of ColumnName to information like units, range, etc
 export async function getDBCForRecording(
     filepath: string,
     isProduction: boolean,
@@ -28,16 +31,19 @@ export async function getDBCForRecording(
     );
 }
 
-// ???
-export async function getRecording(file: string | File) {
+// Fetch the recording data of a filepath (by querying the server) or from a
+// directly uploaded file (drag-n-drop). Returned data is in Arrow Table format.
+export async function getRecording(file: string | File, isProduction: boolean) {
     // Initializes WebAssembly memory for parquet-wasm, and gets a reference to it
     await wasmInit();
     const WASM_MEMORY = wasmMemory();
 
     let arrayBuffer: ArrayBuffer | undefined;
 
+    // If `file` is a string, fetch it from the server. If it's a File, extract
+    // the arraybuffer data from that directly
     if (typeof file == "string") { // string (pathname)
-        const resp = await tryFetch(`/api/get-recording/${file}`, false)
+        const resp = await tryFetch(`/api/get-recording/${file}`, isProduction)
         arrayBuffer = await resp?.arrayBuffer();
     } else { // File object
         arrayBuffer = await file.arrayBuffer();
@@ -49,7 +55,7 @@ export async function getRecording(file: string | File) {
     }
 
     const arrowTableWasm = readParquet(new Uint8Array(arrayBuffer)).intoFFI();
-    const arrowTable = arrowJSFFI.parseTable<DataRow>(
+    const arrowTable = arrowJSFFI.parseTable(
         WASM_MEMORY.buffer,
         arrowTableWasm.arrayAddrs(),
         arrowTableWasm.schemaAddr(),
@@ -72,53 +78,3 @@ export async function availableRecordings(isProduction: boolean) {
     const resp = await tryFetch(`/api/available-recordings`, isProduction);
     return resp?.json();
 }
-
-// export async function initRecordingSource(
-//     filepath: string,
-//     data: RefObject<DataArrays>,
-//     dataTrimmed: RefObject<DataArrays>,
-//     setNumRows: Dispatch<SetStateAction<number>>,
-//     viewLength: number,
-// ) {
-//     const arrowTable = (await getRecording(filepath))!;
-//
-//     // Completely reset data, so keys unused by the recording are left as null
-//     data.current = nullDataArrays();
-//     dataTrimmed.current = nullDataArrays();
-//
-//     for (const [i, key] of arrowTable.schema.names.entries()) {
-//         // Get Arrow vector from table
-//         const vec = arrowTable.getChildAt(i);
-//         // Extract JS array from Arrow vector
-//         let dataArr = vec?.toArray();
-//         let dataArrTrimmed = vec
-//             ?.slice(Math.max(0, arrowTable.numRows - viewLength - 1))
-//             .toArray();
-//
-//         // Chartjs doesn't support bigints ootb. The easiest solution for now is
-//         // just to replace them with supported TypedArrays
-//         if (dataArr instanceof BigInt64Array || dataArr instanceof BigUint64Array) {
-//             dataArr = convertToInt32Array(dataArr);
-//             dataArrTrimmed = convertToInt32Array(dataArrTrimmed);
-//         }
-//
-//         // Set the data arrays directly
-//         data.current[key] = dataArr;
-//         dataTrimmed.current[key] = dataArrTrimmed;
-//     }
-//
-//     setNumRows(arrowTable.numRows);
-// }
-
-// Thanks claude
-function convertToInt32Array(bigIntArray: BigInt64Array | BigUint64Array) {
-    const int32Array = new Int32Array(bigIntArray.length);
-    const INT32_MIN = -2147483648;
-    const INT32_MAX = 2147483647;
-
-    for (let i = 0; i < bigIntArray.length; i++) {
-        const num = Number(bigIntArray[i]);
-        int32Array[i] = Math.max(INT32_MIN, Math.min(INT32_MAX, num));
-    }
-    return int32Array;
-}
diff --git a/frontend/app/page.tsx b/frontend/app/page.tsx
@@ -7,7 +7,9 @@ import TimelineBar from "./components/TimelineBar";
 import { useEffect } from "react";
 
 export default function Page() {
-    useEffect(() => window.addEventListener(`contextmenu`, (e) => e.preventDefault()), []);
+    // This is pretty stupid, but it prevents annoying right-click context menus
+    // on lcjs charts (by disabling right-click menus altogether)
+    useEffect(() => window.addEventListener("contextmenu", (e) => e.preventDefault()), []);
     return (
         <div className="w-[100vw] h-[100vh] flex flex-col">
             <Navbar />
