initial dbc integration - backend and frontend!

Author: bvngee

backend/src/http_server.py

@@ -6,7 +6,7 @@
 data_dir = os.environ.get("DATA_DIR", "")
 parquet_files = set()
 if not os.path.isdir(data_dir):
-    raise ValueError("Invalid recording dir: ", data_dir)
+    raise ValueError("Invalid data dir: ", data_dir)
 
 
 def check_subdir(d):
@@ -33,12 +33,44 @@ async def available_recordings(request):
 routes.static("/api/get-recording", data_dir, show_index=True)
 
 
-@routes.get("/api/get-dbc/{car}")
+# The fs-data repo is structured such that differen't Parquet recordings are
+# "associated" with different DBC files, where the "closest" .dbc files are
+# correct and if the current directory doesn't have any we keep checking its
+# parents. So basically just walk up the directory tree until we find a .dbc
+@routes.get("/api/get-dbc-for-recording/{recording:.*}")
 async def get_dbc(request: web.Request):
-    car = request.match_info.get("car")
-    db = cantools.db.database.Database()
-    
-    return web.json_response(response_obj)
+    recording = request.match_info.get("recording")
+    if not recording:
+        return web.HTTPBadRequest(reason="Invalid recording parameter!")
+
+    dbc_path_dir = os.path.dirname(os.path.join(data_dir, recording))
+    if not os.path.isdir(dbc_path_dir):
+        return web.HTTPNotFound(reason="Recording path invalid!")
+
+    db = cantools.db.Database()
+
+    dir_has_dbc = False
+    while not dir_has_dbc:
+        dbc_path_dir_items = os.listdir(dbc_path_dir)
+        dbcs = [f for f in dbc_path_dir_items if os.path.splitext(f)[1] == ".dbc"]
+        if len(dbcs) > 0:
+            dir_has_dbc = True
+            for dbc in dbcs:
+                db.add_dbc_file(os.path.join(dbc_path_dir, dbc))
+                print(os.path.join(dbc_path_dir, dbc))
+        else:
+            dbc_path_dir = os.path.dirname(dbc_path_dir)
+
+    if not dir_has_dbc:
+        return web.HTTPNotFound(reason="DBC file not found!")
+
+    return web.json_response(
+        [
+            (s.name, s.unit, (s.minimum, s.maximum))
+            for m in db.messages
+            for s in m.signals # m.signal_tree ??
+        ]
+    )
 
 
 app = web.Application()

compose.yaml

@@ -32,10 +32,10 @@ services:
       - "9000"
     environment:
       # Directory where http server will look for parquet files (recursively)
-      RECORDING_DIRS: /recording_dirs
+      DATA_DIR: /data_dir
     volumes:
       # ./fs-data is a git submodule, optional
-      - ./fs-data/FS-2/Parquet:/recording_dirs/FS-2
+      - ./fs-data/FS-2/Parquet:/data_dir/FS-2
 
 volumes:
   caddy_data:

frontend/app/components/Navbar.tsx

@@ -16,7 +16,7 @@ import Link from "next/link";
 import { useEffect, useRef, useState } from "react";
 
 const subsystems = ["accumulator", "suspension", "imu-data", "faults", "3d-tests"];
-import { availableRecordings } from "../data-processing/http";
+import { availableRecordings, getDBCForRecording } from "../data-processing/http";
 import { useDataMethods } from "../data-processing/DataMethodsProvider";
 import AutocompleteSearchbar from "./Autocomplete";
 
@@ -44,7 +44,7 @@ function createFileTree(paths: string[] | undefined) {
             const isLeaf = i === root.length - 1;
 
             let existingNode = currentLevel.find(
-                (node: pathType) => node.value === currentPath
+                (node: pathType) => node.value === currentPath,
             );
 
             if (!existingNode) {
@@ -165,9 +165,17 @@ export default function Navbar() {
                                         if (tree.hoveredNode?.includes("/")) {
                                             setFileName(tree.hoveredNode);
                                             setLoading(true);
-                                            await switchToRecording(tree.hoveredNode);
+                                            switchToRecording(tree.hoveredNode);
                                             setLoading(false);
                                             close();
+
+                                            console.log(
+                                                `Associated DBC for ${tree.hoveredNode}:`,
+                                                await getDBCForRecording(
+                                                    tree.hoveredNode,
+                                                    isProduction,
+                                                ),
+                                            );
                                         }
                                     }}
                                     className={

frontend/app/data-processing/DataMethodsProvider.tsx

@@ -289,6 +289,8 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
         reset();
 
         const table = await getRecording(filename);
+        if (!table) return;
+
         let arraysTyped = {} as DataArraysTyped;
         // TODO: Make this actually use the schema of the incoming
         // websocket stream!!! This assumes all columns are present
@@ -309,6 +311,11 @@ export function DataMethodsProvider({ children }: PropsWithChildren) {
             n == 0.0 ? NaN : n
         );
 
+        // TODO: somehow some BigInt64Array's are getting passed through to lcjs
+        // even though we have the claude filtering in http.ts?? Try out
+        // "2025-01-21-IMU-Calibration.parquet" which fails every time. Also,
+        // uncommenting the above two lines changes the error?? TODO: investigate.
+
         numRowsRef.current = table.numRows;
         subscriptionsNumRows.current.forEach((s) => s(numRowsRef.current));
 

frontend/app/data-processing/http.ts

@@ -6,6 +6,7 @@ import * as arrowJSFFI from "arrow-js-ffi";
 import { DataArrays, DataRow, nullDataArrays, schema } from "./datatypes";
 import { Dispatch, RefObject, SetStateAction } from "react";
 import { RESPONSE_LIMIT_DEFAULT } from "next/dist/server/api-utils";
+import { Table } from "apache-arrow";
 
 // TODO(jack): this setup is kinda bad cuz every time the function is called it
 // tried all URLs, including bad ones, which cause Network Errors to be thrown
@@ -66,26 +67,36 @@ async function tryFetch(path: string, production: boolean) {
     }
 }
 
-export async function availableRecordings(production: boolean): Promise<string[] | null> {
-    // console.log(process.env.NODE_ENV);
-    return tryFetch("/api/available-recordings", production).then((response) =>
-        response != null ? response.json() : null
+export async function getDBCForRecording(
+    filepath: string,
+    production: boolean,
+): Promise<string[] | null> {
+    return tryFetch(`/api/get-dbc-for-recording/${filepath}`, production).then((resp) =>
+        resp?.json(),
     );
 }
 
-export async function getRecording(filepath: string) {
+export async function availableRecordings(production: boolean): Promise<string[] | null> {
+    return tryFetch("/api/available-recordings", production).then((resp) => resp?.json());
+}
+
+export async function getRecording(filepath: string): Promise<Table<DataRow> | undefined> {
     // Initializes WebAssembly memory for parquet-wasm, and gets a reference to it
     await wasmInit();
     const WASM_MEMORY = wasmMemory();
 
-    const parquet = await tryFetch(`/api/get-recording/${filepath}`, true).then((resp) =>
-        resp ? resp.arrayBuffer() : null
-    );
-    const arrowTableWasm = readParquet(new Uint8Array(parquet!)).intoFFI();
+    const resp = await tryFetch(`/api/get-recording/${filepath}`, false)
+    const arrayBuffer = await resp?.arrayBuffer();
+    if (!arrayBuffer) {
+        console.error("Response is not a valid ArrayBuffer!");
+        return;
+    }
+
+    const arrowTableWasm = readParquet(new Uint8Array(arrayBuffer)).intoFFI();
     const arrowTable = arrowJSFFI.parseTable<DataRow>(
         WASM_MEMORY.buffer,
         arrowTableWasm.arrayAddrs(),
-        arrowTableWasm.schemaAddr()
+        arrowTableWasm.schemaAddr(),
     );
 
     // Free arrow table in wasm memory
@@ -105,7 +116,7 @@ export async function initRecordingSource(
     data: RefObject<DataArrays>,
     dataTrimmed: RefObject<DataArrays>,
     setNumRows: Dispatch<SetStateAction<number>>,
-    viewLength: number
+    viewLength: number,
 ) {
     const arrowTable = (await getRecording(filepath))!;