Merge remote-tracking branch 'origin/master' into delete-project-data

Author: haraldschilly

src/compute/compute/package.json

@@ -40,7 +40,7 @@
     "@types/ws": "^8.5.9",
     "awaiting": "^3.0.0",
     "cookie": "^1.0.0",
-    "debug": "^4.3.2",
+    "debug": "^4.4.0",
     "websocketfs": "^0.17.4",
     "ws": "^8.18.0"
   },
@@ -52,6 +52,6 @@
   "devDependencies": {
     "@types/cookie": "^0.6.0",
     "@types/node": "^18.16.14",
-    "typescript": "^5.6.3"
+    "typescript": "^5.7.3"
   }
 }

src/compute/pnpm-lock.yaml

@@ -36,7 +36,7 @@
     "@types/watchpack": "^2.4.4",
     "awaiting": "^3.0.0",
     "chokidar": "^3.6.0",
-    "debug": "^4.3.2",
+    "debug": "^4.4.0",
     "fs-extra": "^11.2.0",
     "lodash": "^4.17.21",
     "lru-cache": "^7.18.3",

src/packages/backend/package.json

@@ -1,19 +1,32 @@
 import { sha1, uuidsha1 } from "./sha1";
 
+const cocalc = "CoCalc";
+const hash = "c898c97dca68742a5a6331f9fa0ca02483cbfd25";
+const uuid = "c898c97d-ca68-4742-a5a6-331f9fa0ca02";
+
 describe("compute some sha1 hashes", () => {
-  it("computes sha1 hash of old SageMathCloud", () => {
+  // This is mainly for long term backwards compatibility
+  it("SageMathCloud/string", () => {
     expect(sha1("SageMathCloud")).toBe(
-      "31acd8ca91346abcf6a49d2b1d88333f439d57a6"
+      "31acd8ca91346abcf6a49d2b1d88333f439d57a6",
     );
   });
 
-  it("computes sha1 hash of new CoCalc", () => {
-    expect(sha1("CoCalc")).toBe("c898c97dca68742a5a6331f9fa0ca02483cbfd25");
+  it("CoCalc/string", () => {
+    expect(sha1(cocalc)).toBe(hash);
+  });
+
+  it("CoCalc/Buffer", () => {
+    expect(sha1(Buffer.from(cocalc))).toBe(hash);
   });
 });
 
-describe("compute some uuids", () => {
-  it("computes uuid associated to 'CoCalc'", () => {
-    expect(uuidsha1("CoCalc")).toBe("c898c97d-ca68-4742-a5a6-331f9fa0ca02");
+describe("UUIDs", () => {
+  it("CoCalc/string", () => {
+    expect(uuidsha1(cocalc)).toBe(uuid);
+  });
+
+  it("CoCalc/Buffer", () => {
+    expect(uuidsha1(Buffer.from(cocalc))).toBe(uuid);
   });
 });

src/packages/backend/sha1.test.ts

@@ -6,14 +6,20 @@ import { createHash } from "crypto";
 
 // compute sha1 hash of data in hex
 export function sha1(data: Buffer | string): string {
+  const sha1sum = createHash("sha1");
+
   if (typeof data === "string") {
-    // CRITICAL: Code below assumes data is a Buffer; it will seem to work on a string, but give
-    // the wrong result where wrong means that it doesn't agree with the frontend version defined
-    // in misc.
-    data = Buffer.from(data);
+    sha1sum.update(data, "utf8");
+  } else {
+    // Convert Buffer to Uint8Array
+    const uint8Array = new Uint8Array(
+      data.buffer,
+      data.byteOffset,
+      data.byteLength,
+    );
+    sha1sum.update(uint8Array);
   }
-  const sha1sum = createHash("sha1");
-  sha1sum.update(data);
+
   return sha1sum.digest("hex");
 }
 

src/packages/backend/sha1.ts

@@ -9,6 +9,7 @@ Enable two new functions write_mesg and recv_mesg on a TCP socket.
 
 */
 
+import { Buffer } from "node:buffer";
 import { Socket } from "node:net";
 
 import getLogger from "@cocalc/backend/logger";
@@ -43,54 +44,56 @@ export interface CoCalcSocket extends Socket {
   write_mesg: (
     type: Type,
     mesg: Message,
-    cb?: (err?: string | Error) => void
+    cb?: (err?: string | Error) => void,
   ) => void;
   recv_mesg: (opts: RecvMesgOpts) => void;
 }
 
 export default function enable(socket: CoCalcSocket, desc: string = "") {
   socket.setMaxListeners(500); // we use a lot of listeners for listening for messages
 
-  let buf: Buffer | null = null;
+  let buf: Uint8Array | null = null;
   let bufTargetLength = -1;
 
-  const listenForMesg = (data: Buffer) => {
-    buf = buf == null ? data : Buffer.concat([buf, data]);
+  const listenForMesg = (data: Uint8Array) => {
+    buf = buf == null ? data : new Uint8Array([...buf, ...data]);
     while (true) {
       if (bufTargetLength === -1) {
         // starting to read a new message
         if (buf.length >= 4) {
-          bufTargetLength = buf.readUInt32BE(0) + 4;
+          const dv = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+          bufTargetLength = dv.getUint32(0) + 4;
         } else {
           return; // have to wait for more data to find out message length
         }
       }
       if (bufTargetLength <= buf.length) {
         // read a new message from our buffer
-        const type = buf.slice(4, 5).toString();
+        const type = String.fromCharCode(buf[4]);
         const mesg = buf.slice(5, bufTargetLength);
+
+        const textDecoder = new TextDecoder();
         switch (type) {
           case "j": // JSON
-            const s = mesg.toString();
-            let obj;
+            const s = textDecoder.decode(mesg);
             try {
               // Do not use "obj = JSON.parse(s)"
-              obj = from_json_socket(s); // this properly parses Date objects
+              const obj = from_json_socket(s); // this properly parses Date objects
+              socket.emit("mesg", "json", obj);
             } catch (err) {
               winston.debug(
                 `WARNING: failed to parse JSON message='${trunc(
                   s,
-                  512
-                )}' on socket ${desc} - ${err}`
+                  512,
+                )}' on socket ${desc} - ${err}`,
               );
               // skip it.
               return;
             }
-            socket.emit("mesg", "json", obj);
             break;
           case "b": // BLOB (tagged by a uuid)
             socket.emit("mesg", "blob", {
-              uuid: mesg.slice(0, 36).toString(),
+              uuid: textDecoder.decode(mesg.slice(0, 36)),
               blob: mesg.slice(36),
             });
             break;
@@ -117,36 +120,34 @@ export default function enable(socket: CoCalcSocket, desc: string = "") {
   socket.write_mesg = (
     type: Type,
     data: Message,
-    cb?: (err?: string | Error) => void
+    cb?: (err?: string | Error) => void,
   ): void => {
     if (data == null) {
       // uncomment this to get a traceback to see what might be causing this...
       //throw Error(`write_mesg(type='${type}': data must be defined`);
       cb?.(`write_mesg(type='${type}': data must be defined`);
       return;
     }
-    const send = function (s: string | Buffer): void {
-      const buf = Buffer.alloc(4);
+    const send = function (s: string | ArrayBuffer): void {
+      const length: Uint8Array = new Uint8Array(4);
       // This line was 4 hours of work.  It is absolutely
       // *critical* to change the (possibly a string) s into a
       // buffer before computing its length and sending it!!
       // Otherwise unicode characters will cause trouble.
-      if (typeof s === "string") {
-        s = Buffer.from(s);
-      }
-      buf.writeInt32BE(s.length, 0);
-      if (!socket.writable) {
-        cb?.("socket not writable");
-        return;
-      } else {
-        socket.write(buf);
-      }
+      const data: Uint8Array = new Uint8Array(
+        typeof s === "string" ? Buffer.from(s) : s,
+      );
+
+      const lengthView = new DataView(length.buffer);
+      // this was buf.writeInt32BE, i.e. big endian
+      lengthView.setInt32(0, data.byteLength, false); // false for big-endian
 
       if (!socket.writable) {
         cb?.("socket not writable");
         return;
       } else {
-        socket.write(s, cb);
+        socket.write(length);
+        socket.write(data, cb);
       }
     };
 
@@ -164,11 +165,13 @@ export default function enable(socket: CoCalcSocket, desc: string = "") {
           return;
         }
         send(
-          Buffer.concat([
-            Buffer.from("b"),
-            Buffer.from(data.uuid),
-            Buffer.from(data.blob),
-          ])
+          new Uint8Array([
+            ...Buffer.from("b"),
+            ...Buffer.from(data.uuid),
+            ...(Buffer.isBuffer(data.blob)
+              ? data.blob
+              : Buffer.from(data.blob)),
+          ]).buffer,
         );
         return;
       default: