feat: proper range request handling (#635)

* fix: update to @types/node@18

this fixes the type error at  line 14 of lib/datasources/Local.ts

* feat: proper range request handling

* fix: docker casing warnings

* fix: infinity in header and cleanup

* fix: types for s3 and supabase size return value

* chore: remove unneeded newline

* chore: remove leftover dev comment

* fix: don't use 206 & content-range when client did not request it

Author: ari-party

Dockerfile

@@ -1,8 +1,8 @@
 # Use the Prisma binaries image as the first stage
-FROM ghcr.io/diced/prisma-binaries:5.1.x as prisma
+FROM ghcr.io/diced/prisma-binaries:5.1.x AS prisma
 
 # Use Alpine Linux as the second stage
-FROM node:18-alpine3.16 as base
+FROM node:18-alpine3.16 AS base
 
 # Set the working directory
 WORKDIR /zipline
@@ -27,7 +27,7 @@ ENV PRISMA_QUERY_ENGINE_BINARY=/prisma-engines/query-engine \
 # Install the dependencies
 RUN yarn install --immutable
 
-FROM base as builder
+FROM base AS builder
 
 COPY src ./src
 COPY next.config.js ./next.config.js

package.json

@@ -79,7 +79,7 @@
     "@types/katex": "^0.16.6",
     "@types/minio": "^7.1.1",
     "@types/multer": "^1.4.10",
-    "@types/node": "^18.18.10",
+    "@types/node": "18",
     "@types/qrcode": "^1.5.5",
     "@types/react": "^18.2.37",
     "@types/sharp": "^0.32.0",

src/lib/datasources/Datasource.ts

@@ -6,7 +6,7 @@ export abstract class Datasource {
   public abstract save(file: string, data: Buffer, options?: { type: string }): Promise<void>;
   public abstract delete(file: string): Promise<void>;
   public abstract clear(): Promise<void>;
-  public abstract size(file: string): Promise<number>;
-  public abstract get(file: string): Readable | Promise<Readable>;
+  public abstract size(file: string): Promise<number | null>;
+  public abstract get(file: string, start?: number, end?: number): Readable | Promise<Readable>;
   public abstract fullSize(): Promise<number>;
 }

src/lib/datasources/Local.ts

@@ -26,20 +26,20 @@ export class Local extends Datasource {
     }
   }
 
-  public get(file: string): ReadStream {
+  public get(file: string, start: number = 0, end: number = Infinity): ReadStream {
     const full = join(this.path, file);
     if (!existsSync(full)) return null;
 
     try {
-      return createReadStream(full);
+      return createReadStream(full, { start, end });
     } catch (e) {
       return null;
     }
   }
 
-  public async size(file: string): Promise<number> {
+  public async size(file: string): Promise<number | null> {
     const full = join(this.path, file);
-    if (!existsSync(full)) return 0;
+    if (!existsSync(full)) return null;
     const stats = await stat(full);
 
     return stats.size;

src/lib/datasources/S3.ts

@@ -1,7 +1,7 @@
 import { Datasource } from '.';
 import { Readable } from 'stream';
 import { ConfigS3Datasource } from 'lib/config/Config';
-import { Client } from 'minio';
+import { BucketItemStat, Client } from 'minio';
 
 export class S3 extends Datasource {
   public name = 'S3';
@@ -45,19 +45,34 @@ export class S3 extends Datasource {
     });
   }
 
-  public get(file: string): Promise<Readable> {
+  public get(file: string, start: number = 0, end: number = Infinity): Promise<Readable> {
     return new Promise((res) => {
-      this.s3.getObject(this.config.bucket, file, (err, stream) => {
-        if (err) res(null);
-        else res(stream);
-      });
+      this.s3.getPartialObject(
+        this.config.bucket,
+        file,
+        start,
+        // undefined means to read the rest of the file from the start (offset)
+        end === Infinity ? undefined : end,
+        (err, stream) => {
+          if (err) res(null);
+          else res(stream);
+        },
+      );
     });
   }
 
-  public async size(file: string): Promise<number> {
-    const stat = await this.s3.statObject(this.config.bucket, file);
-
-    return stat.size;
+  public size(file: string): Promise<number | null> {
+    return new Promise((res) => {
+      this.s3.statObject(
+        this.config.bucket,
+        file,
+        // @ts-expect-error this callback is not in the types but the code for it is there
+        (err: unknown, stat: BucketItemStat) => {
+          if (err) res(null);
+          else res(stat.size);
+        },
+      );
+    });
   }
 
   public async fullSize(): Promise<number> {

src/lib/datasources/Supabase.ts

@@ -72,20 +72,21 @@ export class Supabase extends Datasource {
     }
   }
 
-  public async get(file: string): Promise<Readable> {
+  public async get(file: string, start: number = 0, end: number = Infinity): Promise<Readable> {
     // get a readable stream from the request
     const r = await fetch(`${this.config.url}/storage/v1/object/${this.config.bucket}/${file}`, {
       method: 'GET',
       headers: {
         Authorization: `Bearer ${this.config.key}`,
+        Range: `bytes=${start}-${end === Infinity ? '' : end}`,
       },
     });
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     return Readable.fromWeb(r.body as any);
   }
 
-  public size(file: string): Promise<number> {
+  public size(file: string): Promise<number | null> {
     return new Promise(async (res) => {
       fetch(`${this.config.url}/storage/v1/object/list/${this.config.bucket}`, {
         method: 'POST',
@@ -102,11 +103,11 @@ export class Supabase extends Datasource {
         .then((j) => {
           if (j.error) {
             this.logger.error(`${j.error}: ${j.message}`);
-            res(0);
+            res(null);
           }
 
           if (j.length === 0) {
-            res(0);
+            res(null);
           } else {
             res(j[0].metadata.size);
           }

src/lib/utils/range.ts

@@ -0,0 +1,9 @@
+export function parseRangeHeader(header?: string): [number, number] {
+  if (!header || !header.startsWith('bytes=')) return [0, Infinity];
+
+  const range = header.replace('bytes=', '').split('-');
+  const start = Number(range[0]) || 0;
+  const end = Number(range[1]) || Infinity;
+
+  return [start, end];
+}

src/server/decorators/dbFile.ts

@@ -2,6 +2,7 @@ import { File } from '@prisma/client';
 import { FastifyInstance, FastifyReply } from 'fastify';
 import fastifyPlugin from 'fastify-plugin';
 import exts from 'lib/exts';
+import { parseRangeHeader } from 'lib/utils/range';
 
 function dbFileDecorator(fastify: FastifyInstance, _, done) {
   fastify.decorateReply('dbFile', dbFile);
@@ -13,19 +14,29 @@ function dbFileDecorator(fastify: FastifyInstance, _, done) {
     const ext = file.name.split('.').pop();
     if (Object.keys(exts).includes(ext)) return this.server.nextHandle(this.request.raw, this.raw);
 
-    const data = await this.server.datasource.get(file.name);
-    if (!data) return this.notFound();
-
     const size = await this.server.datasource.size(file.name);
+    if (size === null) return this.notFound();
+
+    // eslint-disable-next-line prefer-const
+    let [rangeStart, rangeEnd] = parseRangeHeader(this.request.headers.range);
+    if (rangeStart >= rangeEnd)
+      return this.code(416)
+        .header('Content-Range', `bytes */${size - 1}`)
+        .send();
+    if (rangeEnd === Infinity) rangeEnd = size - 1;
+
+    const data = await this.server.datasource.get(file.name, rangeStart, rangeEnd);
+
+    // only send content-range if the client asked for it
+    if (this.request.headers.range) {
+      this.code(206);
+      this.header('Content-Range', `bytes ${rangeStart}-${rangeEnd}/${size}`);
+    }
 
-    this.header('Content-Length', size);
+    this.header('Content-Length', rangeEnd - rangeStart + 1);
     this.header('Content-Type', download ? 'application/octet-stream' : file.mimetype);
     this.header('Content-Disposition', `inline; filename="${encodeURI(file.originalName || file.name)}"`);
-    if (file.mimetype.startsWith('video/') || file.mimetype.startsWith('audio/')) {
-      this.header('Accept-Ranges', 'bytes');
-      // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range
-      this.header('Content-Range', `bytes 0-${size - 1}/${size}`);
-    }
+    this.header('Accept-Ranges', 'bytes');
 
     return this.send(data);
   }

yarn.lock

@@ -1956,6 +1956,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/node@npm:18":
+  version: 18.19.67
+  resolution: "@types/node@npm:18.19.67"
+  dependencies:
+    undici-types: ~5.26.4
+  checksum: 700f92c6a0b63352ce6327286392adab30bb17623c2a788811e9cf092c4dc2fb5e36ca4727247a981b3f44185fdceef20950a3b7a8ab72721e514ac037022a08
+  languageName: node
+  linkType: hard
+
 "@types/node@npm:^10.0.3":
   version: 10.17.60
   resolution: "@types/node@npm:10.17.60"
@@ -1970,15 +1979,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/node@npm:^18.18.10":
-  version: 18.18.10
-  resolution: "@types/node@npm:18.18.10"
-  dependencies:
-    undici-types: ~5.26.4
-  checksum: 1245a14a38bfbe115b8af9792dbe87a1c015f2532af5f0a25a073343fefa7b2edfd95ff3830003d1a1278ce7f9ee0e78d4e5454d7a60af65832c8d77f4032ac8
-  languageName: node
-  linkType: hard
-
 "@types/normalize-package-data@npm:^2.4.0":
   version: 2.4.4
   resolution: "@types/normalize-package-data@npm:2.4.4"
@@ -11827,7 +11827,7 @@ __metadata:
     "@types/katex": ^0.16.6
     "@types/minio": ^7.1.1
     "@types/multer": ^1.4.10
-    "@types/node": ^18.18.10
+    "@types/node": 18
     "@types/qrcode": ^1.5.5
     "@types/react": ^18.2.37
     "@types/sharp": ^0.32.0