Support ArrayBuffer, TypedArray, DataView, Blob and Web Streams in every environment

Author: Acconut

lib/browser/BrowserFileReader.ts

@@ -1,15 +1,13 @@
 import { isReactNativeFile, isReactNativePlatform } from '../reactnative/isReactNative.js'
 import { uriToBlob } from '../reactnative/uriToBlob.js'
 
+import {
+  openFile as openBaseFile,
+  supportedTypes as supportedBaseTypes,
+} from '../commonFileReader.js'
 import type { FileReader, FileSource, UploadInput } from '../options.js'
-import { BlobFileSource } from './sources/BlobFileSource.js'
-import { StreamFileSource } from './sources/StreamFileSource.js'
+import { BlobFileSource } from '../sources/BlobFileSource.js'
 
-function isWebStream(input: UploadInput): input is Pick<ReadableStreamDefaultReader, 'read'> {
-  return 'read' in input && typeof input.read === 'function'
-}
-
-// TODO: Make sure that we support ArrayBuffers, TypedArrays, DataViews and Blobs
 export class BrowserFileReader implements FileReader {
   async openFile(input: UploadInput, chunkSize: number): Promise<FileSource> {
     // In React Native, when user selects a file, instead of a File or Blob,
@@ -31,26 +29,11 @@ export class BrowserFileReader implements FileReader {
       }
     }
 
-    // File is a subtype of Blob, so we can check for Blob here.
-    // TODO: Consider turning Blobs, Buffers, and Uint8Arrays into a single type.
-    // Potentially handling it in the same way as in Node.js
-    if (input instanceof Blob) {
-      return new BlobFileSource(input)
-    }
-
-    if (isWebStream(input)) {
-      chunkSize = Number(chunkSize)
-      if (!Number.isFinite(chunkSize)) {
-        throw new Error(
-          'cannot create source for stream without a finite value for the `chunkSize` option',
-        )
-      }
-
-      return new StreamFileSource(input)
-    }
+    const fileSource = openBaseFile(input, chunkSize)
+    if (fileSource) return fileSource
 
     throw new Error(
-      'source object may only be an instance of File, Blob, or Reader in this environment',
+      `in this environment the source object may only be an instance of: ${supportedBaseTypes.join(', ')}`,
     )
   }
 }

lib/commonFileReader.ts

@@ -0,0 +1,48 @@
+import type { FileSource, UploadInput } from './options.js'
+import { ArrayBufferViewFileSource } from './sources/ArrayBufferViewFileSource.js'
+import { BlobFileSource } from './sources/BlobFileSource.js'
+import { WebStreamFileSource, isWebStream } from './sources/WebStreamFileSource.js'
+
+/**
+ * openFile provides FileSources for input types that have to be handled in all environments,
+ * including Node.js and browsers.
+ */
+export function openFile(input: UploadInput, chunkSize: number): FileSource | null {
+  // File is a subtype of Blob, so we only check for Blob here.
+  if (input instanceof Blob) {
+    return new BlobFileSource(input)
+  }
+
+  // ArrayBufferViews can be TypedArray (e.g. Uint8Array) or DataView instances.
+  // Note that Node.js' Buffers are also Uint8Arrays.
+  if (ArrayBuffer.isView(input)) {
+    return new ArrayBufferViewFileSource(input)
+  }
+
+  if (input instanceof ArrayBuffer || input instanceof SharedArrayBuffer) {
+    const view = new DataView(input)
+    return new ArrayBufferViewFileSource(view)
+  }
+
+  if (isWebStream(input)) {
+    chunkSize = Number(chunkSize)
+    if (!Number.isFinite(chunkSize)) {
+      throw new Error(
+        'cannot create source for stream without a finite value for the `chunkSize` option',
+      )
+    }
+
+    return new WebStreamFileSource(input)
+  }
+
+  return null
+}
+
+export const supportedTypes = [
+  'File',
+  'Blob',
+  'ArrayBuffer',
+  'SharedArrayBuffer',
+  'ArrayBufferView',
+  'ReadableStreamDefaultReader (Web Streams)',
+]

lib/node/NodeFileReader.ts

@@ -1,36 +1,35 @@
 import { ReadStream } from 'node:fs'
 import isStream from 'is-stream'
 
+import {
+  openFile as openBaseFile,
+  supportedTypes as supportedBaseTypes,
+} from '../commonFileReader.js'
 import type { FileReader, UploadInput } from '../options.js'
-import { BufferFileSource } from './sources/BufferFileSource.js'
 import { getFileSource } from './sources/NodeFileSource.js'
-import { StreamFileSource } from './sources/StreamFileSource.js'
+import { NodeStreamFileSource } from './sources/NodeStreamFileSource.js'
 
 export class NodeFileReader implements FileReader {
-  // TODO: Use async here and less Promise.resolve
   openFile(input: UploadInput, chunkSize: number) {
-    if (Buffer.isBuffer(input)) {
-      return Promise.resolve(new BufferFileSource(input))
-    }
-
     if (input instanceof ReadStream && input.path != null) {
       return getFileSource(input)
     }
 
     if (isStream.readable(input)) {
       chunkSize = Number(chunkSize)
       if (!Number.isFinite(chunkSize)) {
-        return Promise.reject(
-          new Error(
-            'cannot create source for stream without a finite value for the `chunkSize` option; specify a chunkSize to control the memory consumption',
-          ),
+        throw new Error(
+          'cannot create source for stream without a finite value for the `chunkSize` option; specify a chunkSize to control the memory consumption',
         )
       }
-      return Promise.resolve(new StreamFileSource(input))
+      return Promise.resolve(new NodeStreamFileSource(input))
     }
 
-    return Promise.reject(
-      new Error('source object may only be an instance of Buffer or Readable in this environment'),
+    const fileSource = openBaseFile(input, chunkSize)
+    if (fileSource) return Promise.resolve(fileSource)
+
+    throw new Error(
+      `in this environment the source object may only be an instance of: ${supportedBaseTypes.join(', ')}, fs.ReadStream (Node.js), stream.Readable (Node.js)`,
     )
   }
 }

lib/node/NodeHttpStack.ts

@@ -102,6 +102,14 @@ class Request implements HttpRequest {
         reject(err)
       })
 
+      if (body instanceof ArrayBuffer || body instanceof SharedArrayBuffer) {
+        body = new Uint8Array(body)
+      }
+
+      if (ArrayBuffer.isView(body) && !(body instanceof Uint8Array)) {
+        body = new Uint8Array(body.buffer, body.byteOffset, body.byteLength)
+      }
+
       if (body instanceof Readable) {
         // Readable stream are piped through a PassThrough instance, which
         // counts the number of bytes passed through. This is used, for example,

lib/node/index.ts

@@ -19,8 +19,8 @@ const defaultOptions = {
   fingerprint,
 }
 
-export type FileTypes = Buffer | Readable | ReadStream
-export type FileSliceTypes = Buffer | ReadStream
+export type FileTypes = ArrayBuffer | SharedArrayBuffer | ArrayBufferView | Readable | ReadStream
+export type FileSliceTypes = ArrayBuffer | SharedArrayBuffer | ArrayBufferView | ReadStream
 
 class Upload extends BaseUpload {
   constructor(file: UploadInput, options: Partial<UploadOptions> = {}) {

lib/node/sources/BufferFileSource.ts

@@ -46,7 +46,7 @@ async function readChunk(stream: Readable, size: number) {
  */
 // TODO: Consider converting the node stream in a web stream. Then we can share the stream
 // handling between browsers and node.js.
-export class StreamFileSource implements FileSource {
+export class NodeStreamFileSource implements FileSource {
   // Setting the size to null indicates that we have no calculation available
   // for how much data this stream will emit requiring the user to specify
   // it manually (see the `uploadSize` option).

lib/node/sources/StreamFileSource.ts renamed to lib/node/sources/NodeStreamFileSource.ts

@@ -20,14 +20,16 @@ export interface ReactNativeFile {
 
 export type UploadInput =
   // Blob, File, ReadableStreamDefaultReader are available in browsers and Node.js
-  | Blob
-  | File
+  | Blob // includes File
+  | ArrayBuffer
+  | SharedArrayBuffer
+  | ArrayBufferView // includes Node.js' Buffer
   // TODO: Should we keep the Pick<> here?
+  // TODO: Should we also accept ReadableStreamBYOBReader? What about ReadableStream?
   | Pick<ReadableStreamDefaultReader, 'read'>
   // Buffer, stream.Readable, fs.ReadStream are available in Node.js
-  | Buffer
-  | Readable
-  | ReadStream
+  | Readable // TODO: Replace this with our own interface based on https://github.com/DefinitelyTyped/DefinitelyTyped/blob/3634b01d50c10ce1afaae63e41d39e7da309d8e3/types/node/globals.d.ts#L399
+  | ReadStream // TODO: Replace this with { path: string, start: number?, end: number? }
   // ReactNativeFile is intended for React Native apps
   | ReactNativeFile
 
@@ -111,8 +113,7 @@ export type SliceResult =
   | {
       done: boolean
       // Platform-specific data type which must be usable by the HTTP stack as a body.
-      // TODO: Consider making this a Uint8Array (and optionally a web stream).
-      // This would make typing easier, but then we force file data to be read into memory.
+      // TODO: This shouldn't be unknown, but precise values.
       value: NonNullable<unknown>
       size: number
     }

lib/options.ts

@@ -0,0 +1,37 @@
+import type { FileSource, SliceResult } from '../options.js'
+
+/**
+ * ArrayBufferViewFileSource implements FileSource for ArrayBufferView instances
+ * (e.g. TypedArry or DataView).
+ *
+ * Note that the underlying ArrayBuffer should not change once passed to tus-js-client
+ * or it will lead to weird behavior.
+ */
+export class ArrayBufferViewFileSource implements FileSource {
+  private _view: ArrayBufferView
+
+  size: number
+
+  constructor(view: ArrayBufferView) {
+    this._view = view
+    this.size = view.byteLength
+  }
+
+  slice(start: number, end: number): Promise<SliceResult> {
+    const buffer = this._view.buffer
+    const startInBuffer = this._view.byteOffset + start
+    end = Math.min(end, this.size) // ensure end is finite and not greater than size
+    const byteLength = end - start
+
+    // Use DataView instead of ArrayBuffer.slice to avoid copying the buffer.
+    const value = new DataView(buffer, startInBuffer, byteLength)
+    const size = value.byteLength
+    const done = end >= this.size
+
+    return Promise.resolve({ value, size, done })
+  }
+
+  close() {
+    // Nothing to do here since we don't need to release any resources.
+  }
+}

lib/sources/ArrayBufferViewFileSource.ts

@@ -1,7 +1,10 @@
-import { isCordova } from '../../cordova/isCordova.js'
-import { readAsByteArray } from '../../cordova/readAsByteArray.js'
-import type { FileSource, SliceResult } from '../../options.js'
+import { isCordova } from '../cordova/isCordova.js'
+import { readAsByteArray } from '../cordova/readAsByteArray.js'
+import type { FileSource, SliceResult } from '../options.js'
 
+/**
+ * BlobFileSource implements FileSource for Blobs (and therefore also for File instances).
+ */
 export class BlobFileSource implements FileSource {
   private _file: Blob
 
@@ -13,6 +16,8 @@ export class BlobFileSource implements FileSource {
   }
 
   async slice(start: number, end: number): Promise<SliceResult> {
+    // TODO: This looks fishy. We should test how this actually works in Cordova
+    // and consider moving this into the lib/cordova/ directory.
     // In Apache Cordova applications, a File must be resolved using
     // FileReader instances, see
     // https://cordova.apache.org/docs/en/8.x/reference/cordova-plugin-file/index.html#read-a-file

lib/browser/sources/BlobFileSource.ts renamed to lib/sources/BlobFileSource.ts

@@ -1,6 +1,6 @@
-import type { FileSource, SliceResult } from '../../options.js'
+import type { FileSource, SliceResult, UploadInput } from '../options.js'
 
-function len(blobOrArray: StreamFileSource['_buffer']): number {
+function len(blobOrArray: WebStreamFileSource['_buffer']): number {
   if (blobOrArray === undefined) return 0
   if (blobOrArray instanceof Blob) return blobOrArray.size
   return blobOrArray.length
@@ -10,7 +10,7 @@ function len(blobOrArray: StreamFileSource['_buffer']): number {
   Typed arrays and blobs don't have a concat method.
   This function helps StreamSource accumulate data to reach chunkSize.
 */
-function concat<T extends StreamFileSource['_buffer']>(a: T, b: T): T {
+function concat<T extends WebStreamFileSource['_buffer']>(a: T, b: T): T {
   if (a instanceof Blob && b instanceof Blob) {
     return new Blob([a, b], { type: a.type }) as T
   }
@@ -23,8 +23,11 @@ function concat<T extends StreamFileSource['_buffer']>(a: T, b: T): T {
   throw new Error('Unknown data type')
 }
 
-export class StreamFileSource implements FileSource {
-  private _reader: Pick<ReadableStreamDefaultReader<StreamFileSource['_buffer']>, 'read'>
+/**
+ * WebStreamFileSource implements FileSource for Web Streams.
+ */
+export class WebStreamFileSource implements FileSource {
+  private _reader: Pick<ReadableStreamDefaultReader<WebStreamFileSource['_buffer']>, 'read'>
 
   private _buffer: Blob | Uint8Array | undefined
 
@@ -104,3 +107,9 @@ export class StreamFileSource implements FileSource {
     if (this._reader.cancel) this._reader.cancel()
   }
 }
+
+export function isWebStream(
+  input: UploadInput,
+): input is Pick<ReadableStreamDefaultReader, 'read'> {
+  return 'read' in input && typeof input.read === 'function'
+}

lib/browser/sources/StreamFileSource.ts renamed to lib/sources/WebStreamFileSource.ts

@@ -30,14 +30,14 @@
         }
       }
     },
-    "./node/sources/StreamFileSource": {
+    "./node/sources/NodeStreamFileSource": {
       "import": {
-        "types": "./lib.esm/node/sources/StreamFileSource.d.ts",
-        "default": "./lib.esm/node/sources/StreamFileSource.js"
+        "types": "./lib.esm/node/sources/NodeStreamFileSource.d.ts",
+        "default": "./lib.esm/node/sources/NodeStreamFileSource.js"
       },
       "require": {
-        "types": "./lib.cjs/node/sources/StreamFileSource.d.ts",
-        "default": "./lib.cjs/node/sources/StreamFileSource.js"
+        "types": "./lib.cjs/node/sources/NodeStreamFileSource.d.ts",
+        "default": "./lib.cjs/node/sources/NodeStreamFileSource.js"
       }
     },
     "./node/FileUrlStorage": {

package.json

@@ -153,7 +153,15 @@ async function getBodySize(body) {
     return null
   }
 
-  if (body.size != null) {
+  if (
+    body instanceof ArrayBuffer ||
+    body instanceof SharedArrayBuffer ||
+    ArrayBuffer.isView(body)
+  ) {
+    return body.byteLength
+  }
+
+  if (body instanceof Blob) {
     return body.size
   }