diff --git a/lib/contribute/audio.ts b/lib/contribute/audio.ts index d42fe28..11bfe43 100644 --- a/lib/contribute/audio.ts +++ b/lib/contribute/audio.ts @@ -22,20 +22,30 @@ export class Encoder { } #start(controller: TransformStreamDefaultController) { - this.#encoder = new AudioEncoder({ - output: (frame, metadata) => { - this.#enqueue(controller, frame, metadata) - }, - error: (err) => { - throw err - }, - }) + try { + this.#encoder = new AudioEncoder({ + output: (frame, metadata) => { + this.#enqueue(controller, frame, metadata) + }, + error: (err) => { + throw err + }, + }) - this.#encoder.configure(this.#encoderConfig) + this.#encoder.configure(this.#encoderConfig) + } catch (e) { + console.error("Failed to configure AudioEncoder:", e) + throw e + } } #transform(frame: AudioData) { - this.#encoder.encode(frame) + try { + this.#encoder.encode(frame) + } catch (e) { + console.error("Failed to encode audio frame:", e) + throw e + } frame.close() } diff --git a/lib/contribute/broadcast.ts b/lib/contribute/broadcast.ts index b60a1c2..e00344e 100644 --- a/lib/contribute/broadcast.ts +++ b/lib/contribute/broadcast.ts @@ -73,7 +73,7 @@ export class Broadcast { name: `${track.name}.m4s`, initTrack: `${track.name}.mp4`, selectionParams: { - mimeType: "audio/ogg", + mimeType: "audio/mp4", codec: config.audio.codec, samplerate: settings.sampleRate, //sampleSize: settings.sampleSize, @@ -104,6 +104,7 @@ export class Broadcast { } async #run() { + console.log("[Broadcast] #run loop started") await this.connection.announce(this.namespace) for (;;) { @@ -119,6 +120,7 @@ export class Broadcast { } async #serveSubscribe(subscriber: SubscribeRecv) { + console.log(`[Broadcast] #serveSubscribe for: ${subscriber.track}`) try { const [base, ext] = splitExt(subscriber.track) if (ext === "catalog") { diff --git a/lib/contribute/container.ts b/lib/contribute/container.ts index eac8683..056e59f 100644 --- a/lib/contribute/container.ts +++ b/lib/contribute/container.ts @@ -18,10 +18,16 @@ export class Container { this.encode = new TransformStream({ transform: (frame, controller) => { - if (isDecoderConfig(frame)) { - return this.#init(frame, controller) - } else { - return this.#enqueue(frame, controller) + try { + if (isDecoderConfig(frame)) { + console.log("Container received decoder config:", frame) + return this.#init(frame, controller) + } else { + return this.#enqueue(frame, controller) + } + } catch (e) { + console.error("Container failed to process frame:", e) + throw e } }, }) @@ -64,7 +70,6 @@ export class Container { const data = new MP4.Stream(desc, 8, MP4.Stream.LITTLE_ENDIAN) dops.parse(data) - dops.Version = 0 options.description = dops options.hdlr = "soun" } else { diff --git a/lib/contribute/track.ts b/lib/contribute/track.ts index cec70d8..a6a7d8f 100644 --- a/lib/contribute/track.ts +++ b/lib/contribute/track.ts @@ -21,6 +21,8 @@ export class Track { constructor(media: MediaStreamTrack, config: BroadcastConfig) { this.name = media.kind + console.log(`[Track] constructor for: ${this.name}`) + // We need to split based on type because Typescript is hard if (isAudioTrack(media)) { if (!config.audio) throw new Error("no audio config") @@ -34,6 +36,8 @@ export class Track { } async #runAudio(track: MediaStreamAudioTrack, config: AudioEncoderConfig) { + console.log(`[Track] #runAudio for: ${this.name}`) + const source = new MediaStreamTrackProcessor({ track }) const encoder = new Audio.Encoder(config) const container = new Container() @@ -45,10 +49,19 @@ export class Track { abort: (e) => this.#close(e), }) - return source.readable.pipeThrough(encoder.frames).pipeThrough(container.encode).pipeTo(segments) + return source.readable + .pipeThrough(encoder.frames) + .pipeThrough(container.encode) + .pipeTo(segments) + .catch((err) => { + console.error("Audio pipeline error:", err) + throw err + }) } async #runVideo(track: MediaStreamVideoTrack, config: VideoEncoderConfig) { + console.log(`[Track] #runVideo for: ${this.name}`) + const source = new MediaStreamTrackProcessor({ track }) const encoder = new Video.Encoder(config) const container = new Container() @@ -64,6 +77,8 @@ export class Track { } async #write(chunk: Chunk) { + console.log(`[Track: ${this.name}] #write received chunk of type: ${chunk.type}`) + if (chunk.type === "init") { this.#init = chunk.data this.#notify.wake() @@ -72,6 +87,8 @@ export class Track { let current = this.#segments.at(-1) if (!current || chunk.type === "key") { + console.log(`[Track: ${this.name}] Keyframe received or first segment. Creating new segment.`) + if (current) { await current.input.close() } @@ -99,6 +116,7 @@ export class Track { const writer = current.input.getWriter() if ((writer.desiredSize || 0) > 0) { + console.log(`[Track: ${this.name}] Writing chunk to segment ${current.id}`) await writer.write(chunk) } else { console.warn("dropping chunk", writer.desiredSize) @@ -112,6 +130,7 @@ export class Track { const current = this.#segments.at(-1) if (current) { + console.log(`[Track: ${this.name}] Closing segment ${current.id}`) await current.input.close() } diff --git a/lib/media/mp4/index.ts b/lib/media/mp4/index.ts index 0625198..b8339e0 100644 --- a/lib/media/mp4/index.ts +++ b/lib/media/mp4/index.ts @@ -17,7 +17,11 @@ export function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack { // TODO contribute to mp4box MP4.BoxParser.dOpsBox.prototype.write = function (stream: MP4.Stream) { - this.size = this.ChannelMappingFamily === 0 ? 11 : 13 + this.ChannelMapping!.length + this.size = 11 // Base size for dOps box + if (this.ChannelMappingFamily !== 0) { + this.size += 2 + this.ChannelMapping!.length + } + this.writeHeader(stream) stream.writeUint8(this.Version) @@ -27,9 +31,9 @@ MP4.BoxParser.dOpsBox.prototype.write = function (stream: MP4.Stream) { stream.writeInt16(this.OutputGain) stream.writeUint8(this.ChannelMappingFamily) - if (!this.StreamCount || !this.CoupledCount) throw new Error("failed to write dOps box") - if (this.ChannelMappingFamily !== 0) { + if (!this.StreamCount || !this.CoupledCount) throw new Error("failed to write dOps box with channel mapping") + stream.writeUint8(this.StreamCount) stream.writeUint8(this.CoupledCount) for (const mapping of this.ChannelMapping!) { diff --git a/lib/moq-publisher/index.ts b/lib/moq-publisher/index.ts new file mode 100644 index 0000000..d7f9927 --- /dev/null +++ b/lib/moq-publisher/index.ts @@ -0,0 +1,189 @@ +// src/components/publisher-moq.ts + +import STYLE_SHEET from "./publisher-moq.css" +import { PublisherApi, PublisherOptions } from "../publish" + +export class PublisherMoq extends HTMLElement { + private shadow: ShadowRoot + private cameraSelect!: HTMLSelectElement + private microphoneSelect!: HTMLSelectElement + private previewVideo!: HTMLVideoElement + private connectButton!: HTMLButtonElement + private playbackUrlTextarea!: HTMLTextAreaElement + private mediaStream: MediaStream | null = null + + private publisher?: PublisherApi + private isPublishing = false + private namespace = "" + + constructor() { + super() + this.shadow = this.attachShadow({ mode: "open" }) + + // CSS + const style = document.createElement("style") + style.textContent = STYLE_SHEET + this.shadow.appendChild(style) + + const container = document.createElement("div") + container.classList.add("publisher-container") + + this.cameraSelect = document.createElement("select") + this.microphoneSelect = document.createElement("select") + this.previewVideo = document.createElement("video") + this.connectButton = document.createElement("button") + this.playbackUrlTextarea = document.createElement("textarea") + + this.previewVideo.autoplay = true + this.previewVideo.playsInline = true + this.previewVideo.muted = true + this.connectButton.textContent = "Connect" + + this.playbackUrlTextarea.readOnly = true + this.playbackUrlTextarea.rows = 3 + this.playbackUrlTextarea.style.display = "none" + this.playbackUrlTextarea.style.width = "100%" + this.playbackUrlTextarea.style.marginTop = "1rem" + + container.append( + this.cameraSelect, + this.microphoneSelect, + this.previewVideo, + this.connectButton, + this.playbackUrlTextarea, + ) + this.shadow.appendChild(container) + + // Bindings + this.handleDeviceChange = this.handleDeviceChange.bind(this) + this.handleClick = this.handleClick.bind(this) + + // Listeners + navigator.mediaDevices.addEventListener("devicechange", this.handleDeviceChange) + this.cameraSelect.addEventListener("change", () => this.startPreview()) + this.microphoneSelect.addEventListener("change", () => this.startPreview()) + this.connectButton.addEventListener("click", this.handleClick) + } + + connectedCallback() { + this.populateDeviceLists() + } + + disconnectedCallback() { + navigator.mediaDevices.removeEventListener("devicechange", this.handleDeviceChange) + } + + private async handleDeviceChange() { + await this.populateDeviceLists() + } + + private async populateDeviceLists() { + const devices = await navigator.mediaDevices.enumerateDevices() + const vids = devices.filter((d) => d.kind === "videoinput") + const mics = devices.filter((d) => d.kind === "audioinput") + + this.cameraSelect.innerHTML = "" + this.microphoneSelect.innerHTML = "" + + vids.forEach((d) => { + const o = document.createElement("option") + o.value = d.deviceId + o.textContent = d.label || `Camera ${this.cameraSelect.length + 1}` + this.cameraSelect.append(o) + }) + mics.forEach((d) => { + const o = document.createElement("option") + o.value = d.deviceId + o.textContent = d.label || `Mic ${this.microphoneSelect.length + 1}` + this.microphoneSelect.append(o) + }) + + await this.startPreview() + } + + private async startPreview() { + const vidId = this.cameraSelect.value + const micId = this.microphoneSelect.value + if (this.mediaStream) { + this.mediaStream.getTracks().forEach((t) => t.stop()) + } + this.mediaStream = await navigator.mediaDevices.getUserMedia({ + video: vidId ? { deviceId: { exact: vidId } } : true, + audio: micId ? { deviceId: { exact: micId } } : true, + }) + + this.previewVideo.srcObject = this.mediaStream + } + + private async handleClick() { + if (!this.isPublishing) { + if (!this.mediaStream) { + console.warn("No media stream available") + return + } + + this.namespace = this.getAttribute("namespace") ?? crypto.randomUUID() + + const audioTrack = this.mediaStream.getAudioTracks()[0] + const settings = audioTrack.getSettings() + + const sampleRate = settings.sampleRate ?? (await new AudioContext()).sampleRate + const numberOfChannels = settings.channelCount ?? 2 + + const videoConfig: VideoEncoderConfig = { + codec: "avc1.42E01E", + width: this.previewVideo.videoWidth, + height: this.previewVideo.videoHeight, + bitrate: 1000000, + framerate: 60, + } + const audioConfig: AudioEncoderConfig = { codec: "opus", sampleRate, numberOfChannels, bitrate: 64000 } + + const opts: PublisherOptions = { + url: this.getAttribute("src")!, + fingerprintUrl: this.getAttribute("fingerprint") ?? undefined, + namespace: [this.namespace], + media: this.mediaStream, + video: videoConfig, + audio: audioConfig, + } + + console.log("Publisher Options", opts) + + this.publisher = new PublisherApi(opts) + + try { + await this.publisher.publish() + this.isPublishing = true + this.connectButton.textContent = "Stop" + this.cameraSelect.disabled = true + this.microphoneSelect.disabled = true + + const playbackBaseUrl = this.getAttribute("playbackbaseurl") + if (playbackBaseUrl) { + this.playbackUrlTextarea.value = `${playbackBaseUrl}${this.namespace}` + } else { + this.playbackUrlTextarea.value = this.namespace + } + this.playbackUrlTextarea.style.display = "block" + } catch (err) { + console.error("Publish failed:", err) + } + } else { + try { + await this.publisher!.stop() + } catch (err) { + console.error("Stop failed:", err) + } finally { + this.isPublishing = false + this.connectButton.textContent = "Connect" + this.cameraSelect.disabled = false + this.microphoneSelect.disabled = false + this.playbackUrlTextarea.style.display = "none" + } + } + } +} + +customElements.define("publisher-moq", PublisherMoq) +export default PublisherMoq diff --git a/lib/moq-publisher/publisher-moq.css b/lib/moq-publisher/publisher-moq.css new file mode 100644 index 0000000..35f812f --- /dev/null +++ b/lib/moq-publisher/publisher-moq.css @@ -0,0 +1,17 @@ +.publisher-container { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +#cameraSelect, +#microphoneSelect, +#connect { + font-size: 1rem; + padding: 0.5rem; +} + +#preview { + background: black; + object-fit: cover; +} diff --git a/lib/package.json b/lib/package.json index 7c653f1..9160c58 100644 --- a/lib/package.json +++ b/lib/package.json @@ -4,19 +4,27 @@ "description": "Media over QUIC library", "license": "(MIT OR Apache-2.0)", "wc-player": "video-moq/index.ts", + "wc-publisher": "moq-publisher/index.ts", "simple-player": "playback/index.ts", - "files": ["dist"], + "files": [ + "dist" + ], "exports": { ".": { "import": "./dist/moq-player.esm.js", "require": "./dist/moq-player.cjs.js" }, + "./moq-publisher": { + "import": "./dist/moq-publisher.esm.js", + "require": "./dist/moq-publisher.cjs.js" + }, "./simple-player": { "import": "./dist/moq-simple-player.esm.js", "require": "./dist/moq-simple-player.cjs.js" } }, "iife": "dist/moq-player.iife.js", + "iife-publisher": "dist/moq-publisher.iife.js", "iife-simple": "dist/moq-simple-player.iife.js", "types": "dist/types/moq-player.d.ts", "scripts": { @@ -58,8 +66,18 @@ "mp4box": "^0.5.2" }, "browserslist": { - "production": ["chrome >= 97", "edge >= 98", "firefox >= 130", "opera >= 83", "safari >= 18"], - "development": ["last 1 chrome version", "last 1 firefox version", "last 1 safari version"] + "production": [ + "chrome >= 97", + "edge >= 98", + "firefox >= 130", + "opera >= 83", + "safari >= 18" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] }, "repository": { "type": "git", diff --git a/lib/playback/index.ts b/lib/playback/index.ts index 497e7d0..cc3ce47 100644 --- a/lib/playback/index.ts +++ b/lib/playback/index.ts @@ -49,8 +49,8 @@ export default class Player extends EventTarget { this.#catalog = catalog this.#tracksByName = new Map(catalog.tracks.map((track) => [track.name, track])) this.#tracknum = tracknum - this.#audioTrackName = "" - this.#videoTrackName = "" + this.#audioTrackName = catalog.tracks.find((track) => Catalog.isAudioTrack(track))?.name ?? "" + this.#videoTrackName = catalog.tracks.find((track) => Catalog.isVideoTrack(track))?.name ?? "" this.#muted = false this.#paused = false this.#backend = new Backend({ canvas, catalog }, this) @@ -91,9 +91,11 @@ export default class Player extends EventTarget { const tracks = new Array() this.#catalog.tracks.forEach((track, index) => { - if (index == this.#tracknum || Catalog.isAudioTrack(track)) { + if (track.name === this.#videoTrackName || track.name === this.#audioTrackName) { if (!track.namespace) throw new Error("track has no namespace") if (track.initTrack) inits.add([track.namespace.join("/"), track.initTrack]) + // log every track we push here + console.log("pushing track", track.name) tracks.push(track) } }) @@ -110,6 +112,7 @@ export default class Player extends EventTarget { } async #runInit(namespace: string, name: string) { + console.log("running init for", namespace, name) const sub = await this.#connection.subscribe([namespace], name) try { const init = await Promise.race([sub.data(), this.#running]) diff --git a/lib/playback/worker/video.ts b/lib/playback/worker/video.ts index f433a57..43bac46 100644 --- a/lib/playback/worker/video.ts +++ b/lib/playback/worker/video.ts @@ -45,8 +45,9 @@ export class Renderer { pause() { this.#paused = true + console.log(`[VideoWorker] pause called, decoder state: ${this.#decoder.state}`) this.#decoder.flush().catch((err) => { - console.error(err) + console.error("Failed to flush video decoder on pause:", err) }) this.#waitingForKeyframe = true } @@ -80,13 +81,19 @@ export class Renderer { output: (frame: VideoFrame) => { controller.enqueue(frame) }, - error: console.error, + error: (e: Error) => { + console.error("VideoDecoder error:", e) + }, }) } #transform(frame: Frame) { + console.log( + `[VideoWorker] #transform received frame. track: ${frame.track.codec}, sync: ${frame.sample.is_sync}, state: ${this.#decoder.state}`, + ) + if (this.#decoder.state === "closed" || this.#paused) { - console.warn("Decoder is closed or paused. Skipping frame.") + console.warn("[VideoWorker] Decoder is closed or paused. Skipping frame.") return } @@ -109,9 +116,17 @@ export class Renderer { // Configure the decoder with the first frame if (this.#decoder.state !== "configured") { + console.log("[VideoWorker] Decoder is not configured. Attempting to configure.") + const desc = sample.description + console.log("[VideoWorker] Received init segment description:", desc) const box = desc.avcC ?? desc.hvcC ?? desc.vpcC ?? desc.av1C - if (!box) throw new Error(`unsupported codec: ${track.codec}`) + if (!box) { + console.error( + "[VideoWorker] FAILED: No valid codec configuration box (avcC, etc.) found in init segment.", + ) + throw new Error(`unsupported codec: ${track.codec}`) + } const buffer = new MP4.Stream(undefined, 0, MP4.Stream.BIG_ENDIAN) box.write(buffer) @@ -127,7 +142,16 @@ export class Renderer { // optimizeForLatency: true } - this.#decoder.configure(this.#decoderConfig) + console.log("[VideoWorker] Configuring decoder with:", this.#decoderConfig) + + try { + this.#decoder.configure(this.#decoderConfig) + console.log(`[VideoWorker] Decoder configured successfully. New state: ${this.#decoder.state}`) + } catch (e) { + console.error("[VideoWorker] FAILED to configure decoder:", e) + return // Stop processing if configure fails + } + if (!frame.sample.is_sync) { this.#waitingForKeyframe = true } else { @@ -158,7 +182,12 @@ export class Renderer { timestamp: frame.sample.dts / frame.track.timescale, }) - this.#decoder.decode(chunk) + console.log(`[VideoWorker] Decoding chunk, type: ${chunk.type}, size: ${chunk.byteLength}`) + try { + this.#decoder.decode(chunk) + } catch (e) { + console.error("[VideoWorker] FAILED to decode chunk:", e) + } } } } diff --git a/lib/publish/index.ts b/lib/publish/index.ts new file mode 100644 index 0000000..b82c564 --- /dev/null +++ b/lib/publish/index.ts @@ -0,0 +1,56 @@ +// publisher-api.ts +import { Client } from "../transport/client" +import { Broadcast, BroadcastConfig } from "../contribute" +import { Connection } from "../transport/connection" + +export interface PublisherOptions { + url: string + namespace: string[] + media: MediaStream + video?: VideoEncoderConfig + audio?: AudioEncoderConfig + fingerprintUrl?: string +} + +export class PublisherApi { + private client: Client + private connection?: Connection + private broadcast?: Broadcast + private opts: PublisherOptions + + constructor(opts: PublisherOptions) { + this.opts = opts + this.client = new Client({ + url: opts.url, + fingerprint: opts.fingerprintUrl, + role: "publisher", + }) + } + + async publish(): Promise { + if (!this.connection) { + this.connection = await this.client.connect() + } + + const bcConfig: BroadcastConfig = { + connection: this.connection, + namespace: this.opts.namespace, + media: this.opts.media, + video: this.opts.video, + audio: this.opts.audio, + } + + this.broadcast = new Broadcast(bcConfig) + } + + async stop(): Promise { + if (this.broadcast) { + this.broadcast.close() + await this.broadcast.closed() + } + if (this.connection) { + this.connection.close() + await this.connection.closed() + } + } +} diff --git a/lib/rollup.config.js b/lib/rollup.config.js index 22f5292..fdee046 100644 --- a/lib/rollup.config.js +++ b/lib/rollup.config.js @@ -31,6 +31,23 @@ const basePlugins = [ ] module.exports = [ + { + input: pkg["wc-publisher"], + output: [ + { + file: pkg["iife-publisher"], + format: "iife", + name: "MoqPublisher", + sourcemap: true, + }, + { + file: pkg.exports["./moq-publisher"].import, + format: "esm", + sourcemap: true, + }, + ], + plugins: [...basePlugins, css()], + }, { input: pkg["wc-player"], output: [ diff --git a/lib/transport/publisher.ts b/lib/transport/publisher.ts index 1f328c4..4da766f 100644 --- a/lib/transport/publisher.ts +++ b/lib/transport/publisher.ts @@ -86,12 +86,12 @@ export class Publisher { this.#subscribe.set(msg.id, subscribe) await this.#subscribeQueue.push(subscribe) - await this.#control.send({ - kind: Control.Msg.SubscribeOk, - id: msg.id, - expires: 0n, - group_order: msg.group_order, - }) + // await this.#control.send({ + // kind: Control.Msg.SubscribeOk, + // id: msg.id, + // expires: 0n, + // group_order: msg.group_order, + // }) } recvUnsubscribe(_msg: Control.Unsubscribe) { diff --git a/lib/video-moq/index.ts b/lib/video-moq/index.ts index 9287cee..d118740 100644 --- a/lib/video-moq/index.ts +++ b/lib/video-moq/index.ts @@ -238,11 +238,15 @@ export class VideoMoq extends HTMLElement { const fingerprint = urlParams.get("fingerprint") || this.getAttribute("fingerprint") // TODO: Unsure if fingerprint should be optional - if (namespace === null || fingerprint === null) return + if (namespace === null) return const trackNumStr = urlParams.get("trackNum") || this.trackNum const trackNum: number = this.auxParseInt(trackNumStr, 0) - Player.create({ url: url.origin, fingerprint, canvas: this.#canvas, namespace }, trackNum) + + Player.create( + { url: url.origin, fingerprint: fingerprint ?? undefined, canvas: this.#canvas, namespace }, + trackNum, + ) .then((player) => this.setPlayer(player)) .catch((e) => this.fail(e)) diff --git a/package-lock.json b/package-lock.json index 11b4ae8..dfd4820 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ }, "lib": { "name": "@moq-js/player", - "version": "0.2.0", + "version": "0.4.3", "license": "(MIT OR Apache-2.0)", "dependencies": { "mp4box": "^0.5.2" diff --git a/samples/dynamic-player/index.html b/samples/dynamic-player/index.html new file mode 100644 index 0000000..ae81c95 --- /dev/null +++ b/samples/dynamic-player/index.html @@ -0,0 +1,40 @@ + + + + + + Dynamic MoQ Player + + + + +

Dynamic MoQ Player

+ + + + + + diff --git a/samples/publisher/index.html b/samples/publisher/index.html new file mode 100644 index 0000000..c9f1389 --- /dev/null +++ b/samples/publisher/index.html @@ -0,0 +1,8 @@ + + + + +