Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
c7f00c0
Add build and deploy gh workflow (#1)
rpurdel Nov 26, 2025
6c4cc9c
fix: add libopus dep
rpurdel Nov 26, 2025
87562c2
chore: move node install
rpurdel Nov 26, 2025
7d0e38b
chore: rename gh workflow
rpurdel Nov 26, 2025
9505f2b
fix: activate emsdk
rpurdel Nov 26, 2025
75f4448
fix: also checkout submodules on build
rpurdel Nov 26, 2025
5562848
Fix logging of OpenAI error message.
JonathanLennox Nov 26, 2025
79ebc94
fix: stale context
rpurdel Nov 26, 2025
95f8d5e
Temp: log transcription events chattily.
JonathanLennox Nov 26, 2025
f797f1c
fix: remove libopus
rpurdel Nov 26, 2025
7a16b82
Temp: Remove previous chattiness. Log opus decoding chattily, and lo…
JonathanLennox Nov 26, 2025
aa50e59
Temp: Move chattiness to Opus WASM module loading.
JonathanLennox Nov 26, 2025
01f4300
move build to emscripten step
rpurdel Nov 26, 2025
d7abd85
Properly handle errors in promise.
JonathanLennox Nov 26, 2025
36f7c46
chore: revert build changes, set the default to working emscripten 4.…
rpurdel Nov 27, 2025
dfe9452
Get emscripten code working with latest emscripten (4.0.20).
JonathanLennox Dec 1, 2025
9e6fba2
Format.
JonathanLennox Dec 3, 2025
1cc1287
Include confidence and message IDs in transcription messages.
JonathanLennox Dec 3, 2025
d46c073
Clean up some logic around resetting an outbound stream.
JonathanLennox Dec 3, 2025
d9e259e
Correctly reference the syntax of incoming messages from OpenAI.
JonathanLennox Dec 3, 2025
98aff28
Change default OpenAI model to gpt-4o-mini-transcribe. Make it overr…
JonathanLennox Dec 4, 2025
4c22673
Make turn_detection configurable with an env var.
bgrozev Dec 4, 2025
f2bf1d5
Merge pull request #2 from jitsi/configurable-turn-detection
JonathanLennox Dec 4, 2025
231fc91
Add metrics (#3)
rpurdel Dec 8, 2025
735239a
Close the worker websocket on Opus or OpenAI failure. (#4)
JonathanLennox Dec 8, 2025
5ec0730
Make sendBackInterim a separate URL parameter. (#5)
JonathanLennox Dec 8, 2025
eb24dfc
Catch and handle errors from EventEmitter callbacks. (#6)
JonathanLennox Dec 8, 2025
f81388a
Explicitly compare URL param values with 'true'. (#7)
JonathanLennox Dec 8, 2025
9868f1c
feat: logpush for opus-transcriber-proxy (#8)
aaronkvanmeerten Dec 9, 2025
3a3e0d1
Add a connection language param, leave empty for auto (#9)
rpurdel Dec 9, 2025
38d73f6
disable dev domain (#10)
rpurdel Dec 9, 2025
b6a6be5
Fix PLC logic. (#11)
JonathanLennox Dec 10, 2025
98ceeb2
Add some metrics for normal correct behavior. (#12)
JonathanLennox Dec 11, 2025
6b687f4
Commit audio to OpenAI when resetting a connection, before clearing i…
JonathanLennox Dec 11, 2025
3c3a69b
Receive and process transcription failure messages, and add a metric …
JonathanLennox Dec 11, 2025
dac9ee1
Rpurdel/fixes (#15)
rpurdel Dec 15, 2025
06faf90
chore: log unhandled openai events (#16)
rpurdel Dec 15, 2025
39bbfe2
Add some more normal OpenAI messages that shouldn't trigger an unexpe…
JonathanLennox Dec 15, 2025
7818da1
revert tag switch changes (#19)
rpurdel Dec 16, 2025
12568b4
feat: heartbeat (#21)
rpurdel Dec 16, 2025
29f927c
feat: transcription flush (#20)
rpurdel Dec 16, 2025
4939af6
Ignore the OpenAI error input_audio_buffer_commit_empty. (#22)
JonathanLennox Dec 16, 2025
9954e3b
Avoid double-free of Opus decoder. (#23)
JonathanLennox Dec 16, 2025
f3beeb9
Adjust turn detection settings (#24)
bgrozev Dec 16, 2025
4dbf78b
Fix typeof check. (#25)
bgrozev Dec 16, 2025
0650005
Mirror ping id, add an event field. (#28)
bgrozev Dec 18, 2025
985cec8
log actual OpenAI connection error instead of [object ErrorEvent] (#26)
rpurdel Dec 18, 2025
7eeff8d
Add MetricCache. (#29)
JonathanLennox Dec 18, 2025
b73e9ad
Fix capitalization
JonathanLennox Dec 18, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 103 additions & 0 deletions .github/workflows/build.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
name: "Build and deploy"
on:
# push:
# branches:
# - master
# - main
workflow_dispatch:
inputs:
cf_env:
description: Choose environment
type: choice
required: true
options:
- dev
- staging
- prod
default: dev
branch:
description: Choose branch or tag, defaults to main
type: string
required: true
default: main
emsdk_version:
description: Emscripten SDK version, defaults to 4.0.20
type: string
required: false
default: "4.0.20"
preCommand:
description: Provide a bash script to execute before running wrangler
type: string
required: false
default: echo "No script provided for execution before running Wrangler. Moving along."
postCommand:
description: Provide a bash script to execute after running wrangler
type: string
required: false
default: echo "Nothing to execute after running Wrangler. Finishing..."
jobs:
deploy:
runs-on: "ubuntu-latest"
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ inputs.branch }}
submodules: 'true'

- name: Set environment variables
run: |
if [[ "${{ inputs.cf_env }}" == "dev" ]]; then
echo "CF_ACCOUNT=DEV_ACCOUNT_ID" >> "$GITHUB_ENV"
echo "CF_TOKEN=JITSI_CF_DEV_TOKEN" >> "$GITHUB_ENV"
elif [[ "${{ inputs.cf_env }}" == "staging" ]]; then
echo "CF_ACCOUNT=STAGE_ACCOUNT_ID" >> "$GITHUB_ENV"
echo "CF_TOKEN=JITSI_CF_STAGING_TOKEN" >> "$GITHUB_ENV"
elif [[ "${{ inputs.cf_env }}" == "prod" ]]; then
echo "CF_ACCOUNT=PROD_ACCOUNT_ID" >> "$GITHUB_ENV"
echo "CF_TOKEN=JITSI_CF_PROD_TOKEN" >> "$GITHUB_ENV"
else
echo "Invalid environment specified: ${{ inputs.cf_env }}, exiting."
exit 1
fi

- name: Install Linux deps
run: |
sudo apt update
sudo apt -y install wget unzip

- uses: actions/setup-node@v4
with:
node-version: 22
cache: npm

- name: Install Node Dependencies
run: |
npm ci

- name: Setup Emscripten SDK
run: |
wget https://github.com/emscripten-core/emsdk/archive/main.zip
unzip main.zip
cd emsdk-main
./emsdk install ${{ inputs.emsdk_version }}
./emsdk activate ${{ inputs.emsdk_version }}
source ./emsdk_env.sh
echo "EMSDK=$EMSDK" >> $GITHUB_ENV
echo "EM_CONFIG=$EM_CONFIG" >> $GITHUB_ENV
echo "$EMSDK:$EMSDK/upstream/emscripten" >> $GITHUB_PATH

- name: Build TS App
run: |
source "$EMSDK/emsdk_env.sh"
npm run build

- name: Wrangler Deploy
uses: cloudflare/wrangler-action@v3
with:
wranglerVersion: "4.51.0"
apiToken: ${{ secrets[env.CF_TOKEN] }}
accountId: ${{ secrets[env.CF_ACCOUNT] }}
preCommands: ${{ inputs.preCommand }}
postCommands: ${{ inputs.postCommand }}
command: deploy
11 changes: 9 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,17 @@ OPUS_DECODER_DIST=./dist

OPUS_DECODER_EMSCRIPTEN_BUILD=$(OPUS_DECODER_BUILD)/EmscriptenWasm.tmp.js
OPUS_DECODER_EMSCRIPTEN_WASM=$(OPUS_DECODER_BUILD)/EmscriptenWasm.tmp.wasm
OPUS_DECODER_EMSCRIPTEN_WASM_MAP=$(OPUS_DECODER_BUILD)/EmscriptenWasm.tmp.wasm.map
OPUS_DECODER_MODULE=$(OPUS_DECODER_DIST)/opus-decoder.js
OPUS_DECODER_WASM=$(OPUS_DECODER_DIST)/opus-decoder.wasm
OPUS_DECODER_WASM_MAP=$(OPUS_DECODER_DIST)/opus-decoder.wasm.map

LIBOPUS_SRC=$(OPUS_DECODER_SRC)/opus
LIBOPUS_BUILD=$(OPUS_DECODER_BUILD)/build-opus-wasm
LIBOPUS_WASM_LIB=$(OPUS_DECODER_BUILD)/libopus.a

clean:
rm -rf $(OPUS_DECODER_EMSCRIPTEN_BUILD) $(OPUS_DECODER_EMSCRIPTEN_WASM) $(OPUS_DECODER_MODULE) $(OPUS_DECODER_WASM) $(LIBOPUS_WASM_LIB)
rm -rf $(OPUS_DECODER_EMSCRIPTEN_BUILD) $(OPUS_DECODER_EMSCRIPTEN_WASM) $(OPUS_DECODER_EMSCRIPTEN_WASM_MAP) $(OPUS_DECODER_MODULE) $(OPUS_DECODER_WASM) $(OPUS_DECODER_WASM_MAP) $(LIBOPUS_WASM_LIB)
+emmake $(MAKE) -C $(LIBOPUS_BUILD) clean

configure: libopus-configure
Expand All @@ -35,6 +37,10 @@ opus-decoder: opus-wasmlib $(OPUS_DECODER_EMSCRIPTEN_BUILD)
else \
echo "Warning: WASM file not found, you may need to adjust emscripten settings"; \
fi
@if [ -f "$(OPUS_DECODER_EMSCRIPTEN_WASM_MAP)" ]; then \
cp $(OPUS_DECODER_EMSCRIPTEN_WASM_MAP) $(OPUS_DECODER_WASM_MAP); \
echo "Copied WASM source map to $(OPUS_DECODER_WASM_MAP)"; \
fi

# libopus
opus-wasmlib: $(LIBOPUS_WASM_LIB)
Expand All @@ -44,12 +50,13 @@ define EMCC_OPTS
-O2 \
-msimd128 \
--minify 0 \
-gsource-map \
-s WASM=1 \
-s TEXTDECODER=2 \
-s SINGLE_FILE=0 \
-s MALLOC="emmalloc" \
-s NO_FILESYSTEM=1 \
-s ENVIRONMENT=web \
-s ENVIRONMENT=node \
-s ASSERTIONS=1 \
-s ABORTING_MALLOC=0 \
-s EXIT_RUNTIME=0 \
Expand Down
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"devDependencies": {
"@cloudflare/vitest-pool-workers": "^0.8.19",
"@types/node": "^24.7.2",
"prettier": "3.6.2",
"prettier": "^3.6.2",
"typescript": "^5.9.3",
"vitest": "~3.2.0",
"wrangler": "^4.38.0"
Expand Down
75 changes: 75 additions & 0 deletions src/MetricCache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { writeMetric, MetricEvent } from './metrics';

/**
* Aggregates metric counts and periodically flushes them to Analytics Engine.
* Reduces write frequency by batching metrics over a time interval.
*/
export class MetricCache {
private analytics: AnalyticsEngineDataset | undefined;
private intervalMs: number;
private metrics: Map<string, { event: MetricEvent; count: number; lastWriteTime: number }>;

/**
* @param analytics - The Analytics Engine dataset to write to
* @param intervalMs - Time interval in milliseconds between metric writes (default: 1000ms)
*/
constructor(analytics: AnalyticsEngineDataset | undefined, intervalMs: number = 1000) {
this.analytics = analytics;
this.intervalMs = intervalMs;
this.metrics = new Map();
}

/**
* Increments the count for a metric. If the time interval has elapsed since
* the last write, flushes the accumulated count to Analytics Engine.
*
* @param event - The metric event to increment
*/
increment(event: MetricEvent): void {
const key = this.getKey(event);
const now = Date.now();
const metric = this.metrics.get(key);

if (!metric) {
// First time seeing this metric
this.metrics.set(key, { event, count: 1, lastWriteTime: now });
} else {
// Increment existing metric
metric.count++;

// Check if it's time to flush
if (now - metric.lastWriteTime >= this.intervalMs) {
writeMetric(this.analytics, metric.event, metric.count);
metric.count = 0;
metric.lastWriteTime = now;
}
}
}

/**
* Flushes all accumulated metrics immediately, regardless of time interval.
* Useful for cleanup on shutdown or before long idle periods.
*/
flush(): void {
for (const [_, metric] of this.metrics) {
if (metric.count > 0) {
writeMetric(this.analytics, metric.event, metric.count);
metric.count = 0;
metric.lastWriteTime = Date.now();
}
}
}

/**
* Generates a unique key for a metric event based on its distinguishing properties.
* Does not include sessionId to allow aggregation across sessions.
*/
private getKey(event: MetricEvent): string {
return JSON.stringify({
name: event.name,
worker: event.worker,
errorType: event.errorType ?? '',
targetName: event.targetName ?? '',
});
}
}
84 changes: 67 additions & 17 deletions src/OpusDecoder/OpusDecoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,14 @@
// submodules and compiled into the dist files, may have different
// licensing terms."

// Provide Node.js globals for emscripten module. HACK.
if (typeof globalThis.__filename === 'undefined') {
globalThis.__filename = './opus-decoder.js';
}
if (typeof globalThis.__dirname === 'undefined') {
globalThis.__dirname = '.';
}

import OpusDecoderModule from '../../dist/opus-decoder.js';
// @ts-ignore
import wasm from '../../dist/opus-decoder.wasm';
Expand Down Expand Up @@ -68,22 +76,31 @@ export class OpusDecoder<SampleRate extends OpusDecoderSampleRate | undefined =
static opusModule = new Promise<OpusWasmInstance>((resolve, reject) => {
OpusDecoderModule({
instantiateWasm(info: WebAssembly.Imports, receive: (instance: WebAssembly.Instance) => void) {
let instance = new WebAssembly.Instance(wasm, info);
receive(instance);
return instance.exports;
try {
let instance = new WebAssembly.Instance(wasm, info);
receive(instance);
return instance.exports;
} catch (error) {
reject(error);
throw error;
}
},
}).then((module: any) => {
resolve({
opus_frame_decoder_create: module._opus_frame_decoder_create,
opus_frame_decoder_destroy: module._opus_frame_decoder_destroy,
opus_frame_decoder_reset: module._opus_frame_decoder_reset,
opus_frame_decode: module._opus_frame_decode,
malloc: module._malloc,
free: module._free,
HEAP: module.wasmMemory.buffer,
module,
})
.then((module: any) => {
resolve({
opus_frame_decoder_create: module._opus_frame_decoder_create,
opus_frame_decoder_destroy: module._opus_frame_decoder_destroy,
opus_frame_decoder_reset: module._opus_frame_decoder_reset,
opus_frame_decode: module._opus_frame_decode,
malloc: module._malloc,
free: module._free,
HEAP: module.wasmMemory.buffer,
module,
});
})
.catch((error) => {
reject(error);
});
});
});

private _sampleRate: OpusDecoderSampleRate;
Expand All @@ -98,7 +115,7 @@ export class OpusDecoder<SampleRate extends OpusDecoderSampleRate | undefined =
private wasm!: OpusWasmInstance;
private _input!: TypedArrayAllocation<Uint8Array>;
private _output!: TypedArrayAllocation<Int16Array>;
private _decoder!: number;
private _decoder: number | undefined;

constructor(
options: {
Expand Down Expand Up @@ -136,6 +153,8 @@ export class OpusDecoder<SampleRate extends OpusDecoderSampleRate | undefined =
const wasmInstance = await OpusDecoder.opusModule;
this.wasm = wasmInstance;

console.log('OpusDecoder WASM module loaded');

this._input = this.allocateTypedArray(this._inputSize, Uint8Array);

this._output = this.allocateTypedArray(this._channels * this._outputChannelSize, Int16Array);
Expand All @@ -150,6 +169,9 @@ export class OpusDecoder<SampleRate extends OpusDecoderSampleRate | undefined =
}

reset() {
if (this._decoder === undefined) {
throw new Error('Decoder freed or not initialized');
}
this.wasm.opus_frame_decoder_reset(this._decoder);
}

Expand All @@ -174,7 +196,10 @@ export class OpusDecoder<SampleRate extends OpusDecoderSampleRate | undefined =
});
this._pointers.clear();

this.wasm.opus_frame_decoder_destroy(this._decoder);
if (this._decoder !== undefined) {
this.wasm.opus_frame_decoder_destroy(this._decoder);
this._decoder = undefined;
}
}

addError(
Expand All @@ -197,6 +222,18 @@ export class OpusDecoder<SampleRate extends OpusDecoderSampleRate | undefined =
decodeFrame(opusFrame: Uint8Array): OpusDecodedAudio<SampleRate extends undefined ? OpusDecoderDefaultSampleRate : SampleRate> {
const errors: DecodeError[] = [];

if (this._decoder === undefined) {
this.addError(errors, 'Decoder freed or not initialized', 0, 0, 0, 0);
console.error('Decoder freed or not initialized');
return {
errors,
pcmData: new Int16Array(0),
channels: this._channels,
samplesDecoded: 0,
sampleRate: this._sampleRate,
} as OpusDecodedAudio<SampleRate extends undefined ? OpusDecoderDefaultSampleRate : SampleRate>;
}

this._input.buf.set(opusFrame);

let samplesDecoded = this.wasm.opus_frame_decode(
Expand Down Expand Up @@ -237,10 +274,23 @@ export class OpusDecoder<SampleRate extends OpusDecoderSampleRate | undefined =
opusFrame: Uint8Array | undefined,
samplesToConceal: number,
): OpusDecodedAudio<SampleRate extends undefined ? OpusDecoderDefaultSampleRate : SampleRate> {
const errors: DecodeError[] = [];

if (this._decoder === undefined) {
this.addError(errors, 'Decoder freed or not initialized', 0, 0, 0, 0);
console.error('Decoder freed or not initialized');
return {
errors,
pcmData: new Int16Array(0),
channels: this._channels,
samplesDecoded: 0,
sampleRate: this._sampleRate,
} as OpusDecodedAudio<SampleRate extends undefined ? OpusDecoderDefaultSampleRate : SampleRate>;
}

if (samplesToConceal > this._outputChannelSize) {
samplesToConceal = this._outputChannelSize;
}
const errors: DecodeError[] = [];
let samplesDecoded: number;
let inLength: number;
if (opusFrame !== undefined) {
Expand Down
Loading