Skip to content

Commit a4a6a16

Browse files
authored
Merge pull request #2812 from element-hq/toger5/track-processor-blur
Add background blur feature for supported devices
2 parents f92c683 + eeb3a1a commit a4a6a16

15 files changed

+1328
-85
lines changed

locales/en/app.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,9 @@
164164
"effect_volume_description": "Adjust the volume at which reactions and hand raised effects play.",
165165
"effect_volume_label": "Sound effect volume"
166166
},
167+
"background_blur_header": "Background",
168+
"background_blur_label": "Blur the background of the video",
169+
"blur_not_supported_by_browser": "(Background blur is not supported by this device.)",
167170
"developer_tab_title": "Developer",
168171
"devices": {
169172
"camera": "Camera",

package.json

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
"@livekit/components-core": "^0.12.0",
4141
"@livekit/components-react": "^2.0.0",
4242
"@livekit/protocol": "^1.33.0",
43+
"@livekit/track-processors": "^0.5.5",
44+
"@mediapipe/tasks-vision": "^0.10.18",
4345
"@opentelemetry/api": "^1.4.0",
4446
"@opentelemetry/core": "^1.25.1",
4547
"@opentelemetry/exporter-trace-otlp-http": "^0.57.0",
@@ -70,8 +72,8 @@
7072
"@types/react-dom": "^18.3.0",
7173
"@types/sdp-transform": "^2.4.5",
7274
"@types/uuid": "10",
73-
"@typescript-eslint/eslint-plugin": "^8.0.0",
74-
"@typescript-eslint/parser": "^8.0.0",
75+
"@typescript-eslint/eslint-plugin": "^8.31.0",
76+
"@typescript-eslint/parser": "^8.31.0",
7577
"@use-gesture/react": "^10.2.11",
7678
"@vector-im/compound-design-tokens": "^3.0.0",
7779
"@vector-im/compound-web": "^7.2.0",
@@ -118,7 +120,7 @@
118120
"react-use-measure": "^2.1.1",
119121
"rxjs": "^7.8.1",
120122
"sass": "^1.42.1",
121-
"typescript": "^5.1.6",
123+
"typescript": "^5.8.3",
122124
"typescript-eslint-language-service": "^5.0.5",
123125
"unique-names-generator": "^4.6.0",
124126
"vaul": "^1.0.0",
@@ -131,6 +133,7 @@
131133
},
132134
"resolutions": {
133135
"@livekit/components-core/rxjs": "^7.8.1",
136+
"@livekit/track-processors/@mediapipe/tasks-vision": "^0.10.18",
134137
"matrix-widget-api": "1.11.0"
135138
},
136139
"packageManager": "[email protected]"
Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
/* eslint-disable */
2+
// The contents of this file below the line are copied from
3+
// @types/dom-mediacapture-transform, which is inlined here into Element Call so
4+
// that we can apply the patch to @types/dom-webcodecs found in
5+
// ./dom-webcodecs.d.ts, which it depends on.
6+
// (https://github.com/DefinitelyTyped/DefinitelyTyped/pull/72625)
7+
// Once that PR is merged and released, we can remove this file and return to
8+
// depending on @types/dom-mediacapture-transform.
9+
// -----------------------------------------------------------------------------
10+
11+
// This project is licensed under the MIT license.
12+
// Copyrights are respective of each contributor listed at the beginning of each definition file.
13+
14+
// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
15+
16+
// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
17+
18+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
19+
20+
// In general, these types are only available behind a command line flag or an origin trial in
21+
// Chrome 90+.
22+
23+
// This API depends on WebCodecs.
24+
25+
// Versioning:
26+
// Until the above-mentioned spec is finalized, the major version number is 0. Although not
27+
// necessary for version 0, consider incrementing the minor version number for breaking changes.
28+
29+
// The following modify existing DOM types to allow defining type-safe APIs on audio and video tracks.
30+
31+
/** Specialize MediaStreamTrack so that we can refer specifically to an audio track. */
32+
interface MediaStreamAudioTrack extends MediaStreamTrack {
33+
readonly kind: "audio";
34+
clone(): MediaStreamAudioTrack;
35+
}
36+
37+
/** Specialize MediaStreamTrack so that we can refer specifically to a video track. */
38+
interface MediaStreamVideoTrack extends MediaStreamTrack {
39+
readonly kind: "video";
40+
clone(): MediaStreamVideoTrack;
41+
}
42+
43+
/** Assert that getAudioTracks and getVideoTracks return the tracks with the appropriate kind. */
44+
interface MediaStream {
45+
getAudioTracks(): MediaStreamAudioTrack[];
46+
getVideoTracks(): MediaStreamVideoTrack[];
47+
}
48+
49+
// The following were originally generated from the spec using
50+
// https://github.com/microsoft/TypeScript-DOM-lib-generator, then heavily modified.
51+
52+
/**
53+
* A track sink that is capable of exposing the unencoded frames from the track to a
54+
* ReadableStream, and exposes a control channel for signals going in the oppposite direction.
55+
*/
56+
interface MediaStreamTrackProcessor<T extends AudioData | VideoFrame> {
57+
/**
58+
* Allows reading the frames flowing through the MediaStreamTrack provided to the constructor.
59+
*/
60+
readonly readable: ReadableStream<T>;
61+
/** Allows sending control signals to the MediaStreamTrack provided to the constructor. */
62+
readonly writableControl: WritableStream<MediaStreamTrackSignal>;
63+
}
64+
65+
declare var MediaStreamTrackProcessor: {
66+
prototype: MediaStreamTrackProcessor<any>;
67+
68+
/** Constructor overrides based on the type of track. */
69+
new (
70+
init: MediaStreamTrackProcessorInit & { track: MediaStreamAudioTrack },
71+
): MediaStreamTrackProcessor<AudioData>;
72+
new (
73+
init: MediaStreamTrackProcessorInit & { track: MediaStreamVideoTrack },
74+
): MediaStreamTrackProcessor<VideoFrame>;
75+
};
76+
77+
interface MediaStreamTrackProcessorInit {
78+
track: MediaStreamTrack;
79+
/**
80+
* If media frames are not read from MediaStreamTrackProcessor.readable quickly enough, the
81+
* MediaStreamTrackProcessor will internally buffer up to maxBufferSize of the frames produced
82+
* by the track. If the internal buffer is full, each time the track produces a new frame, the
83+
* oldest frame in the buffer will be dropped and the new frame will be added to the buffer.
84+
*/
85+
maxBufferSize?: number | undefined;
86+
}
87+
88+
/**
89+
* Takes video frames as input, and emits control signals that result from subsequent processing.
90+
*/
91+
interface MediaStreamTrackGenerator<T extends AudioData | VideoFrame>
92+
extends MediaStreamTrack {
93+
/**
94+
* Allows writing media frames to the MediaStreamTrackGenerator, which is itself a
95+
* MediaStreamTrack. When a frame is written to writable, the frame’s close() method is
96+
* automatically invoked, so that its internal resources are no longer accessible from
97+
* JavaScript.
98+
*/
99+
readonly writable: WritableStream<T>;
100+
/**
101+
* Allows reading control signals sent from any sinks connected to the
102+
* MediaStreamTrackGenerator.
103+
*/
104+
readonly readableControl: ReadableStream<MediaStreamTrackSignal>;
105+
}
106+
107+
type MediaStreamAudioTrackGenerator = MediaStreamTrackGenerator<AudioData> &
108+
MediaStreamAudioTrack;
109+
type MediaStreamVideoTrackGenerator = MediaStreamTrackGenerator<VideoFrame> &
110+
MediaStreamVideoTrack;
111+
112+
declare var MediaStreamTrackGenerator: {
113+
prototype: MediaStreamTrackGenerator<any>;
114+
115+
/** Constructor overrides based on the type of track. */
116+
new (
117+
init: MediaStreamTrackGeneratorInit & {
118+
kind: "audio";
119+
signalTarget?: MediaStreamAudioTrack | undefined;
120+
},
121+
): MediaStreamAudioTrackGenerator;
122+
new (
123+
init: MediaStreamTrackGeneratorInit & {
124+
kind: "video";
125+
signalTarget?: MediaStreamVideoTrack | undefined;
126+
},
127+
): MediaStreamVideoTrackGenerator;
128+
};
129+
130+
interface MediaStreamTrackGeneratorInit {
131+
kind: MediaStreamTrackGeneratorKind;
132+
/**
133+
* (Optional) track to which the MediaStreamTrackGenerator will automatically forward control
134+
* signals. If signalTarget is provided and signalTarget.kind and kind do not match, the
135+
* MediaStreamTrackGenerator’s constructor will raise an exception.
136+
*/
137+
signalTarget?: MediaStreamTrack | undefined;
138+
}
139+
140+
type MediaStreamTrackGeneratorKind = "audio" | "video";
141+
142+
type MediaStreamTrackSignalType = "request-frame";
143+
144+
interface MediaStreamTrackSignal {
145+
signalType: MediaStreamTrackSignalType;
146+
}

0 commit comments

Comments
 (0)