diff --git a/crates/libwebrtc-sys/build.rs b/crates/libwebrtc-sys/build.rs index f63dfe1109..e2fc3bea13 100644 --- a/crates/libwebrtc-sys/build.rs +++ b/crates/libwebrtc-sys/build.rs @@ -197,7 +197,7 @@ fn download_libwebrtc() -> anyhow::Result<()> { break; }; hasher.update(&buffer[0..count]); - let _ = out_file.write(&buffer[0..count])?; + let _: usize = out_file.write(&buffer[0..count])?; } if format!("{:x}", hasher.finalize()) != expected_hash { diff --git a/crates/libwebrtc-sys/include/adm.h b/crates/libwebrtc-sys/include/adm.h new file mode 100644 index 0000000000..0e9fb74f85 --- /dev/null +++ b/crates/libwebrtc-sys/include/adm.h @@ -0,0 +1,143 @@ +#pragma once + + +#define WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE 1 +#include +#include "api/task_queue/task_queue_factory.h" +#include "modules/audio_device/audio_device_impl.h" + +#include + +#include "api/media_stream_interface.h" +#include "api/sequence_checker.h" +#include "modules/audio_device/audio_device_buffer.h" +#include "modules/audio_device/audio_device_generic.h" +#include "modules/audio_device/include/audio_device.h" +#include "modules/audio_device/include/audio_device_defines.h" + +#include "rtc_base/event.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/synchronization/mutex.h" +#include "rtc_base/thread_annotations.h" + +#include "modules/audio_mixer/audio_mixer_impl.h" + +#include "api/audio/audio_mixer.h" + +#include "api/audio/audio_frame.h" + +#include "custom_audio.h" + +#if defined(WEBRTC_USE_X11) +#include +#endif + +#include "microphone_module.h" +#include "system_audio_module.h" + +#if defined(WEBRTC_LINUX) +#include "modules/audio_device/linux/audio_mixer_manager_pulse_linux.h" +#include "modules/audio_device/linux/pulseaudiosymboltable_linux.h" +#include "linux_system_audio_module.h" +#include "linux_microphone_module.h" +#endif + +#if defined(WEBRTC_WIN) +#include "windows_microphone_module.h" +#include "windows_system_audio_module.h" +#endif + +#if defined(WEBRTC_MAC) +#include "macos_microphone_module.h" +#include "macos_system_audio_module.h" +#endif + + + +class AudioSourceManager { + public: + // Creates a `AudioSource` from a microphone. + virtual rtc::scoped_refptr CreateMicrophoneSource() = 0; + // Creates a `AudioSource` from a system audio. + virtual rtc::scoped_refptr CreateSystemSource() = 0; + // Enumerates possible system audio sources. + virtual std::vector EnumerateSystemSource() const = 0; + // Sets the system audio source. + virtual void SetRecordingSource(int id) = 0; + // Sets the volume of the system audio capture. + virtual void SetSystemAudioVolume(float volume) = 0; + // Returns the current volume of the system audio capture. + virtual float GetSystemAudioVolume() const = 0; + // Adds `AudioSource` to `AudioSourceManager`. + virtual void AddSource(rtc::scoped_refptr source) = 0; + // Removes `AudioSource` to `AudioSourceManager`. + virtual void RemoveSource(rtc::scoped_refptr source) = 0; +}; + + +class CustomAudioDeviceModule : public webrtc::AudioDeviceModuleImpl, public AudioSourceManager { + public: + CustomAudioDeviceModule(AudioLayer audio_layer, webrtc::TaskQueueFactory* task_queue_factory); + ~CustomAudioDeviceModule(); + + + static rtc::scoped_refptr Create( + AudioLayer audio_layer, + webrtc::TaskQueueFactory* task_queue_factory); + + static rtc::scoped_refptr CreateForTest( + AudioLayer audio_layer, + webrtc::TaskQueueFactory* task_queue_factory); + + // Mixes source and sends on. + void RecordProcess(); + + // Main initializaton and termination. + int32_t Init() override; + int32_t Terminate(); + int32_t StartRecording() override; + int32_t SetRecordingDevice(uint16_t index) override; + int32_t InitMicrophone() override; + bool MicrophoneIsInitialized() const override; + + // Microphone volume controls. + int32_t MicrophoneVolumeIsAvailable(bool* available) override; + int32_t SetMicrophoneVolume(uint32_t volume) override; + int32_t MicrophoneVolume(uint32_t* volume) const override; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const override; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const override; + + // AudioSourceManager interface. + rtc::scoped_refptr CreateMicrophoneSource() override; + rtc::scoped_refptr CreateSystemSource() override; + std::vector EnumerateSystemSource() const override; + void SetRecordingSource(int id) override; + void SetSystemAudioVolume(float level) override; + float GetSystemAudioVolume() const override; + void AddSource(rtc::scoped_refptr source) override; + void RemoveSource(rtc::scoped_refptr source) override; + + // Microphone mute control. + int32_t MicrophoneMuteIsAvailable(bool* available) override; + int32_t SetMicrophoneMute(bool enable) override; + int32_t MicrophoneMute(bool* enabled) const override; + + private: + // Mixes `AudioSource` to send. + rtc::scoped_refptr mixer = webrtc::AudioMixerImpl::Create(); + + // `AudioSource` for mixing. + std::vector> sources; + std::mutex source_mutex; + + // Audio capture module. + std::unique_ptr audio_recorder; + std::unique_ptr system_recorder; + + // Thread for processing audio frames. + rtc::PlatformThread ptrThreadRec; + + // Used to wait for audio sources. + std::condition_variable cv; + bool quit = false; +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/include/audio_source_manager_proxy.h b/crates/libwebrtc-sys/include/audio_source_manager_proxy.h new file mode 100644 index 0000000000..ed9febe24d --- /dev/null +++ b/crates/libwebrtc-sys/include/audio_source_manager_proxy.h @@ -0,0 +1,24 @@ +#pragma once + +#include "adm.h" +#include "pc/proxy.h" + +class AudioSourceManagerProxy : AudioSourceManager { + AudioSourceManagerProxy(rtc::Thread* primary_thread, + rtc::scoped_refptr c); + public: + static std::unique_ptr Create( + rtc::Thread* primary_thread, + rtc::scoped_refptr c); + rtc::scoped_refptr CreateMicrophoneSource() override; + rtc::scoped_refptr CreateSystemSource() override; + std::vector EnumerateSystemSource() const override; + void AddSource(rtc::scoped_refptr source) override; + void RemoveSource(rtc::scoped_refptr source) override; + void SetRecordingSource(int id) override; + void SetSystemAudioVolume(float level) override; + float GetSystemAudioVolume() const override; + private: + rtc::scoped_refptr adm; + rtc::Thread* primary_thread_; +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/include/bridge.h b/crates/libwebrtc-sys/include/bridge.h index 304662fc18..eb3388012b 100644 --- a/crates/libwebrtc-sys/include/bridge.h +++ b/crates/libwebrtc-sys/include/bridge.h @@ -31,6 +31,9 @@ #include "media/base/fake_frame_source.h" #include "pc/test/fake_video_track_source.h" #include "modules/audio_device/include/test_audio_device.h" +#include "audio_source_manager_proxy.h" +#include "adm.h" + namespace bridge { @@ -103,12 +106,60 @@ using RtpReceiverInterface = rtc::scoped_refptr; using MediaStreamTrackInterface = rtc::scoped_refptr; +using CustomAudioDeviceModule = rtc::scoped_refptr<::CustomAudioDeviceModule>; +using AudioSource = rtc::scoped_refptr<::AudioSource>; +using AudioSourceManager = ::AudioSourceManager; +using AudioSourceInfo = ::AudioSourceInfo; + + // Creates a new proxied `AudioDeviceModule` for the given `AudioLayer`. std::unique_ptr create_audio_device_module( Thread& worker_thread, AudioLayer audio_layer, TaskQueueFactory& task_queue_factory); +// Creates a new `CustomAudioDeviceModule` for the given `AudioLayer`. +std::unique_ptr create_custom_audio_device_module( + Thread& worker_thread, + AudioLayer audio_layer, + TaskQueueFactory& task_queue_factory); + +// Creates a new proxied `AudioDeviceModule` from the provided `CustomAudioDeviceModule`. +std::unique_ptr custom_audio_device_module_proxy_upcast(std::unique_ptr adm, Thread& worker_thread); + +// Creates a new `AudioSourceManager` for the given `CustomAudioDeviceModule`. +std::unique_ptr create_source_manager(const CustomAudioDeviceModule& adm, Thread& worker_thread); + +// Creates a new `AudioSource` from microphone. +std::unique_ptr create_source_microphone(AudioSourceManager& manager); + +// Creates a new `AudioSource` from system. +std::unique_ptr create_system_audio_source(AudioSourceManager& manager); + +// Enumerates possible system audio sources. +std::unique_ptr> enumerate_system_audio_source(const AudioSourceManager& manager); + +// Sets the system audio source. +void set_system_audio_source(AudioSourceManager& manager, int64_t id); + +// Returns `AudioSourceInfo` id. +int64_t system_source_id(const AudioSourceInfo& source); + +// Sets the volume of the system audio capture. +void set_system_audio_source_volume(AudioSourceManager& manager, float level); + +// Returns the current volume of the system audio capture. +float system_audio_source_volume(const AudioSourceManager& manager); + +// Returns `AudioSourceInfo` title. +std::unique_ptr system_source_title(const AudioSourceInfo& source); + +// Adds `AudioSource` to `AudioSourceManager`. +void add_source(AudioSourceManager& manager, const AudioSource& source); + +// Removes `AudioSource` from `AudioSourceManager`. +void remove_source(AudioSourceManager& manager, const AudioSource& source); + // Creates a new fake `AudioDeviceModule`. std::unique_ptr create_fake_audio_device_module( TaskQueueFactory& task_queue_factory); diff --git a/crates/libwebrtc-sys/include/custom_audio.h b/crates/libwebrtc-sys/include/custom_audio.h new file mode 100644 index 0000000000..956d4731db --- /dev/null +++ b/crates/libwebrtc-sys/include/custom_audio.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include "api/audio/audio_frame.h" +#include "api/audio/audio_mixer.h" +#include "common_audio/resampler/include/push_resampler.h" + +class RefCountedAudioSource : public webrtc::AudioMixer::Source, + public rtc::RefCountInterface {}; + +class AudioSource : public rtc::RefCountedObject { + public: + // Overwrites `audio_frame`. The data_ field is overwritten with + // 10 ms of new audio (either 1 or 2 interleaved channels) at + // `sample_rate_hz`. All fields in `audio_frame` must be updated. + webrtc::AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo( + int sample_rate_hz, + webrtc::AudioFrame* audio_frame) override; + + // A way for a mixer implementation to distinguish participants. + int Ssrc() const override; + + // A way for this source to say that GetAudioFrameWithInfo called + // with this sample rate or higher will not cause quality loss. + int PreferredSampleRate() const; + + // Updates the audio frame data. + void UpdateFrame(const int16_t* source, + int size, + int sample_rate, + int channels); + + // Mutes the source until the next frame. + void Mute(); + + // Prepares an audio frame. + void FrameProcessing(int sample_rate_hz, webrtc::AudioFrame* audio_frame); + + private: + // Current audio data. + webrtc::AudioFrame frame_; + // Converts an audio date to the specified rate. + webrtc::PushResampler render_resampler_; + // Buffer for converted audio data. + int16_t resample_buffer[webrtc::AudioFrame::kMaxDataSizeSamples]; + + // Provides synchronization for sending audio frames. + std::mutex mutex_; + std::condition_variable cv_; + std::atomic frame_available_ = false; + std::atomic mute_ = false; + std::chrono::time_point mute_clock_; +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/include/linux_microphone_module.h b/crates/libwebrtc-sys/include/linux_microphone_module.h new file mode 100644 index 0000000000..1f4ce9db5e --- /dev/null +++ b/crates/libwebrtc-sys/include/linux_microphone_module.h @@ -0,0 +1,143 @@ +#pragma once + +#include "custom_audio.h" +#include "microphone_module.h" +#include "modules/audio_device/linux/audio_device_pulse_linux.h" + +typedef webrtc::adm_linux_pulse::PulseAudioSymbolTable WebRTCPulseSymbolTable; +WebRTCPulseSymbolTable* _GetPulseSymbolTable(); + +class MicrophoneModule : public MicrophoneModuleInterface { + public: + MicrophoneModule(); + ~MicrophoneModule(); + + // MicrophoneModuleInterface + + // Initialization and terminate. + int32_t Init(); + int32_t Terminate(); + int32_t InitMicrophone(); + + // Microphone control. + int32_t MicrophoneMuteIsAvailable(bool* available); + int32_t SetMicrophoneMute(bool enable); + int32_t MicrophoneMute(bool* enabled) const; + bool MicrophoneIsInitialized(); + int32_t MicrophoneVolumeIsAvailable(bool* available); + int32_t SetMicrophoneVolume(uint32_t volume); + int32_t MicrophoneVolume(uint32_t* volume) const; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const; + + // Settings. + int32_t SetRecordingDevice(uint16_t index); + + // Microphone source. + rtc::scoped_refptr CreateSource(); + void ResetSource(); + int32_t StopRecording(); + int32_t StartRecording(); + int32_t RecordingChannels(); + + // END MicrophoneModuleInterface + + private: + void PaLock(); + void PaUnLock(); + int32_t LatencyUsecs(pa_stream* stream); + static void PaSourceInfoCallback(pa_context* c, + const pa_source_info* i, + int eol, + void* pThis); + void PaSourceInfoCallbackHandler(const pa_source_info* i, int eol); + + bool RecThreadProcess(); + int32_t ReadRecordedData(const void* bufferData, size_t bufferSize); + int32_t ProcessRecordedData(int8_t* bufferData, + uint32_t bufferSizeInSamples, + uint32_t recDelay); + bool KeyPressed() const; + int16_t RecordingDevices(); + + void EnableReadCallback(); + void DisableReadCallback(); + static void PaStreamReadCallback(pa_stream* a /*unused1*/, + size_t b /*unused2*/, + void* pThis); + void PaStreamReadCallbackHandler(); + static void PaContextStateCallback(pa_context* c, void* pThis); + static void PaServerInfoCallback(pa_context* c, + const pa_server_info* i, + void* pThis); + void PaContextStateCallbackHandler(pa_context* c); + void PaServerInfoCallbackHandler(const pa_server_info* i); + + int32_t InitPulseAudio(); + int32_t TerminatePulseAudio(); + void WaitForOperationCompletion(pa_operation* paOperation) const; + int32_t CheckPulseAudioVersion(); + int32_t InitSamplingFrequency(); + int32_t InitRecording(); + int32_t GetDefaultDeviceInfo(bool recDevice, char* name, uint16_t& index); + static void PaStreamStateCallback(pa_stream* p, void* pThis); + void PaStreamStateCallbackHandler(pa_stream* p); + static void PaStreamOverflowCallback(pa_stream* unused, void* pThis); + void PaStreamOverflowCallbackHandler(); + + int32_t StereoRecordingIsAvailable(bool& available); + int32_t SetStereoRecording(bool enable); + int32_t StereoRecording(bool& enabled) const; + + rtc::scoped_refptr source = nullptr; + bool _inputDeviceIsSpecified = false; + int sample_rate_hz_ = 0; + uint8_t _recChannels = 1; + bool _initialized = false; + bool _recording = false; + bool _recIsInitialized = false; + bool quit_ = false; + bool _startRec = false; + uint32_t _sndCardPlayDelay = 0; + int _deviceIndex = -1; + int16_t _numRecDevices = 0; + char* _playDeviceName = nullptr; + char* _recDeviceName = nullptr; + char* _playDisplayDeviceName = nullptr; + char* _recDisplayDeviceName = nullptr; + int8_t* _recBuffer = nullptr; + size_t _recordBufferSize = 0; + size_t _recordBufferUsed = 0; + const void* _tempSampleData = nullptr; + size_t _tempSampleDataSize = 0; + int32_t _configuredLatencyRec = 0; + uint16_t _paDeviceIndex = -1; + bool _paStateChanged = false; + pa_threaded_mainloop* _paMainloop = nullptr; + pa_mainloop_api* _paMainloopApi = nullptr; + pa_context* _paContext = nullptr; + pa_stream* _recStream = nullptr; + uint32_t _recStreamFlags = 0; + pa_stream* _playStream = nullptr; + + // Stores thread ID in constructor. + // We can then use RTC_DCHECK_RUN_ON(&worker_thread_checker_) to ensure that + // other methods are called from the same thread. + // Currently only does RTC_DCHECK(thread_checker_.IsCurrent()). + webrtc::SequenceChecker thread_checker_; + + rtc::Event _timeEventRec; + webrtc::Mutex mutex_; + int16_t _numPlayDevices; + pa_buffer_attr _recBufferAttr; + int _inputDeviceIndex; + rtc::Event _recStartEvent; + rtc::PlatformThread _ptrThreadRec; + char _paServerVersion[32]; + webrtc::AudioMixerManagerLinuxPulse _mixerManager; + +#if defined(WEBRTC_USE_X11) + char _oldKeyState[32]; + Display* _XDisplay; +#endif +}; diff --git a/crates/libwebrtc-sys/include/linux_system_audio_module.h b/crates/libwebrtc-sys/include/linux_system_audio_module.h new file mode 100644 index 0000000000..5be5e6c1ee --- /dev/null +++ b/crates/libwebrtc-sys/include/linux_system_audio_module.h @@ -0,0 +1,119 @@ +#pragma once + +#include "system_audio_module.h" +#include +#include "rtc_base/platform_thread.h" +#include "rtc_base/event.h" +#include "rtc_base/synchronization/mutex.h" + +class SystemModule : public SystemModuleInterface { + public: + SystemModule(); + ~SystemModule(); + + // Initiates a pulse audio. + int32_t InitPulseAudio(); + + // Terminates a pulse audio. + int32_t TerminatePulseAudio(); + + // Returns the current sinc id for the current process. + int32_t UpdateSinkId(); + + // Locks pulse audio. + void PaLock(); + // Unlocks pulse audio. + void PaUnLock(); + + // Redirect callback to `this.PaContextStateCallbackHandler`. + static void PaContextStateCallback(pa_context* c, void* pThis); + + // Callback to keep track of the state of the pulse audio context. + void PaContextStateCallbackHandler(pa_context* c); + + // Waiting for the pulse audio operation to completion. + void WaitForOperationCompletion(pa_operation* paOperation) const; + + // Callback to keep track of the state of the pulse audio stream. + void PaStreamStateCallbackHandler(pa_stream* p); + + // Initiates the audio stream from the provided `_recSinkId`. + void InitRecording(); + + // Redirect callback to `this.PaStreamStateCallbackHandler`. + static void PaStreamStateCallback(pa_stream* p, void* pThis); + + // Redirect callback to `this.PaStreamReadCallbackHandler`. + static void PaStreamReadCallback(pa_stream* a /*unused1*/, + size_t b /*unused2*/, + void* pThis); + + // Used to signal audio reading. + void PaStreamReadCallbackHandler(); + + // Passes an audio frame to an AudioSource. + int32_t ProcessRecordedData(int8_t* bufferData, + uint32_t bufferSizeInSamples, + uint32_t recDelay); + + // Disables signals for audio recording. + void DisableReadCallback(); + + // Enaables signals for audio recording. + void EnableReadCallback(); + + // Function to capture audio in another thread. + bool RecThreadProcess(); + + + // Interface + // Initialization and terminate. + bool Init(); + int32_t Terminate(); + rtc::scoped_refptr CreateSource(); + void ResetSource(); + + // Settings. + void SetRecordingSource(int id); + void SetSystemAudioLevel(float level); + float GetSystemAudioLevel() const; + int32_t StopRecording(); + int32_t StartRecording(); + int32_t RecordingChannels(); + int32_t ReadRecordedData(const void* bufferData, size_t bufferSize); + + // Enumerate system audio outputs. + std::vector EnumerateSystemSource() const; + + bool _initialized = false; + bool _paStateChanged = false; + rtc::PlatformThread _ptrThreadRec; + rtc::Event _timeEventRec; + rtc::Event _recStartEvent; + bool quit_ = false; + bool _startRec = false; + bool _recording = false; + bool _recIsInitialized = false; + const void* _tempSampleData = nullptr; + size_t _tempSampleDataSize = 0; + bool _mute = false; + bool _on_new = false; + float audio_multiplier = 1.0; + int _recProcessId = -1; + int _recSinkId = -1; + int sample_rate_hz_ = 48000; + uint8_t _recChannels = 2; + char _paServerVersion[32]; + int32_t _configuredLatencyRec = 0; + uint32_t _recStreamFlags = 0; + int8_t* _recBuffer = nullptr; + size_t _recordBufferSize = 0; + size_t _recordBufferUsed = 0; + pa_buffer_attr _recBufferAttr; + webrtc::Mutex mutex_; + pa_threaded_mainloop* _paMainloop = nullptr; + pa_mainloop_api* _paMainloopApi = nullptr; + pa_context* _paContext = nullptr; + pa_stream* _recStream = nullptr; + std::vector release_capture_buffer = std::vector(480 * 8); +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/include/macos_microphone_module.h b/crates/libwebrtc-sys/include/macos_microphone_module.h new file mode 100644 index 0000000000..563eb36466 --- /dev/null +++ b/crates/libwebrtc-sys/include/macos_microphone_module.h @@ -0,0 +1,38 @@ +#pragma once + +#include "custom_audio.h" +#include "microphone_module.h" + +class MicrophoneModule : public MicrophoneModuleInterface { + public: + MicrophoneModule() {} + ~MicrophoneModule() {} + + // MicrophoneModuleInterface + + // Initialization and terminate. + int32_t Init() {return 0;} + int32_t Terminate() {return 0;} + int32_t InitMicrophone() {return 0;} + + // Microphone control. + int32_t MicrophoneMuteIsAvailable(bool* available) {return 0;} + int32_t SetMicrophoneMute(bool enable) {return 0;} + int32_t MicrophoneMute(bool* enabled) const {return 0;} + bool MicrophoneIsInitialized() {return 0;} + int32_t MicrophoneVolumeIsAvailable(bool* available) {return 0;} + int32_t SetMicrophoneVolume(uint32_t volume) {return 0;} + int32_t MicrophoneVolume(uint32_t* volume) const {return 0;} + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const {return 0;} + int32_t MinMicrophoneVolume(uint32_t* minVolume) const {return 0;} + + // Settings. + int32_t SetRecordingDevice(uint16_t index) {return 0;} + + // Microphone source. + rtc::scoped_refptr CreateSource() {return nullptr;} + void ResetSource() {} + int32_t StopRecording() {return 0;} + int32_t StartRecording() {return 0;} + int32_t RecordingChannels() {return 0;} +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/include/macos_system_audio_module.h b/crates/libwebrtc-sys/include/macos_system_audio_module.h new file mode 100644 index 0000000000..d630e5b050 --- /dev/null +++ b/crates/libwebrtc-sys/include/macos_system_audio_module.h @@ -0,0 +1,29 @@ +#pragma once + +#include "system_audio_module.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/event.h" +#include "rtc_base/synchronization/mutex.h" + +class SystemModule : public SystemModuleInterface { + public: + SystemModule() {}; + ~SystemModule() {}; + + // Initialization and terminate. + bool Init() { return false; }; + int32_t Terminate() { return 0; }; + rtc::scoped_refptr CreateSource() { return nullptr; }; + void ResetSource() { return; }; + + // Settings. + void SetRecordingSource(int id) { return; }; + void SetSystemAudioLevel(float level) { return; }; + float GetSystemAudioLevel() const { return 0; }; + int32_t StopRecording() { return 0; }; + int32_t StartRecording() { return 0; }; + int32_t RecordingChannels() { return 0; }; + + // Enumerate system audio outputs. + std::vector EnumerateSystemSource() const { return std::vector(); }; +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/include/microphone_module.h b/crates/libwebrtc-sys/include/microphone_module.h new file mode 100644 index 0000000000..91d34c6698 --- /dev/null +++ b/crates/libwebrtc-sys/include/microphone_module.h @@ -0,0 +1,53 @@ + +#pragma once +#include "custom_audio.h" + + +class MicrophoneModuleInterface; + +// Microphone audio source. +class MicrophoneSource : public AudioSource { + public: + MicrophoneSource(MicrophoneModuleInterface* module); + ~MicrophoneSource(); + + private: + // Microphone capture module. + MicrophoneModuleInterface* module; + // Used to control the module. + static int sources_num; +}; + +// Interface to provide microphone audio capture. +class MicrophoneModuleInterface { +public: + +// Initialization and terminate. +virtual int32_t Init() = 0; +virtual int32_t Terminate() = 0; +virtual int32_t InitMicrophone() = 0; + +// Microphone control. +virtual int32_t MicrophoneMuteIsAvailable(bool* available) = 0; +virtual int32_t SetMicrophoneMute(bool enable) = 0; +virtual int32_t MicrophoneMute(bool* enabled) const = 0; +virtual bool MicrophoneIsInitialized () = 0; +virtual int32_t MicrophoneVolumeIsAvailable(bool* available) = 0; +virtual int32_t SetMicrophoneVolume(uint32_t volume) = 0; +virtual int32_t MicrophoneVolume(uint32_t* volume) const = 0; +virtual int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const = 0; +virtual int32_t MinMicrophoneVolume(uint32_t* minVolume) const = 0; + +// Settings. +virtual int32_t SetRecordingDevice(uint16_t index) = 0; + +// Microphone source. +virtual rtc::scoped_refptr CreateSource() = 0; +virtual void ResetSource() = 0; +virtual int32_t StopRecording() = 0; +virtual int32_t StartRecording() = 0; +virtual int32_t RecordingChannels() = 0; +rtc::scoped_refptr source = nullptr; +}; + + diff --git a/crates/libwebrtc-sys/include/system_audio_module.h b/crates/libwebrtc-sys/include/system_audio_module.h new file mode 100644 index 0000000000..2d539e667f --- /dev/null +++ b/crates/libwebrtc-sys/include/system_audio_module.h @@ -0,0 +1,80 @@ + +#pragma once +#include "custom_audio.h" + +// Audio source info. +class AudioSourceInfo { + public: + AudioSourceInfo(int id, + std::string title, + int process_id, + int rate = 48000, + int channels = 2) + : id(id), + title(title), + process_id(process_id), + rate(rate), + channels(channels) {} + + // Returns the audio source id. + int GetId() const { return id; } + // Returns the process id. + int GetProcessId() const { return process_id; } + // Returns the sample rate. + int GetSampleRate() const { return rate; } + // Returns the channels. + int GetChannels() const { return channels; } + // Returns audio source title. + std::string GetTitle() const { return title; } + + private: + // Audio source id. + int id; + // Audio source process id. + int process_id; + // Audio source title. + std::string title; + // Audio source sample rate. + int rate; + // Audio source channels. + int channels; +}; + +class SystemModuleInterface; + +// System audio source. +class SystemSource : public AudioSource { + public: + SystemSource(SystemModuleInterface* module); + ~SystemSource(); + + private: + // Audio system capture module. + SystemModuleInterface* module; + // Used to control the module. + static int sources_num; +}; + +// Interface to provide system audio capture. +class SystemModuleInterface { + public: + // Initialization and terminate. + virtual bool Init() = 0; + virtual int32_t Terminate() = 0; + virtual rtc::scoped_refptr CreateSource() = 0; + virtual void ResetSource() = 0; + + // Settings. + virtual void SetRecordingSource(int id) = 0; + virtual void SetSystemAudioLevel(float level) = 0; + virtual float GetSystemAudioLevel() const = 0; + virtual int32_t StopRecording() = 0; + virtual int32_t StartRecording() = 0; + virtual int32_t RecordingChannels() = 0; + + // Enumerate system audio outputs. + virtual std::vector EnumerateSystemSource() const = 0; + + // System source. + rtc::scoped_refptr source = nullptr; +}; diff --git a/crates/libwebrtc-sys/include/win-help.h b/crates/libwebrtc-sys/include/win-help.h new file mode 100644 index 0000000000..628f8cbb05 --- /dev/null +++ b/crates/libwebrtc-sys/include/win-help.h @@ -0,0 +1,59 @@ +#pragma once + +#include +#include +#include +#include +#include "system_audio_module.h" +#include + +#define UNUSED_PARAMETER(param) (void)param + +enum window_search_mode { + INCLUDE_MINIMIZED, + EXCLUDE_MINIMIZED, +}; + +inline long os_atomic_inc_long(volatile long *val); +size_t wchar_to_utf8(const wchar_t *in, size_t insize, char *out, + size_t outsize, int flags); + +struct dstr { + char *array; + size_t len; /* number of characters, excluding null terminator */ + size_t capacity; +}; + +inline void dstr_resize(dstr& dst, const size_t num); +inline void dstr_free(dstr& dst); +void dstr_copy(dstr& dst, const char *array); +inline void dstr_ensure_capacity(dstr& dst, const size_t new_size); +inline int dstr_cmp(const dstr& str1, const char *str2); +inline bool dstr_is_empty(const dstr& str); + +void bfree(void *ptr); +void *brealloc(void *ptr, size_t size); + +#define LOWER_HALFBYTE(x) ((x)&0xF) +#define UPPER_HALFBYTE(x) (((x) >> 4) & 0xF) + +std::vector ms_fill_window_list(enum window_search_mode mode); +HWND first_window(enum window_search_mode mode, HWND *parent, + bool *use_findwindowex); +bool check_window_valid(HWND window, enum window_search_mode mode); +inline bool IsWindowCloaked(HWND window); +bool ms_is_uwp_window(HWND hwnd); +HWND next_window(HWND window, enum window_search_mode mode, HWND *parent, + bool use_findwindowex); + +bool ms_get_window_exe(dstr& exe, HWND window); +inline HANDLE open_process(DWORD desired_access, bool inherit_handle, + DWORD process_id); +void *ms_get_obfuscated_func(HMODULE module, const char *str, uint64_t val); +void deobfuscate_str(char *str, uint64_t val); +bool is_microsoft_internal_window_exe(const char *exe); +void ms_get_window_title(dstr& name, HWND hwnd); + +int astrcmpi(const char *str1, const char *str2); +int astrcmpi_n(const char *str1, const char *str2, size_t n); +void *a_realloc(void *ptr, size_t size); diff --git a/crates/libwebrtc-sys/include/windows_microphone_module.h b/crates/libwebrtc-sys/include/windows_microphone_module.h new file mode 100644 index 0000000000..0be5ad1314 --- /dev/null +++ b/crates/libwebrtc-sys/include/windows_microphone_module.h @@ -0,0 +1,154 @@ +#pragma once + +#include "custom_audio.h" +#include "microphone_module.h" +#include "modules/audio_device/win/audio_device_core_win.h" +#include "rtc_base\win\scoped_com_initializer.h" + +class MicrophoneModule : public MicrophoneModuleInterface { + public: + MicrophoneModule(); + ~MicrophoneModule(); + + // MicrophoneModuleInterface + + // Initialization and terminate. + int32_t Init(); + int32_t Terminate(); + int32_t InitMicrophone(); + + // Microphone control. + int32_t MicrophoneMuteIsAvailable(bool* available); + int32_t SetMicrophoneMute(bool enable); + int32_t MicrophoneMute(bool* enabled) const; + bool MicrophoneIsInitialized(); + int32_t MicrophoneVolumeIsAvailable(bool* available); + int32_t SetMicrophoneVolume(uint32_t volume); + int32_t MicrophoneVolume(uint32_t* volume) const; + int32_t MaxMicrophoneVolume(uint32_t* maxVolume) const; + int32_t MinMicrophoneVolume(uint32_t* minVolume) const; + + // Settings. + int32_t SetRecordingDevice(uint16_t index); + + // Microphone source. + rtc::scoped_refptr CreateSource(); + void ResetSource(); + int32_t StopRecording(); + int32_t StartRecording(); + int32_t RecordingChannels(); + + // END MicrophoneModuleInterface + + private: + // Сopied from "audio_device_core_win.h" + + int32_t _EnumerateEndpointDevicesAll(EDataFlow dataFlow) const; + void _TraceCOMError(HRESULT hr) const; + int32_t InitMicrophoneLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + int16_t RecordingDevicesLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + int32_t _GetListDevice(EDataFlow dir, int index, IMMDevice** ppDevice); + int32_t _RefreshDeviceList(EDataFlow dir); + int16_t _DeviceListCount(EDataFlow dir); + int32_t _GetDeviceName(IMMDevice* pDevice, LPWSTR pszBuffer, int bufferLen); + int32_t _GetDefaultDeviceID(EDataFlow dir, + ERole role, + LPWSTR szBuffer, + int bufferLen); + int16_t RecordingDevices(); + DWORD DoCaptureThreadPollDMO(); + DWORD InitCaptureThreadPriority(); + static DWORD WINAPI WSAPICaptureThreadPollDMO(LPVOID context); + static DWORD WINAPI WSAPICaptureThread(LPVOID context); + DWORD DoCaptureThread(); + void RevertCaptureThreadPriority(); + int32_t InitRecording() RTC_LOCKS_EXCLUDED(mutex_); + int32_t InitRecordingDMO(); + int SetDMOProperties(); + int SetBoolProperty(IPropertyStore* ptrPS, + REFPROPERTYKEY key, + VARIANT_BOOL value); + int32_t _GetDefaultDeviceIndex(EDataFlow dir, ERole role, int* index); + int SetVtI4Property(IPropertyStore* ptrPS, REFPROPERTYKEY key, LONG value); + int32_t _GetDeviceID(IMMDevice* pDevice, LPWSTR pszBuffer, int bufferLen); + int32_t _GetDefaultDevice(EDataFlow dir, ERole role, IMMDevice** ppDevice); + + webrtc::ScopedCOMInitializer _comInit; + webrtc::AudioDeviceBuffer* _ptrAudioBuffer; + mutable webrtc::Mutex mutex_; + mutable webrtc::Mutex volume_mutex_ RTC_ACQUIRED_AFTER(mutex_); + + IMMDeviceEnumerator* _ptrEnumerator; + IMMDeviceCollection* _ptrRenderCollection; + IMMDeviceCollection* _ptrCaptureCollection; + IMMDevice* _ptrDeviceOut; + IMMDevice* _ptrDeviceIn; + + PAvRevertMmThreadCharacteristics _PAvRevertMmThreadCharacteristics; + PAvSetMmThreadCharacteristicsA _PAvSetMmThreadCharacteristicsA; + PAvSetMmThreadPriority _PAvSetMmThreadPriority; + HMODULE _avrtLibrary; + bool _winSupportAvrt; + + IAudioClient* _ptrClientOut; + IAudioClient* _ptrClientIn; + IAudioRenderClient* _ptrRenderClient; + IAudioCaptureClient* _ptrCaptureClient; + IAudioEndpointVolume* _ptrCaptureVolume; + ISimpleAudioVolume* _ptrRenderSimpleVolume; + + // DirectX Media Object (DMO) for the built-in AEC. + rtc::scoped_refptr _dmo; + rtc::scoped_refptr _mediaBuffer; + bool _builtInAecEnabled; + + HANDLE _hRenderSamplesReadyEvent; + HANDLE _hPlayThread; + HANDLE _hRenderStartedEvent; + HANDLE _hShutdownRenderEvent; + + HANDLE _hCaptureSamplesReadyEvent; + HANDLE _hRecThread; + HANDLE _hCaptureStartedEvent; + HANDLE _hShutdownCaptureEvent; + + HANDLE _hMmTask; + + UINT _playAudioFrameSize; + uint32_t _playSampleRate; + uint32_t _devicePlaySampleRate; + uint32_t _playBlockSize; + uint32_t _devicePlayBlockSize; + uint32_t _playChannels; + uint32_t _sndCardPlayDelay; + UINT64 _writtenSamples; + UINT64 _readSamples; + + UINT _recAudioFrameSize; + uint32_t _recSampleRate; + uint32_t _recBlockSize; + uint32_t _recChannels; + + uint16_t _recChannelsPrioList[3]; + uint16_t _playChannelsPrioList[2]; + + LARGE_INTEGER _perfCounterFreq; + double _perfCounterFactor; + + bool _initialized; + bool _recording; + bool _playing; + bool _recIsInitialized; + bool _playIsInitialized; + bool _speakerIsInitialized; + bool _microphoneIsInitialized; + + bool _usingInputDeviceIndex; + bool _usingOutputDeviceIndex; + webrtc::AudioDeviceModule::WindowsDeviceType _inputDevice; + webrtc::AudioDeviceModule::WindowsDeviceType _outputDevice; + uint16_t _inputDeviceIndex; + uint16_t _outputDeviceIndex; + + // END Сopied from "audio_device_core_win.h" +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/include/windows_system_audio_module.h b/crates/libwebrtc-sys/include/windows_system_audio_module.h new file mode 100644 index 0000000000..1a2611426b --- /dev/null +++ b/crates/libwebrtc-sys/include/windows_system_audio_module.h @@ -0,0 +1,401 @@ +/* + * Copyright (c) 2014 Hugh Bailey + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "system_audio_module.h" +#include +#include // Put this in to get rid of linker errors. +#include // Property keys defined here are now defined inline. + +#include +#include "win-help.h" + +#ifdef _WIN32 +#include +#endif + +#if defined(_MSC_VER) && defined(_M_X64) +#include +#endif + +#define OBS_KSAUDIO_SPEAKER_4POINT1 \ + (KSAUDIO_SPEAKER_SURROUND | SPEAKER_LOW_FREQUENCY) +#define RECONNECT_INTERVAL 3000 +#define BUFFER_TIME_100NS (5 * 10000000) +typedef HRESULT(STDAPICALLTYPE* PFN_ActivateAudioInterfaceAsync)( + LPCWSTR, + REFIID, + PROPVARIANT*, + IActivateAudioInterfaceCompletionHandler*, + IActivateAudioInterfaceAsyncOperation**); + +typedef HRESULT(STDAPICALLTYPE* PFN_RtwqUnlockWorkQueue)(DWORD); +typedef HRESULT(STDAPICALLTYPE* PFN_RtwqLockSharedWorkQueue)(PCWSTR usageClass, + LONG basePriority, + DWORD* taskId, + DWORD* id); +typedef HRESULT(STDAPICALLTYPE* PFN_RtwqCreateAsyncResult)(IUnknown*, + IRtwqAsyncCallback*, + IUnknown*, + IRtwqAsyncResult**); +typedef HRESULT(STDAPICALLTYPE* PFN_RtwqPutWorkItem)(DWORD, + LONG, + IRtwqAsyncResult*); +typedef HRESULT(STDAPICALLTYPE* PFN_RtwqPutWaitingWorkItem)(HANDLE, + LONG, + IRtwqAsyncResult*, + RTWQWORKITEM_KEY*); + +/* Oh no I have my own com pointer class, the world is ending, how dare you + * write your own! */ + +template +class ComPtr { + protected: + T* ptr; + + inline void Kill() { + if (ptr) + ptr->Release(); + } + + inline void Replace(T* p) { + if (ptr != p) { + if (p) + p->AddRef(); + if (ptr) + ptr->Release(); + ptr = p; + } + } + + public: + inline ComPtr() : ptr(nullptr) {} + inline ComPtr(T* p) : ptr(p) { + if (ptr) + ptr->AddRef(); + } + inline ComPtr(const ComPtr& c) : ptr(c.ptr) { + if (ptr) + ptr->AddRef(); + } + inline ComPtr(ComPtr&& c) noexcept : ptr(c.ptr) { c.ptr = nullptr; } + template + inline ComPtr(ComPtr&& c) noexcept : ptr(c.Detach()) {} + inline ~ComPtr() { Kill(); } + + inline void Clear() { + if (ptr) { + ptr->Release(); + ptr = nullptr; + } + } + + inline ComPtr& operator=(T* p) { + Replace(p); + return *this; + } + + inline ComPtr& operator=(const ComPtr& c) { + Replace(c.ptr); + return *this; + } + + inline ComPtr& operator=(ComPtr&& c) noexcept { + if (&ptr != &c.ptr) { + Kill(); + ptr = c.ptr; + c.ptr = nullptr; + } + + return *this; + } + + template + inline ComPtr& operator=(ComPtr&& c) noexcept { + Kill(); + ptr = c.Detach(); + + return *this; + } + + inline T* Detach() { + T* out = ptr; + ptr = nullptr; + return out; + } + + inline void CopyTo(T** out) { + if (out) { + if (ptr) + ptr->AddRef(); + *out = ptr; + } + } + + inline ULONG Release() { + ULONG ref; + + if (!ptr) + return 0; + ref = ptr->Release(); + ptr = nullptr; + return ref; + } + + inline T** Assign() { + Clear(); + return &ptr; + } + inline void Set(T* p) { + Kill(); + ptr = p; + } + + inline T* Get() const { return ptr; } + + inline T** operator&() { return Assign(); } + + inline operator T*() const { return ptr; } + inline T* operator->() const { return ptr; } + + inline bool operator==(T* p) const { return ptr == p; } + inline bool operator!=(T* p) const { return ptr != p; } + + inline bool operator!() const { return !ptr; } +}; + +struct HRError { + const char* str; + HRESULT hr; + + inline HRError(const char* str, HRESULT hr) : str(str), hr(hr) {} +}; + + +class WinHandle { + HANDLE handle = INVALID_HANDLE_VALUE; + + inline void Clear() { + if (handle && handle != INVALID_HANDLE_VALUE) + CloseHandle(handle); + } + + public: + inline WinHandle() {} + inline WinHandle(HANDLE handle_) : handle(handle_) {} + inline ~WinHandle() { Clear(); } + + inline operator HANDLE() const { return handle; } + + inline WinHandle& operator=(HANDLE handle_) { + if (handle_ != handle) { + Clear(); + handle = handle_; + } + + return *this; + } + + inline HANDLE* operator&() { return &handle; } + + inline bool Valid() const { return handle && handle != INVALID_HANDLE_VALUE; } +}; + +class WinModule { + HMODULE handle = NULL; + + inline void Clear() { + if (handle) + FreeLibrary(handle); + } + + public: + inline WinModule() {} + inline WinModule(HMODULE handle_) : handle(handle_) {} + inline ~WinModule() { Clear(); } + + inline operator HMODULE() const { return handle; } + + inline WinModule& operator=(HMODULE handle_) { + if (handle_ != handle) { + Clear(); + handle = handle_; + } + + return *this; + } + + inline HMODULE* operator&() { return &handle; } + + inline bool Valid() const { return handle != NULL; } +}; + + + +#ifdef _WIN32 +template +class ComQIPtr : public ComPtr { + public: + inline ComQIPtr(IUnknown* unk) { + this->ptr = nullptr; + unk->QueryInterface(__uuidof(T), (void**)&this->ptr); + } + + inline ComPtr& operator=(IUnknown* unk) { + ComPtr::Clear(); + unk->QueryInterface(__uuidof(T), (void**)&this->ptr); + return *this; + } +}; +#endif + + + + +enum class SourceType { + Input, + DeviceOutput, + ProcessOutput, +}; + +enum speaker_layout { + SPEAKERS_UNKNOWN, /**< Unknown setting, fallback is stereo. */ + SPEAKERS_MONO, /**< Channels: MONO */ + SPEAKERS_STEREO, /**< Channels: FL, FR */ + SPEAKERS_2POINT1, /**< Channels: FL, FR, LFE */ + SPEAKERS_4POINT0, /**< Channels: FL, FR, FC, RC */ + SPEAKERS_4POINT1, /**< Channels: FL, FR, FC, LFE, RC */ + SPEAKERS_5POINT1, /**< Channels: FL, FR, FC, LFE, RL, RR */ + SPEAKERS_7POINT1 = 8, /**< Channels: FL, FR, FC, LFE, RL, RR, SL, SR */ +}; + +class WASAPIActivateAudioInterfaceCompletionHandler + : public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags, + Microsoft::WRL::FtmBase, + IActivateAudioInterfaceCompletionHandler> { + IUnknown *unknown; + HRESULT activationResult; + WinHandle activationSignal; + +public: + WASAPIActivateAudioInterfaceCompletionHandler(); + HRESULT GetActivateResult(IAudioClient **client); + +private: + virtual HRESULT STDMETHODCALLTYPE ActivateCompleted( + IActivateAudioInterfaceAsyncOperation *activateOperation) + override final; +}; + +class SystemModule : public SystemModuleInterface { + public: + SystemModule(); + ~SystemModule(); + + // Initialization and terminate. + bool Init(); + int32_t Terminate(); + + // Settings. + int32_t SetRecordingDevice(uint16_t index); + + // enumerate source + std::vector SystemModule::EnumerateSystemSource() const; + + void SetRecordingSource(int id); + + // System source. + rtc::scoped_refptr CreateSource(); + void ResetSource(); + int32_t StopRecording(); + int32_t StartRecording(); + int32_t RecordingChannels(); + + + void SetSystemAudioLevel(float level); + float GetSystemAudioLevel() const; + + void Initialize(); + static void InitFormat(const WAVEFORMATEX* wfex, + speaker_layout& speakers, + int& format, + uint32_t& sampleRate); + static ComPtr InitClient( + DWORD process_id, + PFN_ActivateAudioInterfaceAsync activate_audio_interface_async, + speaker_layout& speakers, + int& format, + uint32_t& samples_per_sec); + + bool ProcessCaptureData(); + static DWORD WINAPI CaptureThread(LPVOID param); + static DWORD WINAPI ReconnectThread(LPVOID param); + static speaker_layout SystemModule::ConvertSpeakerLayout(DWORD layout, + WORD channels); + static ComPtr InitCapture(IAudioClient* client, + HANDLE receiveSignal); + + + ComPtr client; + ComPtr capture; + + WinModule mmdevapi_module; + PFN_ActivateAudioInterfaceAsync activate_audio_interface_async = NULL; + + DWORD process_id = 0; + + bool previouslyFailed = false; + + WinHandle reconnectThread = NULL; + WinHandle captureThread = NULL; + + WinHandle idleSignal; + WinHandle stopSignal; + WinHandle receiveSignal; + WinHandle restartSignal; + WinHandle reconnectExitSignal; + WinHandle exitSignal; + WinHandle initSignal; + + DWORD reconnectDuration = 0; + WinHandle reconnectSignal; + + speaker_layout speakers = SPEAKERS_UNKNOWN; + int format; // byte size of frame. + uint32_t sampleRate; + + bool stop = false; + + float audio_multiplier = 1.0; + std::vector capture_buffer; + std::vector release_capture_buffer = std::vector(480 * 8); +}; \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/bridge.rs b/crates/libwebrtc-sys/src/bridge.rs index dd216b772b..13c3203b90 100644 --- a/crates/libwebrtc-sys/src/bridge.rs +++ b/crates/libwebrtc-sys/src/bridge.rs @@ -142,6 +142,7 @@ pub(crate) mod webrtc { first: String, second: String, } + // TODO: Remove once `cxx` crate allows using pointers to opaque types in // vectors: https://github.com/dtolnay/cxx/issues/741 /// Wrapper for an [`RtpEncodingParameters`] usable in Rust/C++ vectors. @@ -1218,14 +1219,83 @@ pub(crate) mod webrtc { unsafe extern "C++" { pub type AudioDeviceModule; + pub type CustomAudioDeviceModule; pub type AudioLayer; + pub type AudioSourceManager; + pub type AudioSource; + pub type AudioSourceInfo; + + /// Creates a new [`AudioSourceManager`] + /// for the given [`CustomAudioDeviceModule`]. + pub fn create_source_manager( + adm: &CustomAudioDeviceModule, + worker_thread: Pin<&mut Thread>, + ) -> UniquePtr; + + /// Enumerates possible system audio sources. + pub fn enumerate_system_audio_source( + manager: &AudioSourceManager, + ) -> UniquePtr>; + + /// Returns [`AudioSourceInfo`] id. + pub fn system_source_id(manager: &AudioSourceInfo) -> i64; + + /// Returns [`AudioSourceInfo`] title. + pub fn system_source_title( + manager: &AudioSourceInfo, + ) -> UniquePtr; + + /// Sets the volume of the system audio capture. + pub fn set_system_audio_source_volume( + manager: Pin<&mut AudioSourceManager>, + level: f32, + ); + + /// Returns the current volume of the system audio capture. + pub fn system_audio_source_volume(manager: &AudioSourceManager) -> f32; + + /// Sets the system audio source. + pub fn set_system_audio_source( + manager: Pin<&mut AudioSourceManager>, + id: i64, + ); - /// Creates a new [`AudioDeviceModule`] for the given [`AudioLayer`]. - pub fn create_audio_device_module( + /// Creates a new proxied [`AudioDeviceModule`] + /// from the provided [`CustomAudioDeviceModule`]. + pub fn custom_audio_device_module_proxy_upcast( + adm: UniquePtr, + worker_thread: Pin<&mut Thread>, + ) -> UniquePtr; + + /// Creates a new [`AudioSource`] from microphone. + pub fn create_source_microphone( + manager: Pin<&mut AudioSourceManager>, + ) -> UniquePtr; + + /// Creates a new [`AudioSource`] from microphone. + pub fn create_system_audio_source( + manager: Pin<&mut AudioSourceManager>, + ) -> UniquePtr; + + /// Adds [`AudioSource`] to [`AudioSourceManager`]. + pub fn add_source( + manager: Pin<&mut AudioSourceManager>, + source: &AudioSource, + ); + + /// Removes [`AudioSource`] to [`AudioSourceManager`]. + pub fn remove_source( + manager: Pin<&mut AudioSourceManager>, + source: &AudioSource, + ); + + /// Creates a new [`CustomAudioDeviceModule`] + /// for the given [`AudioLayer`]. + pub fn create_custom_audio_device_module( worker_thread: Pin<&mut Thread>, audio_layer: AudioLayer, task_queue_factory: Pin<&mut TaskQueueFactory>, - ) -> UniquePtr; + ) -> UniquePtr; /// Creates a new fake [`AudioDeviceModule`], that will not try to /// access real media devices, but will generate pulsed noise. diff --git a/crates/libwebrtc-sys/src/cpp/adm.cc b/crates/libwebrtc-sys/src/cpp/adm.cc new file mode 100644 index 0000000000..b42e1f162c --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/adm.cc @@ -0,0 +1,208 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include +#include +#include "adm.h" +#include "api/make_ref_counted.h" +#include "common_audio/wav_file.h" +#include "modules/audio_device/include/test_audio_device.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/platform_thread.h" + +// Main initializaton and termination +int32_t CustomAudioDeviceModule::Init() { + if (webrtc::AudioDeviceModuleImpl::Init() != 0) { + return -1; + } + return audio_recorder->Init(); +}; + +int32_t CustomAudioDeviceModule::Terminate() { + quit = true; + return 0; +} + +CustomAudioDeviceModule::~CustomAudioDeviceModule() { + Terminate(); +} + +rtc::scoped_refptr CustomAudioDeviceModule::Create( + AudioLayer audio_layer, + webrtc::TaskQueueFactory* task_queue_factory) { + return CustomAudioDeviceModule::CreateForTest(audio_layer, + task_queue_factory); +} + +int32_t CustomAudioDeviceModule::SetRecordingDevice(uint16_t index) { + return audio_recorder->SetRecordingDevice(index); +} + +int32_t CustomAudioDeviceModule::InitMicrophone() { + return audio_recorder->InitMicrophone(); +} + +bool CustomAudioDeviceModule::MicrophoneIsInitialized() const { + return audio_recorder->MicrophoneIsInitialized(); +} + +rtc::scoped_refptr CustomAudioDeviceModule::CreateSystemSource() { + auto system = system_recorder->CreateSource(); + return system; +} + +std::vector CustomAudioDeviceModule::EnumerateSystemSource() + const { + return system_recorder->EnumerateSystemSource(); +} + +void CustomAudioDeviceModule::SetRecordingSource(int id) { + system_recorder->SetRecordingSource(id); +}; + +rtc::scoped_refptr +CustomAudioDeviceModule::CreateMicrophoneSource() { + auto microphone = audio_recorder->CreateSource(); + return microphone; +} + +void CustomAudioDeviceModule::SetSystemAudioVolume(float level) { + system_recorder->SetSystemAudioLevel(level); +} + +float CustomAudioDeviceModule::GetSystemAudioVolume() const { + return system_recorder->GetSystemAudioLevel(); +} + +void CustomAudioDeviceModule::AddSource( + rtc::scoped_refptr source) { + { + std::unique_lock lock(source_mutex); + sources.push_back(source); + cv.notify_all(); + } + mixer->AddSource(source.get()); +} + +void CustomAudioDeviceModule::RemoveSource( + rtc::scoped_refptr source) { + { + std::unique_lock lock(source_mutex); + for (int i = 0; i < sources.size(); ++i) { + if (sources[i] == source) { + sources.erase(sources.begin() + i); + break; + } + } + } + mixer->RemoveSource(source.get()); +} + +// Microphone mute control +int32_t CustomAudioDeviceModule::MicrophoneVolumeIsAvailable(bool* available) { + return audio_recorder->MicrophoneVolumeIsAvailable(available); +} +int32_t CustomAudioDeviceModule::SetMicrophoneVolume(uint32_t volume) { + return audio_recorder->SetMicrophoneVolume(volume); +} +int32_t CustomAudioDeviceModule::MicrophoneVolume(uint32_t* volume) const { + return audio_recorder->MicrophoneVolume(volume); +} +int32_t CustomAudioDeviceModule::MaxMicrophoneVolume( + uint32_t* maxVolume) const { + return audio_recorder->MaxMicrophoneVolume(maxVolume); +} +int32_t CustomAudioDeviceModule::MinMicrophoneVolume( + uint32_t* minVolume) const { + return audio_recorder->MinMicrophoneVolume(minVolume); +} +int32_t CustomAudioDeviceModule::MicrophoneMuteIsAvailable(bool* available) { + return audio_recorder->MicrophoneMuteIsAvailable(available); +} +int32_t CustomAudioDeviceModule::SetMicrophoneMute(bool enable) { + return audio_recorder->SetMicrophoneMute(enable); +} +int32_t CustomAudioDeviceModule::MicrophoneMute(bool* enabled) const { + return audio_recorder->MicrophoneMute(enabled); +} + +// Audio device module delegates StartRecording to `audio_recorder`. +int32_t CustomAudioDeviceModule::StartRecording() { + return 0; +} + +rtc::scoped_refptr +CustomAudioDeviceModule::CreateForTest( + AudioLayer audio_layer, + webrtc::TaskQueueFactory* task_queue_factory) { + // The "AudioDeviceModule::kWindowsCoreAudio2" audio layer has its own + // dedicated factory method which should be used instead. + if (audio_layer == AudioDeviceModule::kWindowsCoreAudio2) { + return nullptr; + } + + // Create the generic reference counted (platform independent) implementation. + auto audio_device = rtc::make_ref_counted( + audio_layer, task_queue_factory); + + // Ensure that the current platform is supported. + if (audio_device->CheckPlatform() == -1) { + return nullptr; + } + + // Create the platform-dependent implementation. + if (audio_device->CreatePlatformSpecificObjects() == -1) { + return nullptr; + } + + // Ensure that the generic audio buffer can communicate with the platform + // specific parts. + if (audio_device->AttachAudioBuffer() == -1) { + return nullptr; + } + + audio_device->RecordProcess(); + return audio_device; +} + +CustomAudioDeviceModule::CustomAudioDeviceModule( + AudioLayer audio_layer, + webrtc::TaskQueueFactory* task_queue_factory) + : webrtc::AudioDeviceModuleImpl(audio_layer, task_queue_factory), + audio_recorder(new MicrophoneModule()), + system_recorder(new SystemModule()) {} + +void CustomAudioDeviceModule::RecordProcess() { + const auto attributes = + rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime); + ptrThreadRec = rtc::PlatformThread::SpawnJoinable( + [this] { + webrtc::AudioFrame frame; + auto cb = GetAudioDeviceBuffer(); + while (!quit) { + { + std::unique_lock lock(source_mutex); + cv.wait(lock, [&]() { return sources.size() > 0; }); + } + + mixer->Mix(std::max(system_recorder->RecordingChannels(), + audio_recorder->RecordingChannels()), + &frame); + cb->SetRecordingChannels(frame.num_channels()); + cb->SetRecordingSampleRate(frame.sample_rate_hz()); + cb->SetRecordedBuffer(frame.data(), frame.sample_rate_hz() / 100); + cb->DeliverRecordedData(); + } + }, + "audio_device_module_rec_thread", attributes); +} diff --git a/crates/libwebrtc-sys/src/cpp/audio_source_manager_proxy.cc b/crates/libwebrtc-sys/src/cpp/audio_source_manager_proxy.cc new file mode 100644 index 0000000000..77814bc001 --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/audio_source_manager_proxy.cc @@ -0,0 +1,72 @@ + +#include "audio_source_manager_proxy.h" + + AudioSourceManagerProxy::AudioSourceManagerProxy(rtc::Thread* primary_thread, + rtc::scoped_refptr c) + : adm(c), primary_thread_(primary_thread) {} + + std::unique_ptr AudioSourceManagerProxy::Create( + rtc::Thread* primary_thread, + rtc::scoped_refptr c) { + return std::unique_ptr( + new AudioSourceManagerProxy(primary_thread, std::move(c))); + } + + rtc::scoped_refptr AudioSourceManagerProxy::CreateMicrophoneSource() { + TRACE_BOILERPLATE(CreateMicrophoneSource); + webrtc::MethodCall> + call(adm.get(), &AudioSourceManager::CreateMicrophoneSource); + return call.Marshal(primary_thread_); + }; + + rtc::scoped_refptr AudioSourceManagerProxy::CreateSystemSource() { + TRACE_BOILERPLATE(CreateSystemSource); + webrtc::MethodCall> + call(adm.get(), &AudioSourceManager::CreateSystemSource); + return call.Marshal(primary_thread_); + } + + std::vector AudioSourceManagerProxy::EnumerateSystemSource() const { + TRACE_BOILERPLATE(EnumerateSystemSource); + webrtc::ConstMethodCall> + call(adm.get(), &AudioSourceManager::EnumerateSystemSource); + return call.Marshal(primary_thread_); + } + + void AudioSourceManagerProxy::SetSystemAudioVolume(float level) { + TRACE_BOILERPLATE(SetSystemAudioVolume); + webrtc::MethodCall + call(adm.get(), &AudioSourceManager::SetSystemAudioVolume, std::move(level)); + return call.Marshal(primary_thread_); + } + + float AudioSourceManagerProxy::GetSystemAudioVolume() const { + TRACE_BOILERPLATE(GetSystemAudioVolume); + webrtc::ConstMethodCall + call(adm.get(), &AudioSourceManager::GetSystemAudioVolume); + return call.Marshal(primary_thread_); + } + + void AudioSourceManagerProxy::SetRecordingSource(int id) { + TRACE_BOILERPLATE(SetRecordingSource); + webrtc::MethodCall + call(adm.get(), &AudioSourceManager::SetRecordingSource, std::move(id)); + return call.Marshal(primary_thread_); + } + + + void AudioSourceManagerProxy::AddSource(rtc::scoped_refptr source) { + TRACE_BOILERPLATE(AddSource); + webrtc::MethodCall> + call(adm.get(), &AudioSourceManager::AddSource, std::move(source)); + return call.Marshal(primary_thread_); + } + + void AudioSourceManagerProxy::RemoveSource(rtc::scoped_refptr source) { + TRACE_BOILERPLATE(RemoveSource); + webrtc::MethodCall> + call(adm.get(), &AudioSourceManager::RemoveSource, std::move(source)); + return call.Marshal(primary_thread_); + } \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/cpp/bridge.cc b/crates/libwebrtc-sys/src/cpp/bridge.cc index 28674fbaaa..9b53d2be9d 100644 --- a/crates/libwebrtc-sys/src/cpp/bridge.cc +++ b/crates/libwebrtc-sys/src/cpp/bridge.cc @@ -1,7 +1,6 @@ #include #include #include - #include #include @@ -13,7 +12,7 @@ #include "libwebrtc-sys/src/bridge.rs.h" namespace bridge { - + // Creates a new `TrackEventObserver`. TrackEventObserver::TrackEventObserver( rust::Box cb) @@ -127,6 +126,59 @@ std::unique_ptr create_audio_device_module( return std::make_unique(proxied); } +// Creates a new `AudioSourceManager` for the given `CustomAudioDeviceModule`. +std::unique_ptr create_source_manager(const CustomAudioDeviceModule& adm, Thread& worker_thread) { + auto a = AudioSourceManagerProxy::Create(&worker_thread, adm); + return a; +} + +// Creates a new proxied `AudioDeviceModule` from the provided `CustomAudioDeviceModule`. +std::unique_ptr custom_audio_device_module_proxy_upcast(std::unique_ptr adm, Thread& worker_thread) { + + AudioDeviceModule admm = *adm.get(); + AudioDeviceModule proxied = + webrtc::AudioDeviceModuleProxy::Create(&worker_thread, admm); + + return std::make_unique(proxied); +} + +// Creates a new `AudioSource` from microphone. +std::unique_ptr create_source_microphone(AudioSourceManager& manager) { + return std::make_unique(manager.CreateMicrophoneSource()); +} + +// Creates a new `AudioSource` from system. +std::unique_ptr create_system_audio_source(AudioSourceManager& manager) { + return std::make_unique(manager.CreateSystemSource()); +} + +// Adds `AudioSource` to `AudioSourceManager`. +void add_source(AudioSourceManager& manager, const AudioSource& source) { + manager.AddSource(source); +} + +// Removes `AudioSource` from `AudioSourceManager`. +void remove_source(AudioSourceManager& manager, const AudioSource& source) { + manager.RemoveSource(source); +} + +// Creates a new `CustomAudioDeviceModule`. +std::unique_ptr create_custom_audio_device_module( + Thread& worker_thread, + AudioLayer audio_layer, + TaskQueueFactory& task_queue_factory) { + CustomAudioDeviceModule adm = worker_thread.Invoke( + RTC_FROM_HERE, [audio_layer, &task_queue_factory] { + return ::CustomAudioDeviceModule::Create(audio_layer, &task_queue_factory); + }); + + if (adm == nullptr) { + return nullptr; + } + + return std::make_unique(adm); +} + // Calls `AudioDeviceModule->Init()`. int32_t init_audio_device_module(const AudioDeviceModule& audio_device_module) { return audio_device_module->Init(); @@ -329,9 +381,8 @@ std::unique_ptr create_display_video_source( // `AudioOptions`. std::unique_ptr create_audio_source( const PeerConnectionFactoryInterface& peer_connection_factory) { - auto src = - peer_connection_factory->CreateAudioSource(cricket::AudioOptions()); - + + auto src = peer_connection_factory->CreateAudioSource(cricket::AudioOptions()); if (src == nullptr) { return nullptr; } @@ -482,6 +533,7 @@ std::unique_ptr create_peer_connection_factory( const std::unique_ptr& signaling_thread, const std::unique_ptr& default_adm, const std::unique_ptr& ap) { + auto factory = webrtc::CreatePeerConnectionFactory( network_thread.get(), worker_thread.get(), signaling_thread.get(), default_adm ? *default_adm : nullptr, @@ -892,4 +944,35 @@ std::unique_ptr display_source_title(const DisplaySource& source) { return std::make_unique(source.title); } +// Enumerates possible system audio sources. +std::unique_ptr> enumerate_system_audio_source(const AudioSourceManager& manager) { + return std::make_unique>(manager.EnumerateSystemSource()); +} + +// Sets the system audio source. +void set_system_audio_source(AudioSourceManager& manager, int64_t id) { + manager.SetRecordingSource(id); +} + +// Returns `AudioSourceInfo` id. +int64_t system_source_id(const AudioSourceInfo& source) { + return source.GetProcessId(); +} + +// Returns `AudioSourceInfo` title. +std::unique_ptr system_source_title(const AudioSourceInfo& source) { + return std::make_unique(source.GetTitle()); +} + +// Sets the volume of the system audio capture. +void set_system_audio_source_volume(AudioSourceManager& manager, float level) { + manager.SetSystemAudioVolume(level); +} + +// Returns the current volume of the system audio capture. +float system_audio_source_volume(const AudioSourceManager& manager) { + return manager.GetSystemAudioVolume(); +} + + } // namespace bridge diff --git a/crates/libwebrtc-sys/src/cpp/custom_audio.cc b/crates/libwebrtc-sys/src/cpp/custom_audio.cc new file mode 100644 index 0000000000..d77f36fe2f --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/custom_audio.cc @@ -0,0 +1,79 @@ +#include "custom_audio.h" +#include +#include + +// Overwrites `audio_frame`. The data_ field is overwritten with +// 10 ms of new audio (either 1 or 2 interleaved channels) at +// `sample_rate_hz`. All fields in `audio_frame` must be updated. +webrtc::AudioMixer::Source::AudioFrameInfo AudioSource::GetAudioFrameWithInfo( + int sample_rate_hz, + webrtc::AudioFrame* audio_frame) { + std::unique_lock lock(mutex_); + cv_.wait(lock, [&]() { return frame_available_.load() || mute_.load(); }); + if (frame_available_.load()) { + FrameProcessing(sample_rate_hz, audio_frame); + } else { // Mute + auto delta = std::chrono::milliseconds(10) - (std::chrono::system_clock::now() - mute_clock_); + if (cv_.wait_for(lock, delta, [&]() { return frame_available_.load(); })) { + FrameProcessing(sample_rate_hz, audio_frame); + } else { + mute_clock_ = std::chrono::system_clock::now(); + return webrtc::AudioMixer::Source::AudioFrameInfo::kMuted; + } + } + return webrtc::AudioMixer::Source::AudioFrameInfo::kNormal; +}; + +// Prepares an audio frame. +void AudioSource::FrameProcessing(int sample_rate_hz, + webrtc::AudioFrame* audio_frame) { + auto* source = frame_.data(); + if (frame_.sample_rate_hz() != sample_rate_hz) { + render_resampler_.InitializeIfNeeded(frame_.sample_rate_hz(), + sample_rate_hz, frame_.num_channels_); + + render_resampler_.Resample( + frame_.data(), frame_.samples_per_channel_ * frame_.num_channels_, + resample_buffer, webrtc::AudioFrame::kMaxDataSizeSamples); + source = resample_buffer; + } + + audio_frame->UpdateFrame(0, (const int16_t*)source, sample_rate_hz / 100, + sample_rate_hz, + webrtc::AudioFrame::SpeechType::kNormalSpeech, + webrtc::AudioFrame::VADActivity::kVadActive); + + frame_available_.store(false); + mute_.store(false); +} + +// Updates the audio frame data. +void AudioSource::UpdateFrame(const int16_t* source, + int size, + int sample_rate, + int channels) { + frame_.UpdateFrame(0, source, size, sample_rate, + webrtc::AudioFrame::SpeechType::kNormalSpeech, + webrtc::AudioFrame::VADActivity::kVadActive, channels); + mute_.store(false); + frame_available_.store(true); + cv_.notify_all(); +} + +// A way for a mixer implementation to distinguish participants. +int AudioSource::Ssrc() const { + return -1; +}; + +// A way for this source to say that GetAudioFrameWithInfo called +// with this sample rate or higher will not cause quality loss. +int AudioSource::PreferredSampleRate() const { + return frame_.sample_rate_hz(); +}; + +// Mutes the source until the next frame. +void AudioSource::Mute() { + mute_.store(true); + mute_clock_ = std::chrono::system_clock::now(); + cv_.notify_all(); +} \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/cpp/linux_microphone_module.cc b/crates/libwebrtc-sys/src/cpp/linux_microphone_module.cc new file mode 100644 index 0000000000..5ab601b449 --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/linux_microphone_module.cc @@ -0,0 +1,1289 @@ +#if WEBRTC_LINUX + +#include "linux_microphone_module.h" +#include "api/make_ref_counted.h" +#include "rtc_base/logging.h" + +WebRTCPulseSymbolTable* _GetPulseSymbolTable() { + static WebRTCPulseSymbolTable* pulse_symbol_table = + new WebRTCPulseSymbolTable(); + return pulse_symbol_table; +} + +// Accesses Pulse functions through our late-binding symbol table instead of +// directly. This way we don't have to link to libpulse, which means our binary +// will work on systems that don't have it. +#define LATE(sym) \ + LATESYM_GET(webrtc::adm_linux_pulse::PulseAudioSymbolTable, \ + _GetPulseSymbolTable(), sym) + +void MicrophoneModule::PaServerInfoCallback(pa_context* c, + const pa_server_info* i, + void* pThis) { + static_cast(pThis)->PaServerInfoCallbackHandler(i); +} + + + +int MicrophoneSource::sources_num = 0; +MicrophoneSource::MicrophoneSource(MicrophoneModuleInterface* module) { + this->module = module; + if (sources_num == 0) { + module->StartRecording(); + } + ++sources_num; +} + +MicrophoneSource::~MicrophoneSource() { + --sources_num; + if (sources_num == 0) { + module->StopRecording(); + module->ResetSource(); + } +} + +int32_t MicrophoneModule::RecordingChannels() { + return _recChannels; +} + +void MicrophoneModule::ResetSource() { + webrtc::MutexLock lock(&mutex_); + source = nullptr; +} + + +rtc::scoped_refptr MicrophoneModule::CreateSource() { + if (!_recording) { + InitRecording(); + } + + if (!source) { + source = rtc::scoped_refptr(new MicrophoneSource(this)); + } + auto result = source; + source->Release(); + return result; +} + +int32_t MicrophoneModule::SetRecordingDevice(uint16_t index) { + RTC_DCHECK(thread_checker_.IsCurrent()); + auto start = _recIsInitialized; + StopRecording(); + if (_recIsInitialized) { + return -1; + } + + const uint16_t nDevices(RecordingDevices()); + + RTC_LOG(LS_VERBOSE) << "number of availiable input devices is " << nDevices; + + if (index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + _inputDeviceIndex = index; + _inputDeviceIsSpecified = true; + + if (start) { + InitRecording(); + StartRecording(); + } + return 0; +} + + +int32_t MicrophoneModule::ProcessRecordedData(int8_t* bufferData, + uint32_t bufferSizeInSamples, + uint32_t recDelay) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { + // TODO(andrew): this is a temporary hack, to avoid non-causal far- and + // near-end signals at the AEC for PulseAudio. I think the system delay is + // being correctly calculated here, but for legacy reasons we add +10 ms + // to the value in the AEC. The real fix will be part of a larger + // investigation into managing system delay in the AEC. + if (recDelay > 10) + recDelay -= 10; + else + recDelay = 0; + + if (source != nullptr) { + source->UpdateFrame((const int16_t*)bufferData, bufferSizeInSamples, + sample_rate_hz_, _recChannels); + } + + // We have been unlocked - check the flag again. + if (!_recording) { + return -1; + } + + return 0; +} + + + +// ------------------------------------------------------------------------------- +// Everything below has been copied unchanged from audio_device_pulse_linux.cc +// ------------------------------------------------------------------------------- + + + +void MicrophoneModule::PaServerInfoCallbackHandler(const pa_server_info* i) { + // Use PA native sampling rate + sample_rate_hz_ = i->sample_spec.rate; + + // Copy the PA server version + strncpy(_paServerVersion, i->server_version, 31); + _paServerVersion[31] = '\0'; + + if (_recDisplayDeviceName) { + // Copy the source name + strncpy(_recDisplayDeviceName, i->default_source_name, + webrtc::kAdmMaxDeviceNameSize); + _recDisplayDeviceName[webrtc::kAdmMaxDeviceNameSize - 1] = '\0'; + } + + if (_playDisplayDeviceName) { + // Copy the sink name + strncpy(_playDisplayDeviceName, i->default_sink_name, + webrtc::kAdmMaxDeviceNameSize); + _playDisplayDeviceName[webrtc::kAdmMaxDeviceNameSize - 1] = '\0'; + } + + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); +} + +void MicrophoneModule::WaitForOperationCompletion( + pa_operation* paOperation) const { + if (!paOperation) { + return; + } + + while (LATE(pa_operation_get_state)(paOperation) == PA_OPERATION_RUNNING) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + LATE(pa_operation_unref)(paOperation); +} + +int32_t MicrophoneModule::CheckPulseAudioVersion() { + PaLock(); + + pa_operation* paOperation = NULL; + + // get the server info and update deviceName + paOperation = + LATE(pa_context_get_server_info)(_paContext, PaServerInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + // RTC_LOG(LS_VERBOSE) << "checking PulseAudio version: " << _paServerVersion; + + return 0; +} + +int32_t MicrophoneModule::InitPulseAudio() { + int retVal = 0; + + // Load libpulse + if (!_GetPulseSymbolTable()->Load()) { + // Most likely the Pulse library and sound server are not installed on + // this system + return -1; + } + + // Create a mainloop API and connection to the default server + // the mainloop is the internal asynchronous API event loop + if (_paMainloop) { + return -1; + } + _paMainloop = LATE(pa_threaded_mainloop_new)(); + if (!_paMainloop) { + return -1; + } + + // Start the threaded main loop + retVal = LATE(pa_threaded_mainloop_start)(_paMainloop); + if (retVal != PA_OK) { + return -1; + } + + PaLock(); + + _paMainloopApi = LATE(pa_threaded_mainloop_get_api)(_paMainloop); + if (!_paMainloopApi) { + PaUnLock(); + return -1; + } + + // Create a new PulseAudio context + if (_paContext) { + PaUnLock(); + return -1; + } + _paContext = LATE(pa_context_new)(_paMainloopApi, "WEBRTC VoiceEngine"); + + if (!_paContext) { + PaUnLock(); + return -1; + } + + // Set state callback function + LATE(pa_context_set_state_callback)(_paContext, PaContextStateCallback, this); + + // Connect the context to a server (default) + _paStateChanged = false; + retVal = + LATE(pa_context_connect)(_paContext, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL); + + if (retVal != PA_OK) { + PaUnLock(); + return -1; + } + + // Wait for state change + while (!_paStateChanged) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + // Now check to see what final state we reached. + pa_context_state_t state = LATE(pa_context_get_state)(_paContext); + + if (state != PA_CONTEXT_READY) { + if (state == PA_CONTEXT_FAILED) { + } else if (state == PA_CONTEXT_TERMINATED) { + } else { + // Shouldn't happen, because we only signal on one of those three + // states + } + PaUnLock(); + return -1; + } + + PaUnLock(); + + // Give the objects to the mixer manager + _mixerManager.SetPulseAudioObjects(_paMainloop, _paContext); + + // Check the version + if (CheckPulseAudioVersion() < 0) { + return -1; + } + + // Initialize sampling frequency + if (InitSamplingFrequency() < 0 || sample_rate_hz_ == 0) { + return -1; + } + + return 0; +} + +int32_t MicrophoneModule::InitSamplingFrequency() { + PaLock(); + + pa_operation* paOperation = NULL; + + // Get the server info and update sample_rate_hz_ + paOperation = + LATE(pa_context_get_server_info)(_paContext, PaServerInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + return 0; +} + +void MicrophoneModule::PaContextStateCallback(pa_context* c, void* pThis) { + static_cast(pThis)->PaContextStateCallbackHandler(c); +} + +void MicrophoneModule::PaContextStateCallbackHandler(pa_context* c) { + pa_context_state_t state = LATE(pa_context_get_state)(c); + switch (state) { + case PA_CONTEXT_UNCONNECTED: + break; + case PA_CONTEXT_CONNECTING: + case PA_CONTEXT_AUTHORIZING: + case PA_CONTEXT_SETTING_NAME: + break; + case PA_CONTEXT_FAILED: + case PA_CONTEXT_TERMINATED: + _paStateChanged = true; + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + break; + case PA_CONTEXT_READY: + _paStateChanged = true; + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + break; + } +} + +int32_t MicrophoneModule::TerminatePulseAudio() { + // Do nothing if the instance doesn't exist + // likely GetPulseSymbolTable.Load() fails + if (!_paMainloop) { + return 0; + } + + PaLock(); + + // Disconnect the context + if (_paContext) { + LATE(pa_context_disconnect)(_paContext); + } + + // Unreference the context + if (_paContext) { + LATE(pa_context_unref)(_paContext); + } + + PaUnLock(); + _paContext = NULL; + + // Stop the threaded main loop + if (_paMainloop) { + LATE(pa_threaded_mainloop_stop)(_paMainloop); + } + + // Free the mainloop + if (_paMainloop) { + LATE(pa_threaded_mainloop_free)(_paMainloop); + } + + _paMainloop = NULL; + + return 0; +} + +int32_t MicrophoneModule::Init() { + RTC_DCHECK(thread_checker_.IsCurrent()); + if (_initialized) { + return 0; + } + + // Initialize PulseAudio + if (InitPulseAudio() < 0) { + RTC_LOG(LS_ERROR) << "failed to initialize PulseAudio"; + if (TerminatePulseAudio() < 0) { + RTC_LOG(LS_ERROR) << "failed to terminate PulseAudio"; + } + return -1; + } + +#if defined(WEBRTC_USE_X11) + // Get X display handle for typing detection + _XDisplay = XOpenDisplay(NULL); + if (!_XDisplay) { + // RTC_LOG(LS_WARNING) + // << "failed to open X display, typing detection will not work"; + } +#endif + + // RECORDING + const auto attributes = + rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime); + _ptrThreadRec = rtc::PlatformThread::SpawnJoinable( + [this] { + while (RecThreadProcess()) { + } + }, + "webrtc_audio_module_rec_thread", attributes); + + _initialized = true; + + return 0; +} + +int32_t MicrophoneModule::Terminate() { + RTC_DCHECK(thread_checker_.IsCurrent()); + if (!_initialized) { + return 0; + } + + { + webrtc::MutexLock lock(&mutex_); + quit_ = true; + } + _mixerManager.Close(); + + // RECORDING + _timeEventRec.Set(); + _ptrThreadRec.Finalize(); + + // Terminate PulseAudio + if (TerminatePulseAudio() < 0) { + // RTC_LOG(LS_ERROR) << "failed to terminate PulseAudio"; + return -1; + } + +#if defined(WEBRTC_USE_X11) + if (_XDisplay) { + XCloseDisplay(_XDisplay); + _XDisplay = NULL; + } +#endif + + _initialized = false; + _inputDeviceIsSpecified = false; + + return 0; +} + +MicrophoneModule::MicrophoneModule() { +#if defined(WEBRTC_USE_X11) + memset(_oldKeyState, 0, sizeof(_oldKeyState)); +#endif +} + +MicrophoneModule::~MicrophoneModule() { + Terminate(); + if (_recBuffer) { + delete[] _recBuffer; + _recBuffer = NULL; + } + if (_playDeviceName) { + delete[] _playDeviceName; + _playDeviceName = NULL; + } + if (_recDeviceName) { + delete[] _recDeviceName; + _recDeviceName = NULL; + } +} + +void MicrophoneModule::PaSourceInfoCallback(pa_context* c, + const pa_source_info* i, + int eol, + void* pThis) { + static_cast(pThis)->PaSourceInfoCallbackHandler(i, eol); +} + +void MicrophoneModule::PaSourceInfoCallbackHandler(const pa_source_info* i, + int eol) { + if (eol) { + // Signal that we are done + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); + return; + } + + // We don't want to list output devices + if (i->monitor_of_sink == PA_INVALID_INDEX) { + if (_numRecDevices == _deviceIndex) { + // Convert the device index to the one of the source + _paDeviceIndex = i->index; + + if (_recDeviceName) { + // copy the source name + strncpy(_recDeviceName, i->name, webrtc::kAdmMaxDeviceNameSize); + _recDeviceName[webrtc::kAdmMaxDeviceNameSize - 1] = '\0'; + } + if (_recDisplayDeviceName) { + // Copy the source display name + strncpy(_recDisplayDeviceName, i->description, + webrtc::kAdmMaxDeviceNameSize); + _recDisplayDeviceName[webrtc::kAdmMaxDeviceNameSize - 1] = '\0'; + } + } + + _numRecDevices++; + } +} + +void MicrophoneModule::PaLock() { + LATE(pa_threaded_mainloop_lock)(_paMainloop); +} + +int16_t MicrophoneModule::RecordingDevices() { + PaLock(); + + pa_operation* paOperation = NULL; + _numRecDevices = 1; // Init to 1 to account for "default" + + // Get the whole list of devices and update _numRecDevices + paOperation = LATE(pa_context_get_source_info_list)( + _paContext, PaSourceInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + return _numRecDevices; +} + +void MicrophoneModule::PaUnLock() { + LATE(pa_threaded_mainloop_unlock)(_paMainloop); +} + + +int32_t MicrophoneModule::StopRecording() { + RTC_DCHECK(thread_checker_.IsCurrent()); + webrtc::MutexLock lock(&mutex_); + + if (!_recIsInitialized) { + return 0; + } + + if (_recStream == NULL) { + return -1; + } + + _recIsInitialized = false; + _recording = false; + + RTC_LOG(LS_VERBOSE) << "stopping recording"; + + // Stop Recording + PaLock(); + + DisableReadCallback(); + LATE(pa_stream_set_overflow_callback)(_recStream, NULL, NULL); + + // Unset this here so that we don't get a TERMINATED callback + LATE(pa_stream_set_state_callback)(_recStream, NULL, NULL); + + if (LATE(pa_stream_get_state)(_recStream) != PA_STREAM_UNCONNECTED) { + // Disconnect the stream + if (LATE(pa_stream_disconnect)(_recStream) != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to disconnect rec stream, err=" + << LATE(pa_context_errno)(_paContext); + PaUnLock(); + return -1; + } + + RTC_LOG(LS_VERBOSE) << "disconnected recording"; + } + + LATE(pa_stream_unref)(_recStream); + _recStream = NULL; + + PaUnLock(); + + // Provide the recStream to the mixer + _mixerManager.SetRecStream(_recStream); + + if (_recBuffer) { + delete[] _recBuffer; + _recBuffer = NULL; + } + + return 0; +} + +int32_t MicrophoneModule::StartRecording() { + RTC_DCHECK(thread_checker_.IsCurrent()); + + if (!_recIsInitialized) { + return -1; + } + + if (_recording) { + return 0; + } + + // Set state to ensure that the recording starts from the audio thread. + _startRec = true; + + // The audio thread will signal when recording has started. + _timeEventRec.Set(); + if (!_recStartEvent.Wait(10000)) { + { + webrtc::MutexLock lock(&mutex_); + _startRec = false; + } + StopRecording(); + RTC_LOG(LS_ERROR) << "failed to activate recording"; + return -1; + } + + { + webrtc::MutexLock lock(&mutex_); + if (_recording) { + // The recording state is set by the audio thread after recording + // has started. + } else { + RTC_LOG(LS_ERROR) << "failed to activate recording"; + return -1; + } + } + + return 0; +} + +int32_t MicrophoneModule::GetDefaultDeviceInfo(bool recDevice, + char* name, + uint16_t& index) { + char tmpName[webrtc::kAdmMaxDeviceNameSize] = {0}; + // subtract length of "default: " + uint16_t nameLen = webrtc::kAdmMaxDeviceNameSize - 9; + char* pName = NULL; + + if (name) { + // Add "default: " + strcpy(name, "default: "); + pName = &name[9]; + } + + // Tell the callback that we want + // the name for this device + if (recDevice) { + _recDisplayDeviceName = tmpName; + } else { + _playDisplayDeviceName = tmpName; + } + + // Set members + _paDeviceIndex = -1; + _deviceIndex = 0; + _numPlayDevices = 0; + _numRecDevices = 0; + + PaLock(); + + pa_operation* paOperation = NULL; + + // Get the server info and update deviceName + paOperation = + LATE(pa_context_get_server_info)(_paContext, PaServerInfoCallback, this); + + WaitForOperationCompletion(paOperation); + + // Get the device index + if (recDevice) { + paOperation = LATE(pa_context_get_source_info_by_name)( + _paContext, (char*)tmpName, PaSourceInfoCallback, this); + } + + WaitForOperationCompletion(paOperation); + + PaUnLock(); + + // Set the index + index = _paDeviceIndex; + + if (name) { + // Copy to name string + strncpy(pName, tmpName, nameLen); + } + + // Clear members + _playDisplayDeviceName = NULL; + _recDisplayDeviceName = NULL; + _paDeviceIndex = -1; + _deviceIndex = -1; + _numPlayDevices = 0; + _numRecDevices = 0; + + return 0; +} + +int32_t MicrophoneModule::InitMicrophone() { + if (_recording) { + return -1; + } + + if (!_inputDeviceIsSpecified) { + return -1; + } + + // Check if default device + if (_inputDeviceIndex == 0) { + uint16_t deviceIndex = 0; + GetDefaultDeviceInfo(true, NULL, deviceIndex); + _paDeviceIndex = deviceIndex; + } else { + // Get the PA device index from + // the callback + _deviceIndex = _inputDeviceIndex; + + // get recording devices + RecordingDevices(); + } + + // The callback has now set the _paDeviceIndex to + // the PulseAudio index of the device + if (_mixerManager.OpenMicrophone(_paDeviceIndex) == -1) { + return -1; + } + + // Clear _deviceIndex + _deviceIndex = -1; + _paDeviceIndex = -1; + + return 0; +} + +int32_t MicrophoneModule::InitRecording() { + RTC_DCHECK(thread_checker_.IsCurrent()); + + if (_recording) { + return -1; + } + + if (!_inputDeviceIsSpecified) { + return -1; + } + + if (_recIsInitialized) { + return 0; + } + + // Initialize the microphone (devices might have been added or removed) + if (InitMicrophone() == -1) { + } + + // Set the rec sample specification + pa_sample_spec recSampleSpec; + recSampleSpec.channels = _recChannels; + recSampleSpec.format = PA_SAMPLE_S16LE; + recSampleSpec.rate = sample_rate_hz_; + + // Create a new rec stream + _recStream = + LATE(pa_stream_new)(_paContext, "recStream", &recSampleSpec, NULL); + if (!_recStream) { + return -1; + } + + // Provide the recStream to the mixer + _mixerManager.SetRecStream(_recStream); + + if (_configuredLatencyRec != WEBRTC_PA_NO_LATENCY_REQUIREMENTS) { + _recStreamFlags = (pa_stream_flags_t)(PA_STREAM_AUTO_TIMING_UPDATE | + PA_STREAM_INTERPOLATE_TIMING); + + // If configuring a specific latency then we want to specify + // PA_STREAM_ADJUST_LATENCY to make the server adjust parameters + // automatically to reach that target latency. However, that flag + // doesn't exist in Ubuntu 8.04 and many people still use that, + // so we have to check the protocol version of libpulse. + if (LATE(pa_context_get_protocol_version)(_paContext) >= + WEBRTC_PA_ADJUST_LATENCY_PROTOCOL_VERSION) { + _recStreamFlags |= PA_STREAM_ADJUST_LATENCY; + } + + const pa_sample_spec* spec = LATE(pa_stream_get_sample_spec)(_recStream); + if (!spec) { + RTC_LOG(LS_ERROR) << "pa_stream_get_sample_spec(rec)"; + return -1; + } + + size_t bytesPerSec = LATE(pa_bytes_per_second)(spec); + uint32_t latency = bytesPerSec * WEBRTC_PA_LOW_CAPTURE_LATENCY_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + // Set the rec buffer attributes + // Note: fragsize specifies a maximum transfer size, not a minimum, so + // it is not possible to force a high latency setting, only a low one. + _recBufferAttr.fragsize = latency; // size of fragment + _recBufferAttr.maxlength = + latency + bytesPerSec * WEBRTC_PA_CAPTURE_BUFFER_EXTRA_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + _configuredLatencyRec = latency; + } + + _recordBufferSize = sample_rate_hz_ / 100 * 2 * _recChannels; + _recordBufferUsed = 0; + _recBuffer = new int8_t[_recordBufferSize]; + + // Enable overflow callback + LATE(pa_stream_set_overflow_callback) + (_recStream, PaStreamOverflowCallback, this); + + // Set the state callback function for the stream + LATE(pa_stream_set_state_callback)(_recStream, PaStreamStateCallback, this); + + // Mark recording side as initialized + _recIsInitialized = true; + + return 0; +} + +void MicrophoneModule::PaStreamOverflowCallback(pa_stream* /*unused*/, + void* pThis) { + static_cast(pThis)->PaStreamOverflowCallbackHandler(); +} + +void MicrophoneModule::PaStreamOverflowCallbackHandler() { + RTC_LOG(LS_WARNING) << "Recording overflow"; +} + +void MicrophoneModule::PaStreamStateCallbackHandler(pa_stream* p) { + RTC_LOG(LS_VERBOSE) << "stream state cb"; + + pa_stream_state_t state = LATE(pa_stream_get_state)(p); + switch (state) { + case PA_STREAM_UNCONNECTED: + RTC_LOG(LS_VERBOSE) << "unconnected"; + break; + case PA_STREAM_CREATING: + RTC_LOG(LS_VERBOSE) << "creating"; + break; + case PA_STREAM_FAILED: + case PA_STREAM_TERMINATED: + RTC_LOG(LS_VERBOSE) << "failed"; + break; + case PA_STREAM_READY: + RTC_LOG(LS_VERBOSE) << "ready"; + break; + } + + LATE(pa_threaded_mainloop_signal)(_paMainloop, 0); +} + +void MicrophoneModule::PaStreamStateCallback(pa_stream* p, void* pThis) { + static_cast(pThis)->PaStreamStateCallbackHandler(p); +} + +bool MicrophoneModule::RecThreadProcess() { + if (!_timeEventRec.Wait(1000)) { + return true; + } + + webrtc::MutexLock lock(&mutex_); + if (quit_) { + return false; + } + if (_startRec) { + RTC_LOG(LS_VERBOSE) << "_startRec true, performing initial actions"; + + _recDeviceName = NULL; + + // Set if not default device + if (_inputDeviceIndex > 0) { + // Get the recording device name + _recDeviceName = new char[webrtc::kAdmMaxDeviceNameSize]; + _deviceIndex = _inputDeviceIndex; + RecordingDevices(); + } + + PaLock(); + + RTC_LOG(LS_VERBOSE) << "connecting stream"; + + // Connect the stream to a source + if (LATE(pa_stream_connect_record)( + _recStream, _recDeviceName, &_recBufferAttr, + (pa_stream_flags_t)_recStreamFlags) != PA_OK) { + RTC_LOG(LS_ERROR) << "failed to connect rec stream, err=" + << LATE(pa_context_errno)(_paContext); + } + + RTC_LOG(LS_VERBOSE) << "connected"; + + // Wait for state change + while (LATE(pa_stream_get_state)(_recStream) != PA_STREAM_READY) { + LATE(pa_threaded_mainloop_wait)(_paMainloop); + } + + RTC_LOG(LS_VERBOSE) << "done"; + + // We can now handle read callbacks + EnableReadCallback(); + + PaUnLock(); + + // Clear device name + if (_recDeviceName) { + delete[] _recDeviceName; + _recDeviceName = NULL; + } + + _startRec = false; + _recording = true; + _recStartEvent.Set(); + + return true; + } + + if (_recording) { + // Read data and provide it to VoiceEngine + if (ReadRecordedData(_tempSampleData, _tempSampleDataSize) == -1) { + return true; + } + + _tempSampleData = NULL; + _tempSampleDataSize = 0; + + PaLock(); + while (true) { + // Ack the last thing we read + if (LATE(pa_stream_drop)(_recStream) != 0) { + RTC_LOG(LS_WARNING) + << "failed to drop, err=" << LATE(pa_context_errno)(_paContext); + } + + if (LATE(pa_stream_readable_size)(_recStream) <= 0) { + // Then that was all the data + break; + } + + // Else more data. + const void* sampleData; + size_t sampleDataSize; + + if (LATE(pa_stream_peek)(_recStream, &sampleData, &sampleDataSize) != 0) { + RTC_LOG(LS_ERROR) << "RECORD_ERROR, error = " + << LATE(pa_context_errno)(_paContext); + break; + } + + // Drop lock for sigslot dispatch, which could take a while. + PaUnLock(); + // Read data and provide it to VoiceEngine + if (ReadRecordedData(sampleData, sampleDataSize) == -1) { + return true; + } + PaLock(); + + // Return to top of loop for the ack and the check for more data. + } + + EnableReadCallback(); + PaUnLock(); + + } // _recording + + return true; +} + +int32_t MicrophoneModule::LatencyUsecs(pa_stream* stream) { + if (!WEBRTC_PA_REPORT_LATENCY) { + return 0; + } + + if (!stream) { + return 0; + } + + pa_usec_t latency; + int negative; + if (LATE(pa_stream_get_latency)(stream, &latency, &negative) != 0) { + RTC_LOG(LS_ERROR) << "Can't query latency"; + // We'd rather continue playout/capture with an incorrect delay than + // stop it altogether, so return a valid value. + return 0; + } + + if (negative) { + RTC_LOG(LS_VERBOSE) + << "warning: pa_stream_get_latency reported negative delay"; + + // The delay can be negative for monitoring streams if the captured + // samples haven't been played yet. In such a case, "latency" + // contains the magnitude, so we must negate it to get the real value. + int32_t tmpLatency = (int32_t)-latency; + if (tmpLatency < 0) { + // Make sure that we don't use a negative delay. + tmpLatency = 0; + } + + return tmpLatency; + } else { + return (int32_t)latency; + } +} + +int32_t MicrophoneModule::ReadRecordedData(const void* bufferData, + size_t bufferSize) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { + size_t size = bufferSize; + uint32_t numRecSamples = _recordBufferSize / (2 * _recChannels); + + // Account for the peeked data and the used data. + uint32_t recDelay = + (uint32_t)((LatencyUsecs(_recStream) / 1000) + + 10 * ((size + _recordBufferUsed) / _recordBufferSize)); + + if (_playStream) { + // Get the playout delay. + _sndCardPlayDelay = (uint32_t)(LatencyUsecs(_playStream) / 1000); + } + + if (_recordBufferUsed > 0) { + // Have to copy to the buffer until it is full. + size_t copy = _recordBufferSize - _recordBufferUsed; + if (size < copy) { + copy = size; + } + + memcpy(&_recBuffer[_recordBufferUsed], bufferData, copy); + _recordBufferUsed += copy; + bufferData = static_cast(bufferData) + copy; + size -= copy; + + if (_recordBufferUsed != _recordBufferSize) { + // Not enough data yet to pass to VoE. + return 0; + } + + // nado=_recBuffer; + + // Provide data to VoiceEngine. + if (ProcessRecordedData(_recBuffer, numRecSamples, recDelay) == -1) { + // We have stopped recording. + return -1; + } + + _recordBufferUsed = 0; + } + + // Now process full 10ms sample sets directly from the input. + while (size >= _recordBufferSize) { + // nado=(int8_t*)bufferData; + + // Provide data to VoiceEngine. + if (ProcessRecordedData(static_cast(const_cast(bufferData)), + numRecSamples, recDelay) == -1) { + // We have stopped recording. + return -1; + } + + bufferData = static_cast(bufferData) + _recordBufferSize; + size -= _recordBufferSize; + + // We have consumed 10ms of data. + recDelay -= 10; + } + + // Now save any leftovers for later. + if (size > 0) { + memcpy(_recBuffer, bufferData, size); + _recordBufferUsed = size; + } + + return 0; +} + +int32_t MicrophoneModule::StereoRecordingIsAvailable(bool& available) { + RTC_DCHECK(thread_checker_.IsCurrent()); + if (_recChannels == 2 && _recording) { + available = true; + return 0; + } + + available = false; + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + int error = 0; + + if (!wasInitialized && InitMicrophone() == -1) { + // Cannot open the specified device + available = false; + return 0; + } + + // Check if the selected microphone can record stereo. + bool isAvailable(false); + + error = _mixerManager.StereoRecordingIsAvailable(isAvailable); + + if (!error) + available = isAvailable; + + // Close the initialized input mixer + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return error; +} + +int32_t MicrophoneModule::SetStereoRecording(bool enable) { + RTC_DCHECK(thread_checker_.IsCurrent()); + if (enable) + _recChannels = 2; + else + _recChannels = 1; + + return 0; +} + +int32_t MicrophoneModule::StereoRecording(bool& enabled) const { + RTC_DCHECK(thread_checker_.IsCurrent()); + if (_recChannels == 2) + enabled = true; + else + enabled = false; + + return 0; +} + + +bool MicrophoneModule::KeyPressed() const { +#if defined(WEBRTC_USE_X11) + char szKey[32]; + unsigned int i = 0; + char state = 0; + + if (!_XDisplay) + return false; + + // Check key map status + XQueryKeymap(_XDisplay, szKey); + + // A bit change in keymap means a key is pressed + for (i = 0; i < sizeof(szKey); i++) + state |= (szKey[i] ^ _oldKeyState[i]) & szKey[i]; + + // Save old state + memcpy((char*)_oldKeyState, (char*)szKey, sizeof(_oldKeyState)); + return (state != 0); +#else + return false; +#endif +} + +void MicrophoneModule::EnableReadCallback() { + LATE(pa_stream_set_read_callback)(_recStream, &PaStreamReadCallback, this); +} + +void MicrophoneModule::DisableReadCallback() { + LATE(pa_stream_set_read_callback)(_recStream, NULL, NULL); +} + +void MicrophoneModule::PaStreamReadCallback(pa_stream* a /*unused1*/, + size_t b /*unused2*/, + void* pThis) { + static_cast(pThis)->PaStreamReadCallbackHandler(); +} + +void MicrophoneModule::PaStreamReadCallbackHandler() { + // We get the data pointer and size now in order to save one Lock/Unlock + // in the worker thread. + if (LATE(pa_stream_peek)(_recStream, &_tempSampleData, + &_tempSampleDataSize) != 0) { + RTC_LOG(LS_ERROR) << "Can't read data!"; + return; + } + + // Since we consume the data asynchronously on a different thread, we have + // to temporarily disable the read callback or else Pulse will call it + // continuously until we consume the data. We re-enable it below. + DisableReadCallback(); + _timeEventRec.Set(); +} + +bool MicrophoneModule::MicrophoneIsInitialized() { + RTC_DCHECK(thread_checker_.IsCurrent()); + return (_mixerManager.MicrophoneIsInitialized()); +}; + +// Microphone volume controls +int32_t MicrophoneModule::MicrophoneVolumeIsAvailable(bool* available) { + RTC_DCHECK(thread_checker_.IsCurrent()); + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected output device. + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no + // volume control. + *available = false; + return 0; + } + + // Given that InitMicrophone was successful, we know that a volume control + // exists. + *available = true; + + // Close the initialized input mixer + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t MicrophoneModule::SetMicrophoneVolume(uint32_t volume) { + return (_mixerManager.SetMicrophoneVolume(volume)); +} + +int32_t MicrophoneModule::MicrophoneVolume(uint32_t* volume) const { + uint32_t level(0); + + if (_mixerManager.MicrophoneVolume(level) == -1) { + RTC_LOG(LS_WARNING) << "failed to retrieve current microphone level"; + return -1; + } + + *volume = level; + + return 0; +} + +int32_t MicrophoneModule::MaxMicrophoneVolume(uint32_t* maxVolume) const { + uint32_t maxVol(0); + + if (_mixerManager.MaxMicrophoneVolume(maxVol) == -1) { + return -1; + } + + *maxVolume = maxVol; + + return 0; +} + +int32_t MicrophoneModule::MinMicrophoneVolume(uint32_t* minVolume) const { + uint32_t minVol(0); + + if (_mixerManager.MinMicrophoneVolume(minVol) == -1) { + return -1; + } + + *minVolume = minVol; + + return 0; +} + +// Microphone mute control +int32_t MicrophoneModule::MicrophoneMuteIsAvailable(bool* available) { + RTC_DCHECK(thread_checker_.IsCurrent()); + bool isAvailable(false); + bool wasInitialized = _mixerManager.MicrophoneIsInitialized(); + + // Make an attempt to open up the + // input mixer corresponding to the currently selected input device. + // + if (!wasInitialized && InitMicrophone() == -1) { + // If we end up here it means that the selected microphone has no + // volume control, hence it is safe to state that there is no + // boost control already at this stage. + *available = false; + return 0; + } + + // Check if the selected microphone has a mute control + // + _mixerManager.MicrophoneMuteIsAvailable(isAvailable); + *available = isAvailable; + + // Close the initialized input mixer + // + if (!wasInitialized) { + _mixerManager.CloseMicrophone(); + } + + return 0; +} + +int32_t MicrophoneModule::SetMicrophoneMute(bool enable) { + RTC_DCHECK(thread_checker_.IsCurrent()); + return (_mixerManager.SetMicrophoneMute(enable)); +} + +int32_t MicrophoneModule::MicrophoneMute(bool* enabled) const { + RTC_DCHECK(thread_checker_.IsCurrent()); + bool muted(0); + if (_mixerManager.MicrophoneMute(muted) == -1) { + return -1; + } + + *enabled = muted; + return 0; +} + +#endif \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/cpp/linux_system_audio_module.cc b/crates/libwebrtc-sys/src/cpp/linux_system_audio_module.cc new file mode 100644 index 0000000000..116a8cc896 --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/linux_system_audio_module.cc @@ -0,0 +1,799 @@ +#if WEBRTC_LINUX + +#include "linux_system_audio_module.h" +#include +const int32_t WEBRTC_PA_NO_LATENCY_REQUIREMENTS = -1; +const uint32_t WEBRTC_PA_ADJUST_LATENCY_PROTOCOL_VERSION = 13; +const uint32_t WEBRTC_PA_LOW_CAPTURE_LATENCY_MSECS = 10; +const uint32_t WEBRTC_PA_MSECS_PER_SEC = 1000; +const uint32_t WEBRTC_PA_CAPTURE_BUFFER_EXTRA_MSECS = 750; + + +int SystemSource::sources_num = 0; +SystemSource::SystemSource(SystemModuleInterface* module) { + this->module = module; + if (sources_num == 0) { + module->StartRecording(); + } + ++sources_num; +} + +SystemSource::~SystemSource() { + --sources_num; + if (sources_num == 0) { + module->StopRecording(); + module->ResetSource(); + } +} + + +static void subscribe_result(pa_context *c, int success, void *userdata) { + auto thisP = (SystemModule*) userdata; + pa_threaded_mainloop_signal(thisP->_paMainloop, 0); +} + +static void subscribe_sink_event(pa_context* c, + pa_subscription_event_type_t t, + uint32_t idx, + void* userdata) { + auto thisP = (SystemModule*)userdata; + if (idx == thisP->_recSinkId) { + if (t & pa_subscription_event_type_t::PA_SUBSCRIPTION_EVENT_REMOVE) { + thisP->_mute = true; + } + } else if (t == + pa_subscription_event_type_t::PA_SUBSCRIPTION_EVENT_SINK_INPUT) { + thisP->_on_new = true; + } +} + +struct SinkData { + std::vector* vec; + pa_threaded_mainloop* pa_mainloop; +}; + +static void subscribe_sink_cb(pa_context* paContext, + pa_sink_input_info* i, + int eol, + void* userdata) { + auto data = (SinkData*)userdata; + if (eol > 0) { + pa_threaded_mainloop_signal(data->pa_mainloop, 0); + return; + } + + if (i != nullptr) { + AudioSourceInfo info( + i->index, + std::string(pa_proplist_gets(i->proplist, "application.name")) + ": " + + std::string(i->name), + atoi(pa_proplist_gets(i->proplist, "application.process.id")), + i->sample_spec.rate, i->sample_spec.channels); + data->vec->push_back(info); + } +} + +// Initialization and terminate. +bool SystemModule::Init() { + if (_initialized) { + return 0; + } + + // Initialize PulseAudio + if (InitPulseAudio() < 0) { + TerminatePulseAudio(); + return -1; + } + + PaLock(); + + pa_operation* paOperation = NULL; + + pa_context_set_subscribe_callback(_paContext, subscribe_sink_event, this); + pa_subscription_mask_t mask; + pa_operation* o = pa_context_subscribe( + _paContext, pa_subscription_mask_t::PA_SUBSCRIPTION_MASK_SINK_INPUT, + subscribe_result, this); + + WaitForOperationCompletion(o); + + PaUnLock(); + + const auto attributes = + rtc::ThreadAttributes().SetPriority(rtc::ThreadPriority::kRealtime); + _ptrThreadRec = rtc::PlatformThread::SpawnJoinable( + [this] { + while (RecThreadProcess()) { + } + }, + "webrtc_system_audio_module_rec_thread", attributes); + + _initialized = true; + + return 0; +} + +SystemModule::SystemModule() { + Init(); +} +SystemModule::~SystemModule() { + Terminate(); +} + +// Unlocks pulse audio. +void SystemModule::PaUnLock() { + pa_threaded_mainloop_unlock(_paMainloop); +} + +// Locks pulse audio. +void SystemModule::PaLock() { + pa_threaded_mainloop_lock(_paMainloop); +} + +// Redirect callback to `this.PaContextStateCallbackHandler`. +void SystemModule::PaContextStateCallback(pa_context* c, void* pThis) { + static_cast(pThis)->PaContextStateCallbackHandler(c); +} + +// Callback to keep track of the state of the pulse audio context. +void SystemModule::PaContextStateCallbackHandler(pa_context* c) { + pa_context_state_t state = pa_context_get_state(c); + switch (state) { + case PA_CONTEXT_UNCONNECTED: + break; + case PA_CONTEXT_CONNECTING: + case PA_CONTEXT_AUTHORIZING: + case PA_CONTEXT_SETTING_NAME: + break; + case PA_CONTEXT_FAILED: + case PA_CONTEXT_TERMINATED: + _paStateChanged = true; + pa_threaded_mainloop_signal(_paMainloop, 0); + break; + case PA_CONTEXT_READY: + _paStateChanged = true; + pa_threaded_mainloop_signal(_paMainloop, 0); + break; + } +} + +// Terminates a pulse audio. +int32_t SystemModule::TerminatePulseAudio() { + // Do nothing if the instance doesn't exist + // likely GetPulseSymbolTable.Load() fails + if (!_paMainloop) { + return 0; + } + + PaLock(); + + // Disconnect the context + if (_paContext) { + pa_context_disconnect(_paContext); + } + + // Unreference the context + if (_paContext) { + pa_context_unref(_paContext); + } + + PaUnLock(); + _paContext = NULL; + + // Stop the threaded main loop + if (_paMainloop) { + pa_threaded_mainloop_stop(_paMainloop); + } + + // Free the mainloop + if (_paMainloop) { + pa_threaded_mainloop_free(_paMainloop); + } + + _paMainloop = NULL; + + return 0; +} + +// Initiates a pulse audio. +int32_t SystemModule::InitPulseAudio() { + int retVal = 0; + + // Create a mainloop API and connection to the default server + // the mainloop is the internal asynchronous API event loop + if (_paMainloop) { + return -1; + } + _paMainloop = pa_threaded_mainloop_new(); + if (!_paMainloop) { + return -1; + } + + // Start the threaded main loop + retVal = pa_threaded_mainloop_start(_paMainloop); + if (retVal != PA_OK) { + return -1; + } + + PaLock(); + + _paMainloopApi = pa_threaded_mainloop_get_api(_paMainloop); + if (!_paMainloopApi) { + PaUnLock(); + return -1; + } + + // Create a new PulseAudio context + if (_paContext) { + PaUnLock(); + return -1; + } + _paContext = pa_context_new(_paMainloopApi, "WEBRTC SystemAudio"); + + if (!_paContext) { + PaUnLock(); + return -1; + } + + // Set state callback function + pa_context_set_state_callback(_paContext, PaContextStateCallback, this); + + // Connect the context to a server (default) + _paStateChanged = false; + retVal = pa_context_connect(_paContext, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL); + + if (retVal != PA_OK) { + PaUnLock(); + return -1; + } + + // Wait for state change + while (!_paStateChanged) { + pa_threaded_mainloop_wait(_paMainloop); + } + + // Now check to see what final state we reached. + pa_context_state_t state = pa_context_get_state(_paContext); + + if (state != PA_CONTEXT_READY) { + PaUnLock(); + return -1; + } + + PaUnLock(); + + return 0; +} + +// Returns the current sinc id for the current process. +int32_t SystemModule::UpdateSinkId() { + auto audio_sources = EnumerateSystemSource(); + + auto sinkId = -1; + for (int i = 0; i < audio_sources.size(); ++i) { + if (audio_sources[i].GetProcessId() == _recProcessId) { + sinkId = audio_sources[i].GetId(); + } + } + + return sinkId; +} + +// Function to capture audio in another thread. +bool SystemModule::RecThreadProcess() { + if (_mute) { + _mute = false; + + StopRecording(); + if (source != nullptr) { + source->Mute(); + } + _recSinkId = -1; + return true; + } + + if (_on_new) { + _on_new = false; + + if (_recSinkId == -1) { + + auto recSinkId = UpdateSinkId(); + if (_recSinkId != recSinkId && recSinkId > 0) { + _recSinkId = recSinkId; + StopRecording(); + InitRecording(); + _startRec = true; + _timeEventRec.Set(); + return true; + } + } + } + + if (!_timeEventRec.Wait(1000)) { + return true; + } + + webrtc::MutexLock lock(&mutex_); + if (quit_) { + return false; + } + + if (_startRec && _recProcessId != -1) { + // get sink id + _recSinkId = UpdateSinkId(); + if (_recSinkId < 0) { + return true; + } + + PaLock(); + + pa_stream_set_monitor_stream(_recStream, _recSinkId); + + // Connect the stream to a source + pa_stream_connect_record(_recStream, nullptr, nullptr, (pa_stream_flags_t)_recStreamFlags); + + // Wait for state change + while (pa_stream_get_state(_recStream) != PA_STREAM_READY) { + pa_threaded_mainloop_wait(_paMainloop); + } + + // We can now handle read callbacks + EnableReadCallback(); + + PaUnLock(); + + _startRec = false; + _recording = true; + _recStartEvent.Set(); + + return true; + } + + if (_recording) { + // Read data and provide it to VoiceEngine + if (ReadRecordedData(_tempSampleData, _tempSampleDataSize) == -1) { + return true; + } + + _tempSampleData = NULL; + _tempSampleDataSize = 0; + + PaLock(); + while (true) { + // Ack the last thing we read + pa_stream_drop(_recStream); + + if (pa_stream_readable_size(_recStream) <= 0) { + // Then that was all the data + break; + } + + // Else more data. + const void* sampleData; + size_t sampleDataSize; + + if (pa_stream_peek(_recStream, &sampleData, &sampleDataSize) != 0) { + break; + } + + // Drop lock for sigslot dispatch, which could take a while. + PaUnLock(); + // Read data and provide it to VoiceEngine + if (ReadRecordedData(sampleData, sampleDataSize) == -1) { + return true; + } + PaLock(); + + // Return to top of loop for the ack and the check for more data. + } + + EnableReadCallback(); + PaUnLock(); + + } // _recording + + return true; +} + +// Disables signals for audio recording. +void SystemModule::DisableReadCallback() { + pa_stream_set_read_callback(_recStream, NULL, NULL); +} + +// Redirect callback to `this.PaStreamReadCallbackHandler`. +void SystemModule::PaStreamReadCallback(pa_stream* a /*unused1*/, + size_t b /*unused2*/, + void* pThis) { + static_cast(pThis)->PaStreamReadCallbackHandler(); +} + +// Used to signal audio reading. +void SystemModule::PaStreamReadCallbackHandler() { + // We get the data pointer and size now in order to save one Lock/Unlock + // in the worker thread. + if (pa_stream_peek(_recStream, &_tempSampleData, &_tempSampleDataSize) != 0) { + return; + } + + // Since we consume the data asynchronously on a different thread, we have + // to temporarily disable the read callback or else Pulse will call it + // continuously until we consume the data. We re-enable it below. + DisableReadCallback(); + _timeEventRec.Set(); +} + +// Sorts and multiplie audio bytes. +void ReorderingBuffer(std::vector& output, + int16_t* data, + int sample_rare, + int channels, + float audio_multiplier) { + int samples = sample_rare / 100; + for (int i = 0; i < channels; ++i) { + int k = i; + for (int j = samples * i; k < samples * channels; ++j, k += channels) { + double inSample = data[k] * audio_multiplier; + if (inSample > INT16_MAX) + inSample = INT16_MAX; + if (inSample < INT16_MIN) + inSample = INT16_MIN; + + output[j] = (int16_t)inSample; + } + } +} + +// Passes an audio frame to an AudioSource. +int32_t SystemModule::ProcessRecordedData(int8_t* bufferData, + uint32_t bufferSizeInSamples, + uint32_t recDelay) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { + if (source != nullptr) { + ReorderingBuffer(release_capture_buffer, (int16_t*)bufferData, sample_rate_hz_, _recChannels, + audio_multiplier); + source->UpdateFrame(release_capture_buffer.data(), bufferSizeInSamples / _recChannels, + sample_rate_hz_, _recChannels); + } + + // We have been unlocked - check the flag again. + if (!_recording) { + return -1; + } + + return 0; +} + +// Enaables signals for audio recording. +void SystemModule::EnableReadCallback() { + pa_stream_set_read_callback(_recStream, &PaStreamReadCallback, this); +} + +int32_t SystemModule::ReadRecordedData(const void* bufferData, + size_t bufferSize) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_) { + size_t size = bufferSize; + uint32_t numRecSamples = _recordBufferSize / (2 * (int)1); + if (_recordBufferUsed > 0) { + // Have to copy to the buffer until it is full. + size_t copy = _recordBufferSize - _recordBufferUsed; + if (size < copy) { + copy = size; + } + + memcpy(&_recBuffer[_recordBufferUsed], bufferData, copy); + _recordBufferUsed += copy; + bufferData = static_cast(bufferData) + copy; + size -= copy; + + if (_recordBufferUsed != _recordBufferSize) { + // Not enough data yet to pass to VoE. + return 0; + } + + // Provide data to VoiceEngine. + if (ProcessRecordedData(_recBuffer, numRecSamples, 0) == -1) { + // We have stopped recording. + return -1; + } + + _recordBufferUsed = 0; + } + + // Now process full 10ms sample sets directly from the input. + while (size >= _recordBufferSize) { + // Provide data to VoiceEngine. + if (ProcessRecordedData(static_cast(const_cast(bufferData)), + numRecSamples, 0) == -1) { + // We have stopped recording. + return -1; + } + + bufferData = static_cast(bufferData) + _recordBufferSize; + size -= _recordBufferSize; + } + + // Now save any leftovers for later. + if (size > 0) { + memcpy(_recBuffer, bufferData, size); + _recordBufferUsed = size; + } + + return 0; +} + +// Callback to keep track of the state of the pulse audio stream. +void SystemModule::PaStreamStateCallbackHandler(pa_stream* p) { + pa_stream_state_t state = pa_stream_get_state(p); + switch (state) { + case PA_STREAM_UNCONNECTED: + break; + case PA_STREAM_CREATING: + break; + case PA_STREAM_FAILED: + case PA_STREAM_TERMINATED: + break; + case PA_STREAM_READY: + break; + } + + pa_threaded_mainloop_signal(_paMainloop, 0); +} + +// Redirect callback to `this.PaStreamStateCallbackHandler`. +void SystemModule::PaStreamStateCallback(pa_stream* p, void* pThis) { + static_cast(pThis)->PaStreamStateCallbackHandler(p); +} + +// Stops `RecThreadProcess`. +int32_t SystemModule::Terminate() { + quit_ = true; + return 0; +} + +// Creates the new `AudioSource`. +rtc::scoped_refptr SystemModule::CreateSource() { + if (!_recording) { + InitRecording(); + } + + if (!source) { + source = rtc::scoped_refptr(new SystemSource(this)); + } + auto result = source; + source->Release(); + source->Mute(); + return result; +} + +// Resets the audio source. +void SystemModule::ResetSource() { + source = nullptr; +} + +// Changes the captured source. +void SystemModule::SetRecordingSource(int id) { + auto start = _recIsInitialized; + StopRecording(); + if (_recIsInitialized) { + return; + } + + _recProcessId = id; + + if (start) { + InitRecording(); + StartRecording(); + } +} + +// Waiting for the pulse audio operation to completion. +void SystemModule::WaitForOperationCompletion(pa_operation* paOperation) const { + if (!paOperation) { + return; + } + + while (pa_operation_get_state(paOperation) == PA_OPERATION_RUNNING) { + pa_threaded_mainloop_wait(_paMainloop); + } + + pa_operation_unref(paOperation); +} + +// Initiates the audio stream from the provided `_recSinkId`. +void SystemModule::InitRecording() { + if (_recording) { + return; + } + + if (_recIsInitialized) { + return; + } + + if (_recProcessId < 0) { + return; + } + + auto audio_sources = EnumerateSystemSource(); + _recSinkId = -1; + for (int i = 0; i < audio_sources.size(); ++i) { + if (audio_sources[i].GetProcessId() == _recProcessId) { + _recSinkId = audio_sources[i].GetId(); + _recChannels = audio_sources[i].GetChannels(); + sample_rate_hz_ = audio_sources[i].GetSampleRate(); + } + } + + if (_recSinkId < 0) { + return; + } + + // Set the rec sample specification + pa_sample_spec recSampleSpec; + recSampleSpec.channels = (int)_recChannels; + recSampleSpec.format = PA_SAMPLE_S16LE; + recSampleSpec.rate = sample_rate_hz_; + + // Create a new rec stream + _recStream = pa_stream_new(_paContext, "System stream", &recSampleSpec, NULL); + + if (!_recStream) { + return; + } + + if (_configuredLatencyRec != WEBRTC_PA_NO_LATENCY_REQUIREMENTS) { + _recStreamFlags = (pa_stream_flags_t)(PA_STREAM_AUTO_TIMING_UPDATE | + PA_STREAM_INTERPOLATE_TIMING); + + // If configuring a specific latency then we want to specify + // PA_STREAM_ADJUST_LATENCY to make the server adjust parameters + // automatically to reach that target latency. However, that flag + // doesn't exist in Ubuntu 8.04 and many people still use that, + // so we have to check the protocol version of libpulse. + if (pa_context_get_protocol_version(_paContext) >= + WEBRTC_PA_ADJUST_LATENCY_PROTOCOL_VERSION) { + _recStreamFlags |= PA_STREAM_ADJUST_LATENCY; + } + + const pa_sample_spec* spec = pa_stream_get_sample_spec(_recStream); + if (!spec) { + return; + } + + size_t bytesPerSec = pa_bytes_per_second(spec); + uint32_t latency = bytesPerSec * WEBRTC_PA_LOW_CAPTURE_LATENCY_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + // Set the rec buffer attributes + // Note: fragsize specifies a maximum transfer size, not a minimum, so + // it is not possible to force a high latency setting, only a low one. + _recBufferAttr.fragsize = latency; // size of fragment + _recBufferAttr.maxlength = + latency + bytesPerSec * WEBRTC_PA_CAPTURE_BUFFER_EXTRA_MSECS / + WEBRTC_PA_MSECS_PER_SEC; + + _configuredLatencyRec = latency; + } + + _recordBufferSize = sample_rate_hz_ / 100 * 2 * _recChannels; + _recordBufferUsed = 0; + _recBuffer = new int8_t[_recordBufferSize]; + + pa_stream_set_state_callback(_recStream, PaStreamStateCallback, this); + + // Mark recording side as initialized + _recIsInitialized = true; +} + +// Sets the volume multiplier. +void SystemModule::SetSystemAudioLevel(float level) { + audio_multiplier = level; +} + +// Returns the volume multiplier. +float SystemModule::GetSystemAudioLevel() const { + return audio_multiplier; +} + +// Stops recording. +int32_t SystemModule::StopRecording() { + webrtc::MutexLock lock(&mutex_); + + if (!_recIsInitialized) { + return 0; + } + + if (_recStream == NULL) { + return -1; + } + + _recIsInitialized = false; + _recording = false; + + // Stop Recording + PaLock(); + + DisableReadCallback(); + pa_stream_set_overflow_callback(_recStream, NULL, NULL); + + // Unset this here so that we don't get a TERM // RTC_LOG(LS_ERROR) << "Can't read data!";INATED callback + pa_stream_set_state_callback(_recStream, NULL, NULL); + + auto state = pa_stream_get_state(_recStream); + if (state != PA_STREAM_UNCONNECTED) { + // Disconnect the stream + if (pa_stream_disconnect(_recStream) != PA_OK) { + PaUnLock(); + return -1; + } + } + + pa_stream_unref(_recStream); + _recStream = NULL; + + PaUnLock(); + + if (_recBuffer) { + delete[] _recBuffer; + _recBuffer = NULL; + } + + return 0; +} + +// Start recording. +int32_t SystemModule::StartRecording() { + if (!_recIsInitialized) { + return -1; + } + + if (_recording) { + return 0; + } + + // Set state to ensure that the recording starts from the audio thread. + _startRec = true; + + // The audio thread will signal when recording has started. + _timeEventRec.Set(); + if (!_recStartEvent.Wait(10000)) { + { + webrtc::MutexLock lock(&mutex_); + _startRec = false; + } + StopRecording(); + return -1; + } + + { + webrtc::MutexLock lock(&mutex_); + if (!_recording) { + return -1; + } + } + return 0; +} + +// Returns recording channels. +int32_t SystemModule::RecordingChannels() { + return _recChannels; +} + +// Enumerate system audio outputs. +std::vector SystemModule::EnumerateSystemSource() const { + auto result = std::vector(); + + SinkData data; + data.vec = &result; + data.pa_mainloop = _paMainloop; + + pa_threaded_mainloop_lock(_paMainloop); + pa_operation* op = pa_context_get_sink_input_info_list( + _paContext, reinterpret_cast(subscribe_sink_cb), + &data); + WaitForOperationCompletion(op); + pa_threaded_mainloop_unlock(_paMainloop); + + return result; +}; + +#endif \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/cpp/win-help.cc b/crates/libwebrtc-sys/src/cpp/win-help.cc new file mode 100644 index 0000000000..fdb9895b71 --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/win-help.cc @@ -0,0 +1,513 @@ +#if WEBRTC_WIN + +#include "win-help.h" +#include "dwmapi.h" + +// log_handler_t log_handler = def_log_handler; +// void (*crash_handler)(const char *, va_list, void *) = def_crash_handler; +const char* astrblank = ""; +long num_allocs = 0; +/* not capturable or internal windows, exact executable names */ +const char* internal_microsoft_exes_exact[] = { + "startmenuexperiencehost.exe", + "applicationframehost.exe", + "peopleexperiencehost.exe", + "shellexperiencehost.exe", + "microsoft.notes.exe", + "systemsettings.exe", + "textinputhost.exe", + "searchapp.exe", + "video.ui.exe", + "searchui.exe", + "lockapp.exe", + "cortana.exe", + "gamebar.exe", + "tabtip.exe", + "time.exe", + NULL, +}; +/* partial matches start from the beginning of the executable name */ +const char* internal_microsoft_exes_partial[] = { + "windowsinternal", + NULL, +}; + +inline long os_atomic_inc_long(volatile long* val) { + return _InterlockedIncrement(val); +} + +void* brealloc(void* ptr, size_t size) { + if (!ptr) + os_atomic_inc_long(&num_allocs); + + if (!size) { + size = 1; + } + + ptr = a_realloc(ptr, size); + + return ptr; +} + +inline void dstr_free(dstr& dst) { + bfree(dst.array); + dst.array = NULL; + dst.len = 0; + dst.capacity = 0; +} + +size_t wchar_to_utf8(const wchar_t* in, + size_t insize, + char* out, + size_t outsize, + int flags) { + int i_insize = (int)insize; + int ret; + + if (i_insize == 0) + i_insize = (int)wcslen(in); + + ret = WideCharToMultiByte(CP_UTF8, 0, in, i_insize, out, (int)outsize, NULL, + NULL); + + UNUSED_PARAMETER(flags); + return (ret > 0) ? (size_t)ret : 0; +} + +inline void dstr_resize(dstr& dst, const size_t num) { + if (!num) { + dstr_free(dst); + return; + } + + dstr_ensure_capacity(dst, num + 1); + dst.array[num] = 0; + dst.len = num; +} + +void dstr_from_wcs(dstr& dst, const wchar_t* wstr) { + size_t len = wchar_to_utf8(wstr, 0, NULL, 0, 0); + + if (len) { + dstr_resize(dst, len); + wchar_to_utf8(wstr, 0, dst.array, len + 1, 0); + } else { + dstr_free(dst); + } +} + +inline void dstr_ensure_capacity(dstr& dst, const size_t new_size) { + size_t new_cap; + if (new_size <= dst.capacity) + return; + + new_cap = (!dst.capacity) ? new_size : dst.capacity * 2; + if (new_size > new_cap) + new_cap = new_size; + dst.array = (char*)brealloc(dst.array, new_cap); + dst.capacity = new_cap; +} + +inline bool dstr_is_empty(dstr& str) { + if (!str.array || !str.len) + return true; + if (!str.array) + return true; + + return false; +} + +void dstr_copy(dstr& dst, const char* array) { + size_t len; + + if (!array || !*array) { + dstr_free(dst); + return; + } + + len = strlen(array); + dstr_ensure_capacity(dst, len + 1); + memcpy(dst.array, array, len + 1); + dst.len = len; +} + +int astrcmpi(const char* str1, const char* str2) { + if (!str1) + str1 = astrblank; + if (!str2) + str2 = astrblank; + + do { + char ch1 = (char)toupper(*str1); + char ch2 = (char)toupper(*str2); + + if (ch1 < ch2) + return -1; + else if (ch1 > ch2) + return 1; + } while (*str1++ && *str2++); + + return 0; +} + +int astrcmpi_n(const char* str1, const char* str2, size_t n) { + if (!n) + return 0; + if (!str1) + str1 = astrblank; + if (!str2) + str2 = astrblank; + + do { + char ch1 = (char)toupper(*str1); + char ch2 = (char)toupper(*str2); + + if (ch1 < ch2) + return -1; + else if (ch1 > ch2) + return 1; + } while (*str1++ && *str2++ && --n); + + return 0; +} + +inline int dstr_cmp(dstr& str1, const char* str2) { + return strcmp(str1.array, str2); +} + +std::vector ms_fill_window_list(enum window_search_mode mode) { + std::vector result; + HWND parent; + bool use_findwindowex = false; + + HWND window = first_window(mode, &parent, &use_findwindowex); + + while (window) { + struct dstr title = {0}; + struct dstr exe = {0}; + if (!ms_get_window_exe(exe, window)) { + window = next_window(window, mode, &parent, use_findwindowex); + continue; + } + + if (is_microsoft_internal_window_exe(exe.array)) { + dstr_free(exe); + window = next_window(window, mode, &parent, use_findwindowex); + continue; + } + + ms_get_window_title(title, window); + if (dstr_cmp(exe, "explorer.exe") == 0 && dstr_is_empty(title)) { + dstr_free(exe); + dstr_free(title); + window = next_window(window, mode, &parent, use_findwindowex); + continue; + } + + DWORD dwProcessId = 0; + if (GetWindowThreadProcessId(window, &dwProcessId)) { + ms_get_window_title(title, window); + AudioSourceInfo info(dwProcessId, std::string(title.array), dwProcessId); + result.push_back(info); + } + + window = next_window(window, mode, &parent, use_findwindowex); + dstr_free(exe); + dstr_free(title); + } + + return result; +} + +bool ms_get_window_exe(dstr& name, HWND window) { + wchar_t wname[MAX_PATH]; + struct dstr temp = {0}; + bool success = false; + HANDLE process = NULL; + char* slash; + DWORD id; + + GetWindowThreadProcessId(window, &id); + if (id == GetCurrentProcessId()) + return false; + + process = open_process(PROCESS_QUERY_LIMITED_INFORMATION, false, id); + if (!process) + goto fail; + + if (!GetProcessImageFileNameW(process, wname, MAX_PATH)) + goto fail; + + dstr_from_wcs(temp, wname); + slash = strrchr(temp.array, '\\'); + if (!slash) + goto fail; + + dstr_copy(name, slash + 1); + success = true; + +fail: + if (!success) + dstr_copy(name, "unknown"); + + dstr_free(temp); + CloseHandle(process); + return true; +} + +HWND ms_get_uwp_actual_window(HWND parent) { + DWORD parent_id = 0; + HWND child; + + GetWindowThreadProcessId(parent, &parent_id); + child = FindWindowEx(parent, NULL, NULL, NULL); + + while (child) { + DWORD child_id = 0; + GetWindowThreadProcessId(child, &child_id); + + if (child_id != parent_id) + return child; + + child = FindWindowEx(parent, child, NULL, NULL); + } + + return NULL; +} + +HWND first_window(enum window_search_mode mode, + HWND* parent, + bool* use_findwindowex) { + HWND window = FindWindowEx(GetDesktopWindow(), NULL, NULL, NULL); + + if (!window) { + *use_findwindowex = false; + window = GetWindow(GetDesktopWindow(), GW_CHILD); + } else { + *use_findwindowex = true; + } + + *parent = NULL; + + if (!check_window_valid(window, mode)) { + window = next_window(window, mode, parent, *use_findwindowex); + + if (!window && *use_findwindowex) { + *use_findwindowex = false; + + window = GetWindow(GetDesktopWindow(), GW_CHILD); + if (!check_window_valid(window, mode)) + window = next_window(window, mode, parent, *use_findwindowex); + } + } + + if (ms_is_uwp_window(window)) { + HWND child = ms_get_uwp_actual_window(window); + if (child) { + *parent = window; + return child; + } + } + + return window; +} + +bool check_window_valid(HWND window, enum window_search_mode mode) { + DWORD styles, ex_styles; + RECT rect; + + if (!IsWindowVisible(window) || + (mode == EXCLUDE_MINIMIZED && + (IsIconic(window) || IsWindowCloaked(window)))) + return false; + + GetClientRect(window, &rect); + styles = (DWORD)GetWindowLongPtr(window, GWL_STYLE); + ex_styles = (DWORD)GetWindowLongPtr(window, GWL_EXSTYLE); + + if (ex_styles & WS_EX_TOOLWINDOW) + return false; + if (styles & WS_CHILD) + return false; + if (mode == EXCLUDE_MINIMIZED && (rect.bottom == 0 || rect.right == 0)) + return false; + + return true; +} + +inline bool IsWindowCloaked(HWND window) { + DWORD cloaked; + HRESULT hr = + DwmGetWindowAttribute(window, DWMWA_CLOAKED, &cloaked, sizeof(cloaked)); + return SUCCEEDED(hr) && cloaked; +} + +bool ms_is_uwp_window(HWND hwnd) { + wchar_t name[256]; + + name[0] = 0; + if (!GetClassNameW(hwnd, name, sizeof(name) / sizeof(wchar_t))) + return false; + + return wcscmp(name, L"ApplicationFrameWindow") == 0; +} + +HWND next_window(HWND window, + enum window_search_mode mode, + HWND* parent, + bool use_findwindowex) { + if (*parent) { + window = *parent; + *parent = NULL; + } + + while (true) { + if (use_findwindowex) + window = FindWindowEx(GetDesktopWindow(), window, NULL, NULL); + else + window = GetNextWindow(window, GW_HWNDNEXT); + + if (!window || check_window_valid(window, mode)) + break; + } + + if (ms_is_uwp_window(window)) { + HWND child = ms_get_uwp_actual_window(window); + if (child) { + *parent = window; + return child; + } + } + + return window; +} + +HMODULE kernel32(void) { + HMODULE kernel32_handle = NULL; + if (!kernel32_handle) + kernel32_handle = GetModuleHandleA("kernel32"); + return kernel32_handle; +} + +inline HANDLE open_process(DWORD desired_access, + bool inherit_handle, + DWORD process_id) { + typedef HANDLE(WINAPI * PFN_OpenProcess)(DWORD, BOOL, DWORD); + PFN_OpenProcess open_process_proc = NULL; + if (!open_process_proc) + open_process_proc = (PFN_OpenProcess)ms_get_obfuscated_func( + kernel32(), "B}caZyah`~q", 0x2D5BEBAF6DDULL); + + return open_process_proc(desired_access, inherit_handle, process_id); +} + +void* ms_get_obfuscated_func(HMODULE module, const char* str, uint64_t val) { + char new_name[128]; + strcpy(new_name, str); + deobfuscate_str(new_name, val); + return GetProcAddress(module, new_name); +} + +void deobfuscate_str(char* str, uint64_t val) { + uint8_t* dec_val = (uint8_t*)&val; + int i = 0; + + while (*str != 0) { + int pos = i / 2; + bool bottom = (i % 2) == 0; + uint8_t* ch = (uint8_t*)str; + uint8_t xor = bottom ? LOWER_HALFBYTE(dec_val[pos]) + : UPPER_HALFBYTE(dec_val[pos]); + + *ch ^= xor; + + if (++i == sizeof(uint64_t) * 2) + i = 0; + + str++; + } +} + +bool is_microsoft_internal_window_exe(const char* exe) { + if (!exe) + return false; + + for (const char** vals = internal_microsoft_exes_exact; *vals; vals++) { + if (astrcmpi(exe, *vals) == 0) + return true; + } + + for (const char** vals = internal_microsoft_exes_partial; *vals; vals++) { + if (astrcmpi_n(exe, *vals, strlen(*vals)) == 0) + return true; + } + + return false; +} + +void ms_get_window_title(dstr& name, HWND hwnd) { + int len; + + len = GetWindowTextLengthW(hwnd); + if (!len) + return; + + if (len > 1024) { + wchar_t* temp; + + temp = (wchar_t*)malloc(sizeof(wchar_t) * (len + 1)); + if (!temp) + return; + + if (GetWindowTextW(hwnd, temp, len + 1)) + dstr_from_wcs(name, temp); + + free(temp); + } else { + wchar_t temp[1024 + 1]; + + if (GetWindowTextW(hwnd, temp, len + 1)) + dstr_from_wcs(name, temp); + } +} + +void* a_realloc(void* ptr, size_t size) { +#ifdef ALIGNED_MALLOC + return _aligned_realloc(ptr, size, ALIGNMENT); +#elif ALIGNMENT_HACK + long diff; + + if (!ptr) + return a_malloc(size); + diff = ((char*)ptr)[-1]; + ptr = realloc((char*)ptr - diff, size + diff); + if (ptr) + ptr = (char*)ptr + diff; + return ptr; +#else + return realloc(ptr, size); +#endif +} + +inline long os_atomic_dec_long(volatile long* val) { + return _InterlockedDecrement(val); +} + +void a_free(void* ptr) { +#ifdef ALIGNED_MALLOC + _aligned_free(ptr); +#elif ALIGNMENT_HACK + if (ptr) + free((char*)ptr - ((char*)ptr)[-1]); +#else + free(ptr); +#endif +} + +void bfree(void* ptr) { + if (ptr) { + os_atomic_dec_long(&num_allocs); + a_free(ptr); + } +} +#endif \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/cpp/windows_microphone_module.cc b/crates/libwebrtc-sys/src/cpp/windows_microphone_module.cc new file mode 100644 index 0000000000..27da016101 --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/windows_microphone_module.cc @@ -0,0 +1,2360 @@ +/* + * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#if WEBRTC_WIN +#pragma warning(disable : 4995) // name was marked as #pragma deprecated + +#if (_MSC_VER >= 1310) && (_MSC_VER < 1400) +// Reports the major and minor versions of the compiler. +// For example, 1310 for Microsoft Visual C++ .NET 2003. 1310 represents version +// 13 and a 1.0 point release. The Visual C++ 2005 compiler version is 1400. +// Type cl /? at the command line to see the major and minor versions of your +// compiler along with the build number. +#pragma message(">> INFO: Windows Core Audio is not supported in VS 2003") +#endif + +#include "modules/audio_device/audio_device_config.h" + +#ifdef WEBRTC_WINDOWS_CORE_AUDIO_BUILD + +// clang-format off +// To get Windows includes in the right order, this must come before the Windows +// includes below. +#include "windows_microphone_module.h" +// clang-format on + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "api/make_ref_counted.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/platform_thread.h" +#include "rtc_base/string_utils.h" +#include "rtc_base/thread_annotations.h" +#include "system_wrappers/include/sleep.h" + +// Macro that calls a COM method returning HRESULT value. +#define EXIT_ON_ERROR(hres) \ + do { \ + if (FAILED(hres)) \ + goto Exit; \ + } while (0) + +// Macro that continues to a COM error. +#define CONTINUE_ON_ERROR(hres) \ + do { \ + if (FAILED(hres)) \ + goto Next; \ + } while (0) + +// Macro that releases a COM object if not NULL. +#define SAFE_RELEASE(p) \ + do { \ + if ((p)) { \ + (p)->Release(); \ + (p) = NULL; \ + } \ + } while (0) + +#define ROUND(x) ((x) >= 0 ? (int)((x) + 0.5) : (int)((x)-0.5)) + +// REFERENCE_TIME time units per millisecond +#define REFTIMES_PER_MILLISEC 10000 + +typedef struct tagTHREADNAME_INFO { + DWORD dwType; // must be 0x1000 + LPCSTR szName; // pointer to name (in user addr space) + DWORD dwThreadID; // thread ID (-1=caller thread) + DWORD dwFlags; // reserved for future use, must be zero +} THREADNAME_INFO; + +namespace webrtc { +namespace { + +enum { COM_THREADING_MODEL = COINIT_MULTITHREADED }; + +enum { kAecCaptureStreamIndex = 0, kAecRenderStreamIndex = 1 }; + +// An implementation of IMediaBuffer, as required for +// IMediaObject::ProcessOutput(). After consuming data provided by +// ProcessOutput(), call SetLength() to update the buffer availability. +// +// Example implementation: +// http://msdn.microsoft.com/en-us/library/dd376684(v=vs.85).aspx +class MediaBufferImpl final : public IMediaBuffer { + public: + explicit MediaBufferImpl(DWORD maxLength) + : _data(new BYTE[maxLength]), + _length(0), + _maxLength(maxLength), + _refCount(0) {} + + // IMediaBuffer methods. + STDMETHOD(GetBufferAndLength(BYTE** ppBuffer, DWORD* pcbLength)) { + if (!ppBuffer || !pcbLength) { + return E_POINTER; + } + + *ppBuffer = _data; + *pcbLength = _length; + + return S_OK; + } + + STDMETHOD(GetMaxLength(DWORD* pcbMaxLength)) { + if (!pcbMaxLength) { + return E_POINTER; + } + + *pcbMaxLength = _maxLength; + return S_OK; + } + + STDMETHOD(SetLength(DWORD cbLength)) { + if (cbLength > _maxLength) { + return E_INVALIDARG; + } + + _length = cbLength; + return S_OK; + } + + // IUnknown methods. + STDMETHOD_(ULONG, AddRef()) { return InterlockedIncrement(&_refCount); } + + STDMETHOD(QueryInterface(REFIID riid, void** ppv)) { + if (!ppv) { + return E_POINTER; + } else if (riid != IID_IMediaBuffer && riid != IID_IUnknown) { + return E_NOINTERFACE; + } + + *ppv = static_cast(this); + AddRef(); + return S_OK; + } + + STDMETHOD_(ULONG, Release()) { + LONG refCount = InterlockedDecrement(&_refCount); + if (refCount == 0) { + delete this; + } + + return refCount; + } + + private: + ~MediaBufferImpl() { delete[] _data; } + + BYTE* _data; + DWORD _length; + const DWORD _maxLength; + LONG _refCount; +}; +} // namespace +} // namespace webrtc + +int MicrophoneSource::sources_num = 0; +MicrophoneSource::MicrophoneSource(MicrophoneModuleInterface* module) { + this->module = module; + if (sources_num == 0) { + module->StartRecording(); + } + ++sources_num; +} + +MicrophoneSource::~MicrophoneSource() { + --sources_num; + if (sources_num == 0) { + module->StopRecording(); + module->ResetSource(); + } +} + +void MicrophoneModule::ResetSource() { + webrtc::MutexLock lock(&mutex_); + source = nullptr; +} + +// ---------------------------------------------------------------------------- +// SetRecordingDevice I (II) +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::SetRecordingDevice(uint16_t index) { + bool start = _recIsInitialized; + StopRecording(); + + if (_recIsInitialized) { + return -1; + } + + // Get current number of available capture endpoint devices and refresh the + // capture collection. + UINT nDevices = RecordingDevices(); + + if (index < 0 || index > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "device index is out of range [0," << (nDevices - 1) + << "]"; + return -1; + } + + webrtc::MutexLock lock(&mutex_); + + HRESULT hr(S_OK); + + RTC_DCHECK(_ptrCaptureCollection); + + // Select an endpoint capture device given the specified index + SAFE_RELEASE(_ptrDeviceIn); + hr = _ptrCaptureCollection->Item(index, &_ptrDeviceIn); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(_ptrDeviceIn); + return -1; + } + + WCHAR szDeviceName[MAX_PATH]; + const int bufferLen = sizeof(szDeviceName) / sizeof(szDeviceName)[0]; + + // Get the endpoint device's friendly-name + if (_GetDeviceName(_ptrDeviceIn, szDeviceName, bufferLen) == 0) { + RTC_LOG(LS_VERBOSE) << "friendly name: \"" << szDeviceName << "\""; + } + + _usingInputDeviceIndex = true; + _inputDeviceIndex = index; + + if (start) { + InitRecording(); + StartRecording(); + } + return 0; +} + +int32_t MicrophoneModule::RecordingChannels() { + return _recChannels; +} + +// ---------------------------------------------------------------------------- +// DoCaptureThread +// ---------------------------------------------------------------------------- + +DWORD MicrophoneModule::DoCaptureThread() { + bool keepRecording = true; + HANDLE waitArray[2] = {_hShutdownCaptureEvent, _hCaptureSamplesReadyEvent}; + HRESULT hr = S_OK; + + LARGE_INTEGER t1; + + BYTE* syncBuffer = NULL; + UINT32 syncBufIndex = 0; + + _readSamples = 0; + + // Initialize COM as MTA in this thread. + webrtc::ScopedCOMInitializer comInit(webrtc::ScopedCOMInitializer::kMTA); + if (!comInit.Succeeded()) { + RTC_LOG(LS_ERROR) << "failed to initialize COM in capture thread"; + return 1; + } + + hr = InitCaptureThreadPriority(); + if (FAILED(hr)) { + return hr; + } + + mutex_.Lock(); + + // Get size of capturing buffer (length is expressed as the number of audio + // frames the buffer can hold). This value is fixed during the capturing + // session. + // + UINT32 bufferLength = 0; + if (_ptrClientIn == NULL) { + RTC_LOG(LS_ERROR) + << "input state has been modified before capture loop starts."; + return 1; + } + hr = _ptrClientIn->GetBufferSize(&bufferLength); + EXIT_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "[CAPT] size of buffer : " << bufferLength; + + // Allocate memory for sync buffer. + // It is used for compensation between native 44.1 and internal 44.0 and + // for cases when the capture buffer is larger than 10ms. + // + const UINT32 syncBufferSize = 2 * (bufferLength * _recAudioFrameSize); + syncBuffer = new BYTE[syncBufferSize]; + if (syncBuffer == NULL) { + return (DWORD)E_POINTER; + } + RTC_LOG(LS_VERBOSE) << "[CAPT] size of sync buffer : " << syncBufferSize + << " [bytes]"; + + // Get maximum latency for the current stream (will not change for the + // lifetime of the IAudioClient object). + // + REFERENCE_TIME latency; + _ptrClientIn->GetStreamLatency(&latency); + RTC_LOG(LS_VERBOSE) << "[CAPT] max stream latency : " << (DWORD)latency + << " (" << (double)(latency / 10000.0) << " ms)"; + + // Get the length of the periodic interval separating successive processing + // passes by the audio engine on the data in the endpoint buffer. + // + REFERENCE_TIME devPeriod = 0; + REFERENCE_TIME devPeriodMin = 0; + _ptrClientIn->GetDevicePeriod(&devPeriod, &devPeriodMin); + RTC_LOG(LS_VERBOSE) << "[CAPT] device period : " << (DWORD)devPeriod + << " (" << (double)(devPeriod / 10000.0) << " ms)"; + + double extraDelayMS = (double)((latency + devPeriod) / 10000.0); + RTC_LOG(LS_VERBOSE) << "[CAPT] extraDelayMS : " << extraDelayMS; + + double endpointBufferSizeMS = + 10.0 * ((double)bufferLength / (double)_recBlockSize); + RTC_LOG(LS_VERBOSE) << "[CAPT] endpointBufferSizeMS : " + << endpointBufferSizeMS; + + // Start up the capturing stream. + // + hr = _ptrClientIn->Start(); + EXIT_ON_ERROR(hr); + + mutex_.Unlock(); + + // Set event which will ensure that the calling thread modifies the recording + // state to true. + // + SetEvent(_hCaptureStartedEvent); + + // >> ---------------------------- THREAD LOOP ---------------------------- + + while (keepRecording) { + // Wait for a capture notification event or a shutdown event + DWORD waitResult = WaitForMultipleObjects(2, waitArray, FALSE, 500); + switch (waitResult) { + case WAIT_OBJECT_0 + 0: // _hShutdownCaptureEvent + keepRecording = false; + break; + case WAIT_OBJECT_0 + 1: // _hCaptureSamplesReadyEvent + break; + case WAIT_TIMEOUT: // timeout notification + RTC_LOG(LS_WARNING) << "capture event timed out after 0.5 seconds"; + goto Exit; + default: // unexpected error + RTC_LOG(LS_WARNING) << "unknown wait termination on capture side"; + goto Exit; + } + + while (keepRecording) { + BYTE* pData = 0; + UINT32 framesAvailable = 0; + DWORD flags = 0; + UINT64 recTime = 0; + UINT64 recPos = 0; + + mutex_.Lock(); + + // Sanity check to ensure that essential states are not modified + // during the unlocked period. + if (_ptrCaptureClient == NULL || _ptrClientIn == NULL) { + mutex_.Unlock(); + RTC_LOG(LS_ERROR) + << "input state has been modified during unlocked period"; + goto Exit; + } + + // Find out how much capture data is available + // + hr = _ptrCaptureClient->GetBuffer( + &pData, // packet which is ready to be read by used + &framesAvailable, // #frames in the captured packet (can be zero) + &flags, // support flags (check) + &recPos, // device position of first audio frame in data packet + &recTime); // value of performance counter at the time of recording + // the first audio frame + + if (SUCCEEDED(hr)) { + if (AUDCLNT_S_BUFFER_EMPTY == hr) { + // Buffer was empty => start waiting for a new capture notification + // event + mutex_.Unlock(); + break; + } + + if (flags & AUDCLNT_BUFFERFLAGS_SILENT) { + // Treat all of the data in the packet as silence and ignore the + // actual data values. + RTC_LOG(LS_WARNING) << "AUDCLNT_BUFFERFLAGS_SILENT"; + pData = NULL; + } + + RTC_DCHECK_NE(framesAvailable, 0); + + if (pData) { + CopyMemory(&syncBuffer[syncBufIndex * _recAudioFrameSize], pData, + framesAvailable * _recAudioFrameSize); + } else { + ZeroMemory(&syncBuffer[syncBufIndex * _recAudioFrameSize], + framesAvailable * _recAudioFrameSize); + } + RTC_DCHECK_GE(syncBufferSize, (syncBufIndex * _recAudioFrameSize) + + framesAvailable * _recAudioFrameSize); + + // Release the capture buffer + // + hr = _ptrCaptureClient->ReleaseBuffer(framesAvailable); + EXIT_ON_ERROR(hr); + + _readSamples += framesAvailable; + syncBufIndex += framesAvailable; + + QueryPerformanceCounter(&t1); + + // Get the current recording and playout delay. + uint32_t sndCardRecDelay = + (uint32_t)(((((UINT64)t1.QuadPart * _perfCounterFactor) - recTime) / + 10000) + + (10 * syncBufIndex) / _recBlockSize - 10); + uint32_t sndCardPlayDelay = static_cast(_sndCardPlayDelay); + + while (syncBufIndex >= _recBlockSize) { + if (source) { + source->UpdateFrame((const int16_t*)syncBuffer, _recBlockSize, + _recSampleRate, _recChannels); + } + + // store remaining data which was not able to deliver as 10ms segment + MoveMemory(&syncBuffer[0], + &syncBuffer[_recBlockSize * _recAudioFrameSize], + (syncBufIndex - _recBlockSize) * _recAudioFrameSize); + syncBufIndex -= _recBlockSize; + sndCardRecDelay -= 10; + } + } else { + // If GetBuffer returns AUDCLNT_E_BUFFER_ERROR, the thread consuming the + // audio samples must wait for the next processing pass. The client + // might benefit from keeping a count of the failed GetBuffer calls. If + // GetBuffer returns this error repeatedly, the client can start a new + // processing loop after shutting down the current client by calling + // IAudioClient::Stop, IAudioClient::Reset, and releasing the audio + // client. + RTC_LOG(LS_ERROR) << "IAudioCaptureClient::GetBuffer returned" + " AUDCLNT_E_BUFFER_ERROR, hr = 0x" + << rtc::ToHex(hr); + goto Exit; + } + + mutex_.Unlock(); + } + } + + // ---------------------------- THREAD LOOP ---------------------------- << + + if (_ptrClientIn) { + hr = _ptrClientIn->Stop(); + } + +Exit: + if (FAILED(hr)) { + _ptrClientIn->Stop(); + mutex_.Unlock(); + _TraceCOMError(hr); + } + + RevertCaptureThreadPriority(); + + mutex_.Lock(); + + if (keepRecording) { + if (_ptrClientIn != NULL) { + hr = _ptrClientIn->Stop(); + if (FAILED(hr)) { + _TraceCOMError(hr); + } + hr = _ptrClientIn->Reset(); + if (FAILED(hr)) { + _TraceCOMError(hr); + } + } + + RTC_LOG(LS_ERROR) + << "Recording error: capturing thread has ended pre-maturely"; + } else { + RTC_LOG(LS_VERBOSE) << "_Capturing thread is now terminated properly"; + } + + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrCaptureClient); + + mutex_.Unlock(); + + if (syncBuffer) { + delete[] syncBuffer; + } + + return (DWORD)hr; +} + +rtc::scoped_refptr MicrophoneModule::CreateSource() { + if (!_recording) { + InitRecording(); + } + + if (!source) { + source = rtc::scoped_refptr(new MicrophoneSource(this)); + } + auto result = source; + source->Release(); + return result; +} + +DWORD MicrophoneModule::DoCaptureThreadPollDMO() { + RTC_DCHECK(_mediaBuffer); + bool keepRecording = true; + + // Initialize COM as MTA in this thread. + webrtc::ScopedCOMInitializer comInit(webrtc::ScopedCOMInitializer::kMTA); + if (!comInit.Succeeded()) { + RTC_LOG(LS_ERROR) << "failed to initialize COM in polling DMO thread"; + return 1; + } + + HRESULT hr = InitCaptureThreadPriority(); + if (FAILED(hr)) { + return hr; + } + + // Set event which will ensure that the calling thread modifies the + // recording state to true. + SetEvent(_hCaptureStartedEvent); + + // >> ---------------------------- THREAD LOOP ---------------------------- + while (keepRecording) { + // Poll the DMO every 5 ms. + // (The same interval used in the Wave implementation.) + DWORD waitResult = WaitForSingleObject(_hShutdownCaptureEvent, 5); + switch (waitResult) { + case WAIT_OBJECT_0: // _hShutdownCaptureEvent + keepRecording = false; + break; + case WAIT_TIMEOUT: // timeout notification + break; + default: // unexpected error + RTC_LOG(LS_WARNING) << "Unknown wait termination on capture side"; + hr = -1; // To signal an error callback. + keepRecording = false; + break; + } + + while (keepRecording) { + webrtc::MutexLock lockScoped(&mutex_); + + DWORD dwStatus = 0; + { + DMO_OUTPUT_DATA_BUFFER dmoBuffer = {0}; + dmoBuffer.pBuffer = _mediaBuffer.get(); + dmoBuffer.pBuffer->AddRef(); + + // Poll the DMO for AEC processed capture data. The DMO will + // copy available data to `dmoBuffer`, and should only return + // 10 ms frames. The value of `dwStatus` should be ignored. + hr = _dmo->ProcessOutput(0, 1, &dmoBuffer, &dwStatus); + SAFE_RELEASE(dmoBuffer.pBuffer); + dwStatus = dmoBuffer.dwStatus; + } + if (FAILED(hr)) { + _TraceCOMError(hr); + keepRecording = false; + RTC_DCHECK_NOTREACHED(); + break; + } + + ULONG bytesProduced = 0; + BYTE* data; + // Get a pointer to the data buffer. This should be valid until + // the next call to ProcessOutput. + hr = _mediaBuffer->GetBufferAndLength(&data, &bytesProduced); + if (FAILED(hr)) { + _TraceCOMError(hr); + keepRecording = false; + RTC_DCHECK_NOTREACHED(); + break; + } + + if (bytesProduced > 0) { + const int kSamplesProduced = bytesProduced / _recAudioFrameSize; + // TODO(andrew): verify that this is always satisfied. It might + // be that ProcessOutput will try to return more than 10 ms if + // we fail to call it frequently enough. + RTC_DCHECK_EQ(kSamplesProduced, static_cast(_recBlockSize)); + RTC_DCHECK_EQ(sizeof(BYTE), sizeof(int8_t)); + + if (source) { + source->UpdateFrame((const int16_t*)data, kSamplesProduced, + _recSampleRate, _recChannels); + } + + // mutex_.Unlock(); // Release lock while making the callback. + // _ptrAudioBuffer->DeliverRecordedData(); + // mutex_.Lock(); + } + + // Reset length to indicate buffer availability. + hr = _mediaBuffer->SetLength(0); + if (FAILED(hr)) { + _TraceCOMError(hr); + keepRecording = false; + RTC_DCHECK_NOTREACHED(); + break; + } + + if (!(dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE)) { + // The DMO cannot currently produce more data. This is the + // normal case; otherwise it means the DMO had more than 10 ms + // of data available and ProcessOutput should be called again. + break; + } + } + } +} + +// ---------------------------------------------------------------------------- +// Everything below has been copied unchanged from audio_device_core_win.сс +// ---------------------------------------------------------------------------- + +// ---------------------------------------------------------------------------- +// Terminate +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::Terminate() { + webrtc::MutexLock lock(&mutex_); + + if (!_initialized) { + return 0; + } + + _initialized = false; + _speakerIsInitialized = false; + _microphoneIsInitialized = false; + _playing = false; + _recording = false; + + SAFE_RELEASE(_ptrRenderCollection); + SAFE_RELEASE(_ptrCaptureCollection); + SAFE_RELEASE(_ptrDeviceOut); + SAFE_RELEASE(_ptrDeviceIn); + SAFE_RELEASE(_ptrClientOut); + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrRenderClient); + SAFE_RELEASE(_ptrCaptureClient); + SAFE_RELEASE(_ptrCaptureVolume); + SAFE_RELEASE(_ptrRenderSimpleVolume); + + return 0; +} + +// ---------------------------------------------------------------------------- +// MicrophoneModule() - ctor +// ---------------------------------------------------------------------------- + +MicrophoneModule::MicrophoneModule() + : _avrtLibrary(nullptr), + _winSupportAvrt(false), + _comInit(webrtc::ScopedCOMInitializer::kMTA), + _ptrAudioBuffer(nullptr), + _ptrEnumerator(nullptr), + _ptrRenderCollection(nullptr), + _ptrCaptureCollection(nullptr), + _ptrDeviceOut(nullptr), + _ptrDeviceIn(nullptr), + _ptrClientOut(nullptr), + _ptrClientIn(nullptr), + _ptrRenderClient(nullptr), + _ptrCaptureClient(nullptr), + _ptrCaptureVolume(nullptr), + _ptrRenderSimpleVolume(nullptr), + _dmo(nullptr), + _mediaBuffer(nullptr), + _builtInAecEnabled(false), + _hRenderSamplesReadyEvent(nullptr), + _hPlayThread(nullptr), + _hRenderStartedEvent(nullptr), + _hShutdownRenderEvent(nullptr), + _hCaptureSamplesReadyEvent(nullptr), + _hRecThread(nullptr), + _hCaptureStartedEvent(nullptr), + _hShutdownCaptureEvent(nullptr), + _hMmTask(nullptr), + _playAudioFrameSize(0), + _playSampleRate(0), + _playBlockSize(0), + _playChannels(2), + _sndCardPlayDelay(0), + _writtenSamples(0), + _readSamples(0), + _recAudioFrameSize(0), + _recSampleRate(0), + _recBlockSize(0), + _recChannels(2), + _initialized(false), + _recording(false), + _playing(false), + _recIsInitialized(false), + _playIsInitialized(false), + _speakerIsInitialized(false), + _microphoneIsInitialized(false), + _usingInputDeviceIndex(false), + _usingOutputDeviceIndex(false), + _inputDevice(webrtc::AudioDeviceModule::kDefaultCommunicationDevice), + _outputDevice(webrtc::AudioDeviceModule::kDefaultCommunicationDevice), + _inputDeviceIndex(0), + _outputDeviceIndex(0) { + RTC_DLOG(LS_INFO) << __FUNCTION__ << " created"; + RTC_DCHECK(_comInit.Succeeded()); + + // Try to load the Avrt DLL + if (!_avrtLibrary) { + // Get handle to the Avrt DLL module. + _avrtLibrary = LoadLibrary(TEXT("Avrt.dll")); + if (_avrtLibrary) { + // Handle is valid (should only happen if OS larger than vista & win7). + // Try to get the function addresses. + RTC_LOG(LS_VERBOSE) << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + " The Avrt DLL module is now loaded"; + + _PAvRevertMmThreadCharacteristics = + (PAvRevertMmThreadCharacteristics)GetProcAddress( + _avrtLibrary, "AvRevertMmThreadCharacteristics"); + _PAvSetMmThreadCharacteristicsA = + (PAvSetMmThreadCharacteristicsA)GetProcAddress( + _avrtLibrary, "AvSetMmThreadCharacteristicsA"); + _PAvSetMmThreadPriority = (PAvSetMmThreadPriority)GetProcAddress( + _avrtLibrary, "AvSetMmThreadPriority"); + + if (_PAvRevertMmThreadCharacteristics && + _PAvSetMmThreadCharacteristicsA && _PAvSetMmThreadPriority) { + RTC_LOG(LS_VERBOSE) + << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + " AvRevertMmThreadCharacteristics() is OK"; + RTC_LOG(LS_VERBOSE) + << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + " AvSetMmThreadCharacteristicsA() is OK"; + RTC_LOG(LS_VERBOSE) + << "AudioDeviceWindowsCore::AudioDeviceWindowsCore()" + " AvSetMmThreadPriority() is OK"; + _winSupportAvrt = true; + } + } + } + + // Create our samples ready events - we want auto reset events that start in + // the not-signaled state. The state of an auto-reset event object remains + // signaled until a single waiting thread is released, at which time the + // system automatically sets the state to nonsignaled. If no threads are + // waiting, the event object's state remains signaled. (Except for + // _hShutdownCaptureEvent, which is used to shutdown multiple threads). + _hRenderSamplesReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hCaptureSamplesReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hShutdownRenderEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hShutdownCaptureEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + _hRenderStartedEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + _hCaptureStartedEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + + _perfCounterFreq.QuadPart = 1; + _perfCounterFactor = 0.0; + + // list of number of channels to use on recording side + _recChannelsPrioList[0] = 2; // stereo is prio 1 + _recChannelsPrioList[1] = 1; // mono is prio 2 + _recChannelsPrioList[2] = 4; // quad is prio 3 + + // list of number of channels to use on playout side + _playChannelsPrioList[0] = 2; // stereo is prio 1 + _playChannelsPrioList[1] = 1; // mono is prio 2 + + HRESULT hr; + + // We know that this API will work since it has already been verified in + // CoreAudioIsSupported, hence no need to check for errors here as well. + + // Retrive the IMMDeviceEnumerator API (should load the MMDevAPI.dll) + // TODO(henrika): we should probably move this allocation to Init() instead + // and deallocate in Terminate() to make the implementation more symmetric. + CoCreateInstance(__uuidof(MMDeviceEnumerator), NULL, CLSCTX_ALL, + __uuidof(IMMDeviceEnumerator), + reinterpret_cast(&_ptrEnumerator)); + RTC_DCHECK(_ptrEnumerator); + + // DMO initialization for built-in WASAPI AEC. + { + IMediaObject* ptrDMO = NULL; + hr = CoCreateInstance(CLSID_CWMAudioAEC, NULL, CLSCTX_INPROC_SERVER, + IID_IMediaObject, reinterpret_cast(&ptrDMO)); + if (FAILED(hr) || ptrDMO == NULL) { + // Since we check that _dmo is non-NULL in EnableBuiltInAEC(), the + // feature is prevented from being enabled. + _builtInAecEnabled = false; + _TraceCOMError(hr); + } + _dmo = ptrDMO; + SAFE_RELEASE(ptrDMO); + } +} + +// ---------------------------------------------------------------------------- +// MicrophoneModule() - dtor +// ---------------------------------------------------------------------------- + +MicrophoneModule::~MicrophoneModule() { + RTC_DLOG(LS_INFO) << __FUNCTION__ << " destroyed"; + + Terminate(); + + // The IMMDeviceEnumerator is created during construction. Must release + // it here and not in Terminate() since we don't recreate it in Init(). + SAFE_RELEASE(_ptrEnumerator); + + _ptrAudioBuffer = NULL; + + if (NULL != _hRenderSamplesReadyEvent) { + CloseHandle(_hRenderSamplesReadyEvent); + _hRenderSamplesReadyEvent = NULL; + } + + if (NULL != _hCaptureSamplesReadyEvent) { + CloseHandle(_hCaptureSamplesReadyEvent); + _hCaptureSamplesReadyEvent = NULL; + } + + if (NULL != _hRenderStartedEvent) { + CloseHandle(_hRenderStartedEvent); + _hRenderStartedEvent = NULL; + } + + if (NULL != _hCaptureStartedEvent) { + CloseHandle(_hCaptureStartedEvent); + _hCaptureStartedEvent = NULL; + } + + if (NULL != _hShutdownRenderEvent) { + CloseHandle(_hShutdownRenderEvent); + _hShutdownRenderEvent = NULL; + } + + if (NULL != _hShutdownCaptureEvent) { + CloseHandle(_hShutdownCaptureEvent); + _hShutdownCaptureEvent = NULL; + } + + if (_avrtLibrary) { + BOOL freeOK = FreeLibrary(_avrtLibrary); + if (!freeOK) { + RTC_LOG(LS_WARNING) + << "AudioDeviceWindowsCore::~AudioDeviceWindowsCore()" + " failed to free the loaded Avrt DLL module correctly"; + } else { + RTC_LOG(LS_WARNING) << "AudioDeviceWindowsCore::~AudioDeviceWindowsCore()" + " the Avrt DLL module is now unloaded"; + } + } +} + +// ---------------------------------------------------------------------------- +// _EnumerateEndpointDevicesAll +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::_EnumerateEndpointDevicesAll( + EDataFlow dataFlow) const { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + RTC_DCHECK(_ptrEnumerator); + + HRESULT hr = S_OK; + IMMDeviceCollection* pCollection = NULL; + IMMDevice* pEndpoint = NULL; + IPropertyStore* pProps = NULL; + IAudioEndpointVolume* pEndpointVolume = NULL; + LPWSTR pwszID = NULL; + + // Generate a collection of audio endpoint devices in the system. + // Get states for *all* endpoint devices. + // Output: IMMDeviceCollection interface. + hr = _ptrEnumerator->EnumAudioEndpoints( + dataFlow, // data-flow direction (input parameter) + DEVICE_STATE_ACTIVE | DEVICE_STATE_DISABLED | DEVICE_STATE_UNPLUGGED, + &pCollection); // release interface when done + + EXIT_ON_ERROR(hr); + + // use the IMMDeviceCollection interface... + + UINT count = 0; + + // Retrieve a count of the devices in the device collection. + hr = pCollection->GetCount(&count); + EXIT_ON_ERROR(hr); + if (dataFlow == eRender) + RTC_LOG(LS_VERBOSE) << "#rendering endpoint devices (counting all): " + << count; + else if (dataFlow == eCapture) + RTC_LOG(LS_VERBOSE) << "#capturing endpoint devices (counting all): " + << count; + + if (count == 0) { + return 0; + } + + // Each loop prints the name of an endpoint device. + for (ULONG i = 0; i < count; i++) { + RTC_LOG(LS_VERBOSE) << "Endpoint " << i << ":"; + + // Get pointer to endpoint number i. + // Output: IMMDevice interface. + hr = pCollection->Item(i, &pEndpoint); + CONTINUE_ON_ERROR(hr); + + // use the IMMDevice interface of the specified endpoint device... + + // Get the endpoint ID string (uniquely identifies the device among all + // audio endpoint devices) + hr = pEndpoint->GetId(&pwszID); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "ID string : " << pwszID; + + // Retrieve an interface to the device's property store. + // Output: IPropertyStore interface. + hr = pEndpoint->OpenPropertyStore(STGM_READ, &pProps); + CONTINUE_ON_ERROR(hr); + + // use the IPropertyStore interface... + + PROPVARIANT varName; + // Initialize container for property value. + PropVariantInit(&varName); + + // Get the endpoint's friendly-name property. + // Example: "Speakers (Realtek High Definition Audio)" + hr = pProps->GetValue(PKEY_Device_FriendlyName, &varName); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "friendly name: \"" << varName.pwszVal << "\""; + + // Get the endpoint's current device state + DWORD dwState; + hr = pEndpoint->GetState(&dwState); + CONTINUE_ON_ERROR(hr); + if (dwState & DEVICE_STATE_ACTIVE) + RTC_LOG(LS_VERBOSE) << "state (0x" << rtc::ToHex(dwState) + << ") : *ACTIVE*"; + if (dwState & DEVICE_STATE_DISABLED) + RTC_LOG(LS_VERBOSE) << "state (0x" << rtc::ToHex(dwState) + << ") : DISABLED"; + if (dwState & DEVICE_STATE_NOTPRESENT) + RTC_LOG(LS_VERBOSE) << "state (0x" << rtc::ToHex(dwState) + << ") : NOTPRESENT"; + if (dwState & DEVICE_STATE_UNPLUGGED) + RTC_LOG(LS_VERBOSE) << "state (0x" << rtc::ToHex(dwState) + << ") : UNPLUGGED"; + + // Check the hardware volume capabilities. + DWORD dwHwSupportMask = 0; + hr = pEndpoint->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + (void**)&pEndpointVolume); + CONTINUE_ON_ERROR(hr); + hr = pEndpointVolume->QueryHardwareSupport(&dwHwSupportMask); + CONTINUE_ON_ERROR(hr); + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_VOLUME) + // The audio endpoint device supports a hardware volume control + RTC_LOG(LS_VERBOSE) << "hwmask (0x" << rtc::ToHex(dwHwSupportMask) + << ") : HARDWARE_SUPPORT_VOLUME"; + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_MUTE) + // The audio endpoint device supports a hardware mute control + RTC_LOG(LS_VERBOSE) << "hwmask (0x" << rtc::ToHex(dwHwSupportMask) + << ") : HARDWARE_SUPPORT_MUTE"; + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_METER) + // The audio endpoint device supports a hardware peak meter + RTC_LOG(LS_VERBOSE) << "hwmask (0x" << rtc::ToHex(dwHwSupportMask) + << ") : HARDWARE_SUPPORT_METER"; + + // Check the channel count (#channels in the audio stream that enters or + // leaves the audio endpoint device) + UINT nChannelCount(0); + hr = pEndpointVolume->GetChannelCount(&nChannelCount); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "#channels : " << nChannelCount; + + if (dwHwSupportMask & ENDPOINT_HARDWARE_SUPPORT_VOLUME) { + // Get the volume range. + float fLevelMinDB(0.0); + float fLevelMaxDB(0.0); + float fVolumeIncrementDB(0.0); + hr = pEndpointVolume->GetVolumeRange(&fLevelMinDB, &fLevelMaxDB, + &fVolumeIncrementDB); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "volume range : " << fLevelMinDB << " (min), " + << fLevelMaxDB << " (max), " << fVolumeIncrementDB + << " (inc) [dB]"; + + // The volume range from vmin = fLevelMinDB to vmax = fLevelMaxDB is + // divided into n uniform intervals of size vinc = fVolumeIncrementDB, + // where n = (vmax ?vmin) / vinc. The values vmin, vmax, and vinc are + // measured in decibels. The client can set the volume level to one of n + + // 1 discrete values in the range from vmin to vmax. + int n = (int)((fLevelMaxDB - fLevelMinDB) / fVolumeIncrementDB); + RTC_LOG(LS_VERBOSE) << "#intervals : " << n; + + // Get information about the current step in the volume range. + // This method represents the volume level of the audio stream that enters + // or leaves the audio endpoint device as an index or "step" in a range of + // discrete volume levels. Output value nStepCount is the number of steps + // in the range. Output value nStep is the step index of the current + // volume level. If the number of steps is n = nStepCount, then step index + // nStep can assume values from 0 (minimum volume) to n ?1 (maximum + // volume). + UINT nStep(0); + UINT nStepCount(0); + hr = pEndpointVolume->GetVolumeStepInfo(&nStep, &nStepCount); + CONTINUE_ON_ERROR(hr); + RTC_LOG(LS_VERBOSE) << "volume steps : " << nStep << " (nStep), " + << nStepCount << " (nStepCount)"; + } + Next: + if (FAILED(hr)) { + RTC_LOG(LS_VERBOSE) << "Error when logging device information"; + } + CoTaskMemFree(pwszID); + pwszID = NULL; + PropVariantClear(&varName); + SAFE_RELEASE(pProps); + SAFE_RELEASE(pEndpoint); + SAFE_RELEASE(pEndpointVolume); + } + SAFE_RELEASE(pCollection); + return 0; + +Exit: + _TraceCOMError(hr); + CoTaskMemFree(pwszID); + pwszID = NULL; + SAFE_RELEASE(pCollection); + SAFE_RELEASE(pEndpoint); + SAFE_RELEASE(pEndpointVolume); + SAFE_RELEASE(pProps); + return -1; +} + +// ---------------------------------------------------------------------------- +// Init +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::Init() { + webrtc::MutexLock lock(&mutex_); + + if (_initialized) { + return 0; + } + + // Enumerate all audio rendering and capturing endpoint devices. + // Note that, some of these will not be able to select by the user. + // The complete collection is for internal use only. + _EnumerateEndpointDevicesAll(eCapture); + + _initialized = true; + + return 0; +} + +// ---------------------------------------------------------------------------- +// _TraceCOMError +// ---------------------------------------------------------------------------- + +void MicrophoneModule::_TraceCOMError(HRESULT hr) const { + wchar_t buf[MAXERRORLENGTH]; + wchar_t errorText[MAXERRORLENGTH]; + + const DWORD dwFlags = + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS; + const DWORD dwLangID = MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US); + + // Gets the system's human readable message string for this HRESULT. + // All error message in English by default. + DWORD messageLength = ::FormatMessageW(dwFlags, 0, hr, dwLangID, errorText, + MAXERRORLENGTH, NULL); + + RTC_DCHECK_LE(messageLength, MAXERRORLENGTH); + + // Trims tailing white space (FormatMessage() leaves a trailing cr-lf.). + for (; messageLength && ::isspace(errorText[messageLength - 1]); + --messageLength) { + errorText[messageLength - 1] = '\0'; + } + + RTC_LOG(LS_ERROR) << "Core Audio method failed (hr=" << hr << ")"; + StringCchPrintfW(buf, MAXERRORLENGTH, L"Error details: "); + StringCchCatW(buf, MAXERRORLENGTH, errorText); + RTC_LOG(LS_ERROR) << rtc::ToUtf8(buf); +} + +// ---------------------------------------------------------------------------- +// InitMicrophone +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::InitMicrophone() { + webrtc::MutexLock lock(&mutex_); + return InitMicrophoneLocked(); +} + +int32_t MicrophoneModule::InitMicrophoneLocked() { + if (_recording) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + + if (_usingInputDeviceIndex) { + int16_t nDevices = RecordingDevicesLocked(); + if (_inputDeviceIndex > (nDevices - 1)) { + RTC_LOG(LS_ERROR) << "current device selection is invalid => unable to" + " initialize"; + return -1; + } + } + + int32_t ret(0); + + SAFE_RELEASE(_ptrDeviceIn); + if (_usingInputDeviceIndex) { + // Refresh the selected capture endpoint device using current index + ret = _GetListDevice(eCapture, _inputDeviceIndex, &_ptrDeviceIn); + } else { + ERole role; + (_inputDevice == webrtc::AudioDeviceModule::kDefaultDevice) + ? role = eConsole + : role = eCommunications; + // Refresh the selected capture endpoint device using role + ret = _GetDefaultDevice(eCapture, role, &_ptrDeviceIn); + } + + if (ret != 0 || (_ptrDeviceIn == NULL)) { + RTC_LOG(LS_ERROR) << "failed to initialize the capturing enpoint device"; + SAFE_RELEASE(_ptrDeviceIn); + return -1; + } + + SAFE_RELEASE(_ptrCaptureVolume); + ret = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&_ptrCaptureVolume)); + if (ret != 0 || _ptrCaptureVolume == NULL) { + RTC_LOG(LS_ERROR) << "failed to initialize the capture volume"; + SAFE_RELEASE(_ptrCaptureVolume); + return -1; + } + + _microphoneIsInitialized = true; + + return 0; +} + +// ---------------------------------------------------------------------------- +// _GetListDevice +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::_GetListDevice(EDataFlow dir, + int index, + IMMDevice** ppDevice) { + HRESULT hr(S_OK); + + RTC_DCHECK(_ptrEnumerator); + + IMMDeviceCollection* pCollection = NULL; + + hr = _ptrEnumerator->EnumAudioEndpoints( + dir, + DEVICE_STATE_ACTIVE, // only active endpoints are OK + &pCollection); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pCollection); + return -1; + } + + hr = pCollection->Item(index, ppDevice); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pCollection); + return -1; + } + + SAFE_RELEASE(pCollection); + + return 0; +} + +// ---------------------------------------------------------------------------- +// _DeviceListCount +// +// Gets a count of the endpoint rendering or capture devices in the +// current list of such devices. +// ---------------------------------------------------------------------------- + +int16_t MicrophoneModule::_DeviceListCount(EDataFlow dir) { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + UINT count = 0; + RTC_DCHECK(eRender == dir || eCapture == dir); + + if (eRender == dir && NULL != _ptrRenderCollection) { + hr = _ptrRenderCollection->GetCount(&count); + } else if (NULL != _ptrCaptureCollection) { + hr = _ptrCaptureCollection->GetCount(&count); + } + + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + return static_cast(count); +} + +// ---------------------------------------------------------------------------- +// _RefreshDeviceList +// +// Creates a new list of endpoint rendering or capture devices after +// deleting any previously created (and possibly out-of-date) list of +// such devices. +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::_RefreshDeviceList(EDataFlow dir) { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + IMMDeviceCollection* pCollection = NULL; + + RTC_DCHECK(dir == eRender || dir == eCapture); + RTC_DCHECK(_ptrEnumerator); + + // Create a fresh list of devices using the specified direction + hr = _ptrEnumerator->EnumAudioEndpoints(dir, DEVICE_STATE_ACTIVE, + &pCollection); + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pCollection); + return -1; + } + + if (dir == eRender) { + SAFE_RELEASE(_ptrRenderCollection); + _ptrRenderCollection = pCollection; + } else { + SAFE_RELEASE(_ptrCaptureCollection); + _ptrCaptureCollection = pCollection; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// RecordingDevices +// ---------------------------------------------------------------------------- + +int16_t MicrophoneModule::RecordingDevices() { + webrtc::MutexLock lock(&mutex_); + return RecordingDevicesLocked(); +} + +int16_t MicrophoneModule::RecordingDevicesLocked() { + if (_RefreshDeviceList(eCapture) != -1) { + return (_DeviceListCount(eCapture)); + } + + return -1; +} + +// ---------------------------------------------------------------------------- +// _GetDeviceName +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::_GetDeviceName(IMMDevice* pDevice, + LPWSTR pszBuffer, + int bufferLen) { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + static const WCHAR szDefault[] = L""; + + HRESULT hr = E_FAIL; + IPropertyStore* pProps = NULL; + PROPVARIANT varName; + + RTC_DCHECK(pszBuffer); + RTC_DCHECK_GT(bufferLen, 0); + + if (pDevice != NULL) { + hr = pDevice->OpenPropertyStore(STGM_READ, &pProps); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "IMMDevice::OpenPropertyStore failed, hr = 0x" + << rtc::ToHex(hr); + } + } + + // Initialize container for property value. + PropVariantInit(&varName); + + if (SUCCEEDED(hr)) { + // Get the endpoint device's friendly-name property. + hr = pProps->GetValue(PKEY_Device_FriendlyName, &varName); + if (FAILED(hr)) { + RTC_LOG(LS_ERROR) << "IPropertyStore::GetValue failed, hr = 0x" + << rtc::ToHex(hr); + } + } + + if ((SUCCEEDED(hr)) && (VT_EMPTY == varName.vt)) { + hr = E_FAIL; + RTC_LOG(LS_ERROR) << "IPropertyStore::GetValue returned no value," + " hr = 0x" + << rtc::ToHex(hr); + } + + if ((SUCCEEDED(hr)) && (VT_LPWSTR != varName.vt)) { + // The returned value is not a wide null terminated string. + hr = E_UNEXPECTED; + RTC_LOG(LS_ERROR) << "IPropertyStore::GetValue returned unexpected" + " type, hr = 0x" + << rtc::ToHex(hr); + } + + if (SUCCEEDED(hr) && (varName.pwszVal != NULL)) { + // Copy the valid device name to the provided ouput buffer. + wcsncpy_s(pszBuffer, bufferLen, varName.pwszVal, _TRUNCATE); + } else { + // Failed to find the device name. + wcsncpy_s(pszBuffer, bufferLen, szDefault, _TRUNCATE); + } + + PropVariantClear(&varName); + SAFE_RELEASE(pProps); + + return 0; +} + +// ---------------------------------------------------------------------------- +// MicrophoneIsInitialized +// ---------------------------------------------------------------------------- + +bool MicrophoneModule::MicrophoneIsInitialized() { + return (_microphoneIsInitialized); +} + +// ---------------------------------------------------------------------------- +// MicrophoneVolumeIsAvailable +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::MicrophoneVolumeIsAvailable(bool* available) { + webrtc::MutexLock lock(&mutex_); + + if (_ptrDeviceIn == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + float volume(0.0f); + hr = pVolume->GetMasterVolumeLevelScalar(&volume); + if (FAILED(hr)) { + *available = false; + } + *available = true; + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// SetMicrophoneVolume +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::SetMicrophoneVolume(uint32_t volume) { + RTC_LOG(LS_VERBOSE) << "AudioDeviceWindowsCore::SetMicrophoneVolume(volume=" + << volume << ")"; + + { + webrtc::MutexLock lock(&mutex_); + + if (!_microphoneIsInitialized) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + } + + if (volume < static_cast(webrtc::MIN_CORE_MICROPHONE_VOLUME) || + volume > static_cast(webrtc::MAX_CORE_MICROPHONE_VOLUME)) { + return -1; + } + + HRESULT hr = S_OK; + // scale input volume to valid range (0.0 to 1.0) + const float fLevel = + static_cast(volume) / webrtc::MAX_CORE_MICROPHONE_VOLUME; + volume_mutex_.Lock(); + _ptrCaptureVolume->SetMasterVolumeLevelScalar(fLevel, NULL); + volume_mutex_.Unlock(); + EXIT_ON_ERROR(hr); + + return 0; + +Exit: + _TraceCOMError(hr); + return -1; +} + +// ---------------------------------------------------------------------------- +// MicrophoneVolume +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::MicrophoneVolume(uint32_t* volume) const { + { + webrtc::MutexLock lock(&mutex_); + + if (!_microphoneIsInitialized) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + } + + HRESULT hr = S_OK; + float fLevel(0.0f); + *volume = 0; + volume_mutex_.Lock(); + hr = _ptrCaptureVolume->GetMasterVolumeLevelScalar(&fLevel); + volume_mutex_.Unlock(); + EXIT_ON_ERROR(hr); + + // scale input volume range [0.0,1.0] to valid output range + *volume = static_cast(fLevel * webrtc::MAX_CORE_MICROPHONE_VOLUME); + + return 0; + +Exit: + _TraceCOMError(hr); + return -1; +} + +// ---------------------------------------------------------------------------- +// MaxMicrophoneVolume +// +// The internal range for Core Audio is 0.0 to 1.0, where 0.0 indicates +// silence and 1.0 indicates full volume (no attenuation). +// We add our (webrtc-internal) own max level to match the Wave API and +// how it is used today in VoE. +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::MaxMicrophoneVolume(uint32_t* maxVolume) const { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + if (!_microphoneIsInitialized) { + return -1; + } + + *maxVolume = static_cast(webrtc::MAX_CORE_MICROPHONE_VOLUME); + + return 0; +} + +// ---------------------------------------------------------------------------- +// MinMicrophoneVolume +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::MinMicrophoneVolume(uint32_t* minVolume) const { + if (!_microphoneIsInitialized) { + return -1; + } + + *minVolume = static_cast(webrtc::MIN_CORE_MICROPHONE_VOLUME); + + return 0; +} + +// ---------------------------------------------------------------------------- +// MicrophoneMuteIsAvailable +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::MicrophoneMuteIsAvailable(bool* available) { + webrtc::MutexLock lock(&mutex_); + + if (_ptrDeviceIn == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Query the microphone system mute state. + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + BOOL mute; + hr = pVolume->GetMute(&mute); + if (FAILED(hr)) + *available = false; + else + *available = true; + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// SetMicrophoneMute +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::SetMicrophoneMute(bool enable) { + if (!_microphoneIsInitialized) { + return -1; + } + + if (_ptrDeviceIn == NULL) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Set the microphone system mute state. + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + const BOOL mute(enable); + hr = pVolume->SetMute(mute, NULL); + EXIT_ON_ERROR(hr); + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// MicrophoneMute +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::MicrophoneMute(bool* enabled) const { + if (!_microphoneIsInitialized) { + return -1; + } + + HRESULT hr = S_OK; + IAudioEndpointVolume* pVolume = NULL; + + // Query the microphone system mute state. + hr = _ptrDeviceIn->Activate(__uuidof(IAudioEndpointVolume), CLSCTX_ALL, NULL, + reinterpret_cast(&pVolume)); + EXIT_ON_ERROR(hr); + + BOOL mute; + hr = pVolume->GetMute(&mute); + EXIT_ON_ERROR(hr); + + *enabled = (mute == TRUE) ? true : false; + + SAFE_RELEASE(pVolume); + return 0; + +Exit: + _TraceCOMError(hr); + SAFE_RELEASE(pVolume); + return -1; +} + +// ---------------------------------------------------------------------------- +// InitCaptureThreadPriority +// ---------------------------------------------------------------------------- + +DWORD MicrophoneModule::InitCaptureThreadPriority() { + _hMmTask = NULL; + + rtc::SetCurrentThreadName("webrtc_core_audio_capture_thread"); + + // Use Multimedia Class Scheduler Service (MMCSS) to boost the thread + // priority. + if (_winSupportAvrt) { + DWORD taskIndex(0); + _hMmTask = _PAvSetMmThreadCharacteristicsA("Pro Audio", &taskIndex); + if (_hMmTask) { + if (!_PAvSetMmThreadPriority(_hMmTask, AVRT_PRIORITY_CRITICAL)) { + RTC_LOG(LS_WARNING) << "failed to boost rec-thread using MMCSS"; + } + RTC_LOG(LS_VERBOSE) + << "capture thread is now registered with MMCSS (taskIndex=" + << taskIndex << ")"; + } else { + RTC_LOG(LS_WARNING) << "failed to enable MMCSS on capture thread (err=" + << GetLastError() << ")"; + _TraceCOMError(GetLastError()); + } + } + + return S_OK; +} + +// ---------------------------------------------------------------------------- +// RevertCaptureThreadPriority +// ---------------------------------------------------------------------------- + +void MicrophoneModule::RevertCaptureThreadPriority() { + if (_winSupportAvrt) { + if (NULL != _hMmTask) { + _PAvRevertMmThreadCharacteristics(_hMmTask); + } + } + + _hMmTask = NULL; +} + +// ---------------------------------------------------------------------------- +// InitRecording +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::InitRecording() { + webrtc::MutexLock lock(&mutex_); + + if (_recording) { + return -1; + } + + if (_recIsInitialized) { + return 0; + } + + if (QueryPerformanceFrequency(&_perfCounterFreq) == 0) { + return -1; + } + _perfCounterFactor = 10000000.0 / (double)_perfCounterFreq.QuadPart; + + if (_ptrDeviceIn == NULL) { + return -1; + } + + // Initialize the microphone (devices might have been added or removed) + if (InitMicrophoneLocked() == -1) { + RTC_LOG(LS_WARNING) << "InitMicrophone() failed"; + } + + // Ensure that the updated capturing endpoint device is valid + if (_ptrDeviceIn == NULL) { + return -1; + } + + if (_builtInAecEnabled) { + // The DMO will configure the capture device. + return InitRecordingDMO(); + } + + HRESULT hr = S_OK; + WAVEFORMATEX* pWfxIn = NULL; + WAVEFORMATEXTENSIBLE Wfx = WAVEFORMATEXTENSIBLE(); + WAVEFORMATEX* pWfxClosestMatch = NULL; + + // Create COM object with IAudioClient interface. + SAFE_RELEASE(_ptrClientIn); + hr = _ptrDeviceIn->Activate(__uuidof(IAudioClient), CLSCTX_ALL, NULL, + (void**)&_ptrClientIn); + EXIT_ON_ERROR(hr); + + // Retrieve the stream format that the audio engine uses for its internal + // processing (mixing) of shared-mode streams. + hr = _ptrClientIn->GetMixFormat(&pWfxIn); + if (SUCCEEDED(hr)) { + RTC_LOG(LS_VERBOSE) << "Audio Engine's current capturing mix format:"; + // format type + RTC_LOG(LS_VERBOSE) << "wFormatTag : 0x" + << rtc::ToHex(pWfxIn->wFormatTag) << " (" + << pWfxIn->wFormatTag << ")"; + // number of channels (i.e. mono, stereo...) + RTC_LOG(LS_VERBOSE) << "nChannels : " << pWfxIn->nChannels; + // sample rate + RTC_LOG(LS_VERBOSE) << "nSamplesPerSec : " << pWfxIn->nSamplesPerSec; + // for buffer estimation + RTC_LOG(LS_VERBOSE) << "nAvgBytesPerSec: " << pWfxIn->nAvgBytesPerSec; + // block size of data + RTC_LOG(LS_VERBOSE) << "nBlockAlign : " << pWfxIn->nBlockAlign; + // number of bits per sample of mono data + RTC_LOG(LS_VERBOSE) << "wBitsPerSample : " << pWfxIn->wBitsPerSample; + RTC_LOG(LS_VERBOSE) << "cbSize : " << pWfxIn->cbSize; + } + + // Set wave format + Wfx.Format.wFormatTag = WAVE_FORMAT_EXTENSIBLE; + Wfx.Format.wBitsPerSample = 16; + Wfx.Format.cbSize = 22; + Wfx.dwChannelMask = 0; + Wfx.Samples.wValidBitsPerSample = Wfx.Format.wBitsPerSample; + Wfx.SubFormat = KSDATAFORMAT_SUBTYPE_PCM; + + const int freqs[6] = {48000, 44100, 16000, 96000, 32000, 8000}; + hr = S_FALSE; + + // Iterate over frequencies and channels, in order of priority + for (unsigned int freq = 0; freq < sizeof(freqs) / sizeof(freqs[0]); freq++) { + for (unsigned int chan = 0; + chan < sizeof(_recChannelsPrioList) / sizeof(_recChannelsPrioList[0]); + chan++) { + Wfx.Format.nChannels = _recChannelsPrioList[chan]; + Wfx.Format.nSamplesPerSec = freqs[freq]; + Wfx.Format.nBlockAlign = + Wfx.Format.nChannels * Wfx.Format.wBitsPerSample / 8; + Wfx.Format.nAvgBytesPerSec = + Wfx.Format.nSamplesPerSec * Wfx.Format.nBlockAlign; + // If the method succeeds and the audio endpoint device supports the + // specified stream format, it returns S_OK. If the method succeeds and + // provides a closest match to the specified format, it returns S_FALSE. + hr = _ptrClientIn->IsFormatSupported( + AUDCLNT_SHAREMODE_SHARED, (WAVEFORMATEX*)&Wfx, &pWfxClosestMatch); + if (hr == S_OK) { + break; + } else { + if (pWfxClosestMatch) { + RTC_LOG(LS_INFO) << "nChannels=" << Wfx.Format.nChannels + << ", nSamplesPerSec=" << Wfx.Format.nSamplesPerSec + << " is not supported. Closest match: " + "nChannels=" + << pWfxClosestMatch->nChannels << ", nSamplesPerSec=" + << pWfxClosestMatch->nSamplesPerSec; + CoTaskMemFree(pWfxClosestMatch); + pWfxClosestMatch = NULL; + } else { + RTC_LOG(LS_INFO) << "nChannels=" << Wfx.Format.nChannels + << ", nSamplesPerSec=" << Wfx.Format.nSamplesPerSec + << " is not supported. No closest match."; + } + } + } + if (hr == S_OK) + break; + } + + if (hr == S_OK) { + _recAudioFrameSize = Wfx.Format.nBlockAlign; + _recSampleRate = Wfx.Format.nSamplesPerSec; + _recBlockSize = Wfx.Format.nSamplesPerSec / 100; + _recChannels = Wfx.Format.nChannels; + + RTC_LOG(LS_VERBOSE) << "VoE selected this capturing format:"; + RTC_LOG(LS_VERBOSE) << "wFormatTag : 0x" + << rtc::ToHex(Wfx.Format.wFormatTag) << " (" + << Wfx.Format.wFormatTag << ")"; + RTC_LOG(LS_VERBOSE) << "nChannels : " << Wfx.Format.nChannels; + RTC_LOG(LS_VERBOSE) << "nSamplesPerSec : " << Wfx.Format.nSamplesPerSec; + RTC_LOG(LS_VERBOSE) << "nAvgBytesPerSec : " << Wfx.Format.nAvgBytesPerSec; + RTC_LOG(LS_VERBOSE) << "nBlockAlign : " << Wfx.Format.nBlockAlign; + RTC_LOG(LS_VERBOSE) << "wBitsPerSample : " << Wfx.Format.wBitsPerSample; + RTC_LOG(LS_VERBOSE) << "cbSize : " << Wfx.Format.cbSize; + RTC_LOG(LS_VERBOSE) << "Additional settings:"; + RTC_LOG(LS_VERBOSE) << "_recAudioFrameSize: " << _recAudioFrameSize; + RTC_LOG(LS_VERBOSE) << "_recBlockSize : " << _recBlockSize; + RTC_LOG(LS_VERBOSE) << "_recChannels : " << _recChannels; + } + + // Create a capturing stream. + hr = _ptrClientIn->Initialize( + AUDCLNT_SHAREMODE_SHARED, // share Audio Engine with other applications + AUDCLNT_STREAMFLAGS_EVENTCALLBACK | // processing of the audio buffer by + // the client will be event driven + AUDCLNT_STREAMFLAGS_NOPERSIST, // volume and mute settings for an + // audio session will not persist + // across system restarts + 0, // required for event-driven shared mode + 0, // periodicity + (WAVEFORMATEX*)&Wfx, // selected wave format + NULL); // session GUID + + if (hr != S_OK) { + RTC_LOG(LS_ERROR) << "IAudioClient::Initialize() failed:"; + } + EXIT_ON_ERROR(hr); + + // Get the actual size of the shared (endpoint buffer). + // Typical value is 960 audio frames <=> 20ms @ 48kHz sample rate. + UINT bufferFrameCount(0); + hr = _ptrClientIn->GetBufferSize(&bufferFrameCount); + if (SUCCEEDED(hr)) { + RTC_LOG(LS_VERBOSE) << "IAudioClient::GetBufferSize() => " + << bufferFrameCount << " (<=> " + << bufferFrameCount * _recAudioFrameSize << " bytes)"; + } + + // Set the event handle that the system signals when an audio buffer is ready + // to be processed by the client. + hr = _ptrClientIn->SetEventHandle(_hCaptureSamplesReadyEvent); + EXIT_ON_ERROR(hr); + + // Get an IAudioCaptureClient interface. + SAFE_RELEASE(_ptrCaptureClient); + hr = _ptrClientIn->GetService(__uuidof(IAudioCaptureClient), + (void**)&_ptrCaptureClient); + EXIT_ON_ERROR(hr); + + // Mark capture side as initialized + _recIsInitialized = true; + + CoTaskMemFree(pWfxIn); + CoTaskMemFree(pWfxClosestMatch); + + RTC_LOG(LS_VERBOSE) << "capture side is now initialized"; + return 0; + +Exit: + _TraceCOMError(hr); + CoTaskMemFree(pWfxIn); + CoTaskMemFree(pWfxClosestMatch); + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrCaptureClient); + return -1; +} + +int MicrophoneModule::SetVtI4Property(IPropertyStore* ptrPS, + REFPROPERTYKEY key, + LONG value) { + PROPVARIANT pv; + PropVariantInit(&pv); + pv.vt = VT_I4; + pv.lVal = value; + HRESULT hr = ptrPS->SetValue(key, pv); + PropVariantClear(&pv); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + return 0; +} + +int MicrophoneModule::SetBoolProperty(IPropertyStore* ptrPS, + REFPROPERTYKEY key, + VARIANT_BOOL value) { + PROPVARIANT pv; + PropVariantInit(&pv); + pv.vt = VT_BOOL; + pv.boolVal = value; + HRESULT hr = ptrPS->SetValue(key, pv); + PropVariantClear(&pv); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + return 0; +} + +int MicrophoneModule::SetDMOProperties() { + HRESULT hr = S_OK; + RTC_DCHECK(_dmo); + + rtc::scoped_refptr ps; + { + IPropertyStore* ptrPS = NULL; + hr = _dmo->QueryInterface(IID_IPropertyStore, + reinterpret_cast(&ptrPS)); + if (FAILED(hr) || ptrPS == NULL) { + _TraceCOMError(hr); + return -1; + } + ps = ptrPS; + SAFE_RELEASE(ptrPS); + } + + // Set the AEC system mode. + // SINGLE_CHANNEL_AEC - AEC processing only. + if (SetVtI4Property(ps.get(), MFPKEY_WMAAECMA_SYSTEM_MODE, + SINGLE_CHANNEL_AEC)) { + return -1; + } + + // Set the AEC source mode. + // VARIANT_TRUE - Source mode (we poll the AEC for captured data). + if (SetBoolProperty(ps.get(), MFPKEY_WMAAECMA_DMO_SOURCE_MODE, + VARIANT_TRUE) == -1) { + return -1; + } + + // Enable the feature mode. + // This lets us override all the default processing settings below. + if (SetBoolProperty(ps.get(), MFPKEY_WMAAECMA_FEATURE_MODE, VARIANT_TRUE) == + -1) { + return -1; + } + + // Disable analog AGC (default enabled). + if (SetBoolProperty(ps.get(), MFPKEY_WMAAECMA_MIC_GAIN_BOUNDER, + VARIANT_FALSE) == -1) { + return -1; + } + + // Disable noise suppression (default enabled). + // 0 - Disabled, 1 - Enabled + if (SetVtI4Property(ps.get(), MFPKEY_WMAAECMA_FEATR_NS, 0) == -1) { + return -1; + } + + // Relevant parameters to leave at default settings: + // MFPKEY_WMAAECMA_FEATR_AGC - Digital AGC (disabled). + // MFPKEY_WMAAECMA_FEATR_CENTER_CLIP - AEC center clipping (enabled). + // MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH - Filter length (256 ms). + // TODO(andrew): investigate decresing the length to 128 ms. + // MFPKEY_WMAAECMA_FEATR_FRAME_SIZE - Frame size (0). + // 0 is automatic; defaults to 160 samples (or 10 ms frames at the + // selected 16 kHz) as long as mic array processing is disabled. + // MFPKEY_WMAAECMA_FEATR_NOISE_FILL - Comfort noise (enabled). + // MFPKEY_WMAAECMA_FEATR_VAD - VAD (disabled). + + // Set the devices selected by VoE. If using a default device, we need to + // search for the device index. + int inDevIndex = _inputDeviceIndex; + int outDevIndex = _outputDeviceIndex; + if (!_usingInputDeviceIndex) { + ERole role = eCommunications; + if (_inputDevice == webrtc::AudioDeviceModule::kDefaultDevice) { + role = eConsole; + } + + if (_GetDefaultDeviceIndex(eCapture, role, &inDevIndex) == -1) { + return -1; + } + } + + if (!_usingOutputDeviceIndex) { + ERole role = eCommunications; + if (_outputDevice == webrtc::AudioDeviceModule::kDefaultDevice) { + role = eConsole; + } + + if (_GetDefaultDeviceIndex(eRender, role, &outDevIndex) == -1) { + return -1; + } + } + + DWORD devIndex = static_cast(outDevIndex << 16) + + static_cast(0x0000ffff & inDevIndex); + RTC_LOG(LS_VERBOSE) << "Capture device index: " << inDevIndex + << ", render device index: " << outDevIndex; + if (SetVtI4Property(ps.get(), MFPKEY_WMAAECMA_DEVICE_INDEXES, devIndex) == + -1) { + return -1; + } + + return 0; +} + +int32_t MicrophoneModule::_GetDefaultDeviceIndex(EDataFlow dir, + ERole role, + int* index) { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + WCHAR szDefaultDeviceID[MAX_PATH] = {0}; + WCHAR szDeviceID[MAX_PATH] = {0}; + + const size_t kDeviceIDLength = sizeof(szDeviceID) / sizeof(szDeviceID[0]); + RTC_DCHECK_EQ(kDeviceIDLength, + sizeof(szDefaultDeviceID) / sizeof(szDefaultDeviceID[0])); + + if (_GetDefaultDeviceID(dir, role, szDefaultDeviceID, kDeviceIDLength) == + -1) { + return -1; + } + + IMMDeviceCollection* collection = _ptrCaptureCollection; + if (dir == eRender) { + collection = _ptrRenderCollection; + } + + if (!collection) { + RTC_LOG(LS_ERROR) << "Device collection not valid"; + return -1; + } + + UINT count = 0; + hr = collection->GetCount(&count); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + *index = -1; + for (UINT i = 0; i < count; i++) { + memset(szDeviceID, 0, sizeof(szDeviceID)); + rtc::scoped_refptr device; + { + IMMDevice* ptrDevice = NULL; + hr = collection->Item(i, &ptrDevice); + if (FAILED(hr) || ptrDevice == NULL) { + _TraceCOMError(hr); + return -1; + } + device = ptrDevice; + SAFE_RELEASE(ptrDevice); + } + + if (_GetDeviceID(device.get(), szDeviceID, kDeviceIDLength) == -1) { + return -1; + } + + if (wcsncmp(szDefaultDeviceID, szDeviceID, kDeviceIDLength) == 0) { + // Found a match. + *index = i; + break; + } + } + + if (*index == -1) { + RTC_LOG(LS_ERROR) << "Unable to find collection index for default device"; + return -1; + } + + return 0; +} + +// ---------------------------------------------------------------------------- +// _GetDefaultDeviceID +// +// Gets the uniqe device ID of an endpoint rendering or capture device +// given a specified device role. +// +// Uses: _ptrEnumerator +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::_GetDefaultDeviceID(EDataFlow dir, + ERole role, + LPWSTR szBuffer, + int bufferLen) { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr = S_OK; + IMMDevice* pDevice = NULL; + + RTC_DCHECK(dir == eRender || dir == eCapture); + RTC_DCHECK(role == eConsole || role == eCommunications); + RTC_DCHECK(_ptrEnumerator); + + hr = _ptrEnumerator->GetDefaultAudioEndpoint(dir, role, &pDevice); + + if (FAILED(hr)) { + _TraceCOMError(hr); + SAFE_RELEASE(pDevice); + return -1; + } + + int32_t res = _GetDeviceID(pDevice, szBuffer, bufferLen); + SAFE_RELEASE(pDevice); + return res; +} + +// ---------------------------------------------------------------------------- +// _GetDeviceID +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::_GetDeviceID(IMMDevice* pDevice, + LPWSTR pszBuffer, + int bufferLen) { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + static const WCHAR szDefault[] = L""; + + HRESULT hr = E_FAIL; + LPWSTR pwszID = NULL; + + RTC_DCHECK(pszBuffer); + RTC_DCHECK_GT(bufferLen, 0); + + if (pDevice != NULL) { + hr = pDevice->GetId(&pwszID); + } + + if (hr == S_OK) { + // Found the device ID. + wcsncpy_s(pszBuffer, bufferLen, pwszID, _TRUNCATE); + } else { + // Failed to find the device ID. + wcsncpy_s(pszBuffer, bufferLen, szDefault, _TRUNCATE); + } + + CoTaskMemFree(pwszID); + return 0; +} + +// ---------------------------------------------------------------------------- +// _GetDefaultDevice +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::_GetDefaultDevice(EDataFlow dir, + ERole role, + IMMDevice** ppDevice) { + RTC_DLOG(LS_VERBOSE) << __FUNCTION__; + + HRESULT hr(S_OK); + + RTC_DCHECK(_ptrEnumerator); + + hr = _ptrEnumerator->GetDefaultAudioEndpoint(dir, role, ppDevice); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + return 0; +} + +// Capture initialization when the built-in AEC DirectX Media Object (DMO) is +// used. Called from InitRecording(), most of which is skipped over. The DMO +// handles device initialization itself. +// Reference: http://msdn.microsoft.com/en-us/library/ff819492(v=vs.85).aspx +int32_t MicrophoneModule::InitRecordingDMO() { + RTC_DCHECK(_builtInAecEnabled); + RTC_DCHECK(_dmo); + + if (SetDMOProperties() == -1) { + return -1; + } + + DMO_MEDIA_TYPE mt = {}; + HRESULT hr = MoInitMediaType(&mt, sizeof(WAVEFORMATEX)); + if (FAILED(hr)) { + MoFreeMediaType(&mt); + _TraceCOMError(hr); + return -1; + } + mt.majortype = MEDIATYPE_Audio; + mt.subtype = MEDIASUBTYPE_PCM; + mt.formattype = FORMAT_WaveFormatEx; + + // Supported formats + // nChannels: 1 (in AEC-only mode) + // nSamplesPerSec: 8000, 11025, 16000, 22050 + // wBitsPerSample: 16 + WAVEFORMATEX* ptrWav = reinterpret_cast(mt.pbFormat); + ptrWav->wFormatTag = WAVE_FORMAT_PCM; + ptrWav->nChannels = 1; + // 16000 is the highest we can support with our resampler. + ptrWav->nSamplesPerSec = 16000; + ptrWav->nAvgBytesPerSec = 32000; + ptrWav->nBlockAlign = 2; + ptrWav->wBitsPerSample = 16; + ptrWav->cbSize = 0; + + // Set the VoE format equal to the AEC output format. + _recAudioFrameSize = ptrWav->nBlockAlign; + _recSampleRate = ptrWav->nSamplesPerSec; + _recBlockSize = ptrWav->nSamplesPerSec / 100; + _recChannels = ptrWav->nChannels; + + // Set the DMO output format parameters. + hr = _dmo->SetOutputType(webrtc::kAecCaptureStreamIndex, &mt, 0); + MoFreeMediaType(&mt); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + _mediaBuffer = rtc::make_ref_counted( + _recBlockSize * _recAudioFrameSize); + + // Optional, but if called, must be after media types are set. + hr = _dmo->AllocateStreamingResources(); + if (FAILED(hr)) { + _TraceCOMError(hr); + return -1; + } + + _recIsInitialized = true; + RTC_LOG(LS_VERBOSE) << "Capture side is now initialized"; + + return 0; +} + +// ---------------------------------------------------------------------------- +// [static] WSAPICaptureThread +// ---------------------------------------------------------------------------- + +DWORD WINAPI MicrophoneModule::WSAPICaptureThread(LPVOID context) { + return reinterpret_cast(context)->DoCaptureThread(); +} + +// ---------------------------------------------------------------------------- +// StartRecording +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::StartRecording() { + if (!_recIsInitialized) { + return -1; + } + + if (_hRecThread != NULL) { + return 0; + } + + if (_recording) { + return 0; + } + + { + webrtc::MutexLock lockScoped(&mutex_); + + // Create thread which will drive the capturing + LPTHREAD_START_ROUTINE lpStartAddress = WSAPICaptureThread; + if (_builtInAecEnabled) { + // Redirect to the DMO polling method. + lpStartAddress = WSAPICaptureThreadPollDMO; + + if (!_playing) { + // The DMO won't provide us captured output data unless we + // give it render data to process. + RTC_LOG(LS_ERROR) + << "Playout must be started before recording when using" + " the built-in AEC"; + return -1; + } + } + + RTC_DCHECK(_hRecThread == NULL); + _hRecThread = CreateThread(NULL, 0, lpStartAddress, this, 0, NULL); + if (_hRecThread == NULL) { + RTC_LOG(LS_ERROR) << "failed to create the recording thread"; + return -1; + } + + // Set thread priority to highest possible + SetThreadPriority(_hRecThread, THREAD_PRIORITY_TIME_CRITICAL); + } // critScoped + + DWORD ret = WaitForSingleObject(_hCaptureStartedEvent, 1000); + if (ret != WAIT_OBJECT_0) { + RTC_LOG(LS_VERBOSE) << "capturing did not start up properly"; + return -1; + } + RTC_LOG(LS_VERBOSE) << "capture audio stream has now started..."; + + _recording = true; + + return 0; +} + +DWORD WINAPI MicrophoneModule::WSAPICaptureThreadPollDMO(LPVOID context) { + return reinterpret_cast(context)->DoCaptureThreadPollDMO(); +} + +// ---------------------------------------------------------------------------- +// StopRecording +// ---------------------------------------------------------------------------- + +int32_t MicrophoneModule::StopRecording() { + int32_t err = 0; + + if (!_recIsInitialized) { + return 0; + } + + mutex_.Lock(); + + if (_hRecThread == NULL) { + RTC_LOG(LS_VERBOSE) + << "no capturing stream is active => close down WASAPI only"; + SAFE_RELEASE(_ptrClientIn); + SAFE_RELEASE(_ptrCaptureClient); + _recIsInitialized = false; + _recording = false; + mutex_.Unlock(); + return 0; + } + + // Stop the driving thread... + RTC_LOG(LS_VERBOSE) << "closing down the webrtc_core_audio_capture_thread..."; + // Manual-reset event; it will remain signalled to stop all capture threads. + SetEvent(_hShutdownCaptureEvent); + + mutex_.Unlock(); + DWORD ret = WaitForSingleObject(_hRecThread, 2000); + if (ret != WAIT_OBJECT_0) { + RTC_LOG(LS_ERROR) + << "failed to close down webrtc_core_audio_capture_thread"; + err = -1; + } else { + RTC_LOG(LS_VERBOSE) << "webrtc_core_audio_capture_thread is now closed"; + } + mutex_.Lock(); + + ResetEvent(_hShutdownCaptureEvent); // Must be manually reset. + // Ensure that the thread has released these interfaces properly. + RTC_DCHECK(err == -1 || _ptrClientIn == NULL); + RTC_DCHECK(err == -1 || _ptrCaptureClient == NULL); + + _recIsInitialized = false; + _recording = false; + + // These will create thread leaks in the result of an error, + // but we can at least resume the call. + CloseHandle(_hRecThread); + _hRecThread = NULL; + + if (_builtInAecEnabled) { + RTC_DCHECK(_dmo); + // This is necessary. Otherwise the DMO can generate garbage render + // audio even after rendering has stopped. + HRESULT hr = _dmo->FreeStreamingResources(); + if (FAILED(hr)) { + _TraceCOMError(hr); + err = -1; + } + } + + mutex_.Unlock(); + + return err; +} + +#endif WEBRTC_WINDOWS_CORE_AUDIO_BUILD +#endif \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/cpp/windows_system_audio_module.cc b/crates/libwebrtc-sys/src/cpp/windows_system_audio_module.cc new file mode 100644 index 0000000000..e11b52c5cc --- /dev/null +++ b/crates/libwebrtc-sys/src/cpp/windows_system_audio_module.cc @@ -0,0 +1,563 @@ + +#if WEBRTC_WIN +#include "windows_system_audio_module.h" + +DWORD GetSpeakerChannelMask(speaker_layout layout) { + switch (layout) { + case SPEAKERS_STEREO: + return KSAUDIO_SPEAKER_STEREO; + case SPEAKERS_2POINT1: + return KSAUDIO_SPEAKER_2POINT1; + case SPEAKERS_4POINT0: + return KSAUDIO_SPEAKER_SURROUND; + case SPEAKERS_4POINT1: + return OBS_KSAUDIO_SPEAKER_4POINT1; + case SPEAKERS_5POINT1: + return KSAUDIO_SPEAKER_5POINT1_SURROUND; + case SPEAKERS_7POINT1: + return KSAUDIO_SPEAKER_7POINT1_SURROUND; + } + + return (DWORD)layout; +} + +speaker_layout SystemModule::ConvertSpeakerLayout(DWORD layout, WORD channels) { + switch (layout) { + case KSAUDIO_SPEAKER_2POINT1: + return SPEAKERS_2POINT1; + case KSAUDIO_SPEAKER_SURROUND: + return SPEAKERS_4POINT0; + case OBS_KSAUDIO_SPEAKER_4POINT1: + return SPEAKERS_4POINT1; + case KSAUDIO_SPEAKER_5POINT1_SURROUND: + return SPEAKERS_5POINT1; + case KSAUDIO_SPEAKER_7POINT1_SURROUND: + return SPEAKERS_7POINT1; + } + + return (speaker_layout)channels; +} + +int GetChannels(speaker_layout layout) { + return (int)layout; +} + +WASAPIActivateAudioInterfaceCompletionHandler:: + WASAPIActivateAudioInterfaceCompletionHandler() { + activationSignal = CreateEvent(nullptr, false, false, nullptr); + if (!activationSignal.Valid()) + throw "Could not create receive signal"; +} + +HRESULT +WASAPIActivateAudioInterfaceCompletionHandler::GetActivateResult( + IAudioClient** client) { + WaitForSingleObject(activationSignal, INFINITE); + *client = static_cast(unknown); + return activationResult; +} + +HRESULT +WASAPIActivateAudioInterfaceCompletionHandler::ActivateCompleted( + IActivateAudioInterfaceAsyncOperation* activateOperation) { + HRESULT hr, hr_activate; + hr = activateOperation->GetActivateResult(&hr_activate, &unknown); + hr = SUCCEEDED(hr) ? hr_activate : hr; + activationResult = hr; + + SetEvent(activationSignal); + return hr; +} + +void ReorderingBuffer(std::vector& output, + float* data, + int channels, + float audio_multiplier) { + for (int i = 0; i < channels; ++i) { + int k = i; + for (int j = 480 * i; k < 480 * channels; ++j, k += channels) { + float inSample = data[k] * audio_multiplier; + if (inSample > 1.0) + inSample = 1.0; + if (inSample < -1.0) + inSample = -1.0; + + if (inSample >= 0) { + output[j] = (int16_t)lrintf(inSample * 32767.0); + } else { + output[j] = (int16_t)lrintf(inSample * 32768.0); + } + } + } +} + +int SystemSource::sources_num = 0; +SystemSource::SystemSource(SystemModuleInterface* module) { + this->module = module; + if (sources_num == 0) { + module->StartRecording(); + } + ++sources_num; +} + +SystemSource::~SystemSource() { + --sources_num; + if (sources_num == 0) { + module->StopRecording(); + module->ResetSource(); + } +} + +SystemModule::SystemModule() { + mmdevapi_module = LoadLibrary("Mmdevapi"); + if (mmdevapi_module) { + activate_audio_interface_async = + (PFN_ActivateAudioInterfaceAsync)GetProcAddress( + mmdevapi_module, "ActivateAudioInterfaceAsync"); + } + + idleSignal = CreateEvent(nullptr, true, false, nullptr); + if (!idleSignal.Valid()) + throw "Could not create idle signal"; + + stopSignal = CreateEvent(nullptr, true, false, nullptr); + if (!stopSignal.Valid()) + throw "Could not create stop signal"; + + receiveSignal = CreateEvent(nullptr, false, false, nullptr); + if (!receiveSignal.Valid()) + throw "Could not create receive signal"; + + restartSignal = CreateEvent(nullptr, true, false, nullptr); + if (!restartSignal.Valid()) + throw "Could not create restart signal"; + + reconnectExitSignal = CreateEvent(nullptr, true, false, nullptr); + if (!reconnectExitSignal.Valid()) + throw "Could not create reconnect exit signal"; + + exitSignal = CreateEvent(nullptr, true, false, nullptr); + if (!exitSignal.Valid()) + throw "Could not create exit signal"; + + initSignal = CreateEvent(nullptr, false, false, nullptr); + if (!initSignal.Valid()) + throw "Could not create init signal"; + + reconnectSignal = CreateEvent(nullptr, false, false, nullptr); + if (!reconnectSignal.Valid()) + throw "Could not create reconnect signal"; + + captureThread = + CreateThread(nullptr, 0, SystemModule::CaptureThread, this, 0, nullptr); + + if (!reconnectThread.Valid()) { + ResetEvent(reconnectExitSignal); + reconnectThread = CreateThread(nullptr, 0, SystemModule::ReconnectThread, + this, 0, nullptr); + } + + if (!captureThread.Valid()) { + throw "Failed to create capture thread"; + } + + StartRecording(); +} + +SystemModule::~SystemModule() { + Terminate(); +} + +ComPtr SystemModule::InitClient( + DWORD process_id, + PFN_ActivateAudioInterfaceAsync activate_audio_interface_async, + speaker_layout& channels, + int& format, + uint32_t& samples_per_sec) { + WAVEFORMATEXTENSIBLE wfextensible; + const WAVEFORMATEX* pFormat; + HRESULT res; + ComPtr client; + + if (activate_audio_interface_async == NULL) { + throw "ActivateAudioInterfaceAsync is not available"; + } + + const WORD nChannels = (WORD)channels; + const DWORD nSamplesPerSec = 48000; + constexpr WORD wBitsPerSample = 32; + const WORD nBlockAlign = nChannels * wBitsPerSample / 8; + + WAVEFORMATEX& wf = wfextensible.Format; + wf.wFormatTag = WAVE_FORMAT_EXTENSIBLE; + wf.nChannels = nChannels; + wf.nSamplesPerSec = nSamplesPerSec; + wf.nAvgBytesPerSec = nSamplesPerSec * nBlockAlign; + wf.nBlockAlign = nBlockAlign; + wf.wBitsPerSample = wBitsPerSample; + wf.cbSize = sizeof(wfextensible) - sizeof(format); + wfextensible.Samples.wValidBitsPerSample = wBitsPerSample; + wfextensible.dwChannelMask = + GetSpeakerChannelMask(speaker_layout::SPEAKERS_2POINT1); + wfextensible.SubFormat = KSDATAFORMAT_SUBTYPE_IEEE_FLOAT; + + AUDIOCLIENT_ACTIVATION_PARAMS audioclientActivationParams; + audioclientActivationParams.ActivationType = + AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK; + + audioclientActivationParams.ProcessLoopbackParams.TargetProcessId = + process_id; + + audioclientActivationParams.ProcessLoopbackParams.ProcessLoopbackMode = + PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE; + PROPVARIANT activateParams{}; + activateParams.vt = VT_BLOB; + activateParams.blob.cbSize = sizeof(audioclientActivationParams); + activateParams.blob.pBlobData = + reinterpret_cast(&audioclientActivationParams); + + { + Microsoft::WRL::ComPtr + handler = Microsoft::WRL::Make< + WASAPIActivateAudioInterfaceCompletionHandler>(); + ComPtr asyncOp; + res = activate_audio_interface_async( + VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, __uuidof(IAudioClient), + &activateParams, handler.Get(), &asyncOp); + + if (FAILED(res)) { + throw HRError("Failed to get activate audio client", res); + } + + res = handler->GetActivateResult(client.Assign()); + if (FAILED(res)) { + throw HRError("Async activation failed", res); + } + } + + pFormat = &wf; + + InitFormat(pFormat, channels, format, samples_per_sec); + + DWORD flags = AUDCLNT_STREAMFLAGS_EVENTCALLBACK; + flags |= AUDCLNT_STREAMFLAGS_LOOPBACK; + res = client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, BUFFER_TIME_100NS, + 0, pFormat, nullptr); + if (FAILED(res)) + throw HRError("Failed to initialize audio client", res); + + return client; +} + +DWORD WINAPI SystemModule::ReconnectThread(LPVOID param) { + SystemModule* source = (SystemModule*)param; + + const HANDLE sigs[] = { + source->reconnectExitSignal, + source->reconnectSignal, + }; + + const HANDLE reconnect_sigs[] = { + source->reconnectExitSignal, + source->stopSignal, + }; + + bool exit = false; + while (!exit) { + const DWORD ret = + WaitForMultipleObjects(_countof(sigs), sigs, false, INFINITE); + switch (ret) { + case WAIT_OBJECT_0: + exit = true; + break; + default: + assert(ret == (WAIT_OBJECT_0 + 1)); + if (source->reconnectDuration > 0) { + WaitForMultipleObjects(_countof(reconnect_sigs), reconnect_sigs, + false, source->reconnectDuration); + } + SetEvent(source->initSignal); + } + } + + return 0; +} + +DWORD WINAPI SystemModule::CaptureThread(LPVOID param) { + const HRESULT hr = CoInitializeEx(0, COINIT_MULTITHREADED); + const bool com_initialized = SUCCEEDED(hr); + + DWORD unused = 0; + const HANDLE handle = AvSetMmThreadCharacteristics("Audio", &unused); + + SystemModule* source = (SystemModule*)param; + + const HANDLE inactive_sigs[] = { + source->exitSignal, + source->stopSignal, + source->initSignal, + }; + + const HANDLE active_sigs[] = { + source->exitSignal, + source->stopSignal, + source->receiveSignal, + source->restartSignal, + }; + + DWORD sig_count = _countof(inactive_sigs); + const HANDLE* sigs = inactive_sigs; + + bool exit = false; + while (!exit) { + bool idle = false; + bool stop = false; + bool reconnect = false; + do { + /* Windows 7 does not seem to wake up for LOOPBACK */ + const DWORD dwMilliseconds = sigs == active_sigs ? 10 : INFINITE; + const DWORD ret = + WaitForMultipleObjects(sig_count, sigs, false, dwMilliseconds); + switch (ret) { + case WAIT_OBJECT_0: { + exit = true; + stop = true; + idle = true; + break; + } + + case WAIT_OBJECT_0 + 1: + stop = true; + idle = true; + break; + + case WAIT_OBJECT_0 + 2: + case WAIT_TIMEOUT: + if (sigs == inactive_sigs) { + assert(ret != WAIT_TIMEOUT); + if (source->Init()) { + sig_count = _countof(active_sigs); + sigs = active_sigs; + } else { + stop = true; + reconnect = true; + source->reconnectDuration = RECONNECT_INTERVAL; + } + } else { + stop = !source->ProcessCaptureData(); + if (stop) { + stop = true; + reconnect = true; + source->reconnectDuration = RECONNECT_INTERVAL; + } + } + break; + + default: + assert(sigs == active_sigs); + assert(ret == WAIT_OBJECT_0 + 3); + stop = true; + reconnect = true; + source->reconnectDuration = 0; + ResetEvent(source->restartSignal); + } + } while (!stop); + + sig_count = _countof(inactive_sigs); + sigs = inactive_sigs; + + if (source->client) { + source->client->Stop(); + + source->capture.Clear(); + source->client.Clear(); + } + + if (idle) { + SetEvent(source->idleSignal); + } else if (reconnect) { + SetEvent(source->reconnectSignal); + } + } + + if (handle) { + AvRevertMmThreadCharacteristics(handle); + } + + if (com_initialized) { + CoUninitialize(); + } + + return 0; +} + +bool SystemModule::ProcessCaptureData() { + if (source) { + int channels = GetChannels(speakers); + if (stop) { + return false; + } + + HRESULT res; + LPBYTE buffer; + UINT32 frames; + DWORD flags; + UINT64 pos, ts; + UINT captureSize = 0; + + while (true) { + res = capture->GetNextPacketSize(&captureSize); + if (FAILED(res)) { + if (res != AUDCLNT_E_DEVICE_INVALIDATED) + return false; + } + + if (!captureSize) { + source->Mute(); + break; + } + + res = capture->GetBuffer(&buffer, &frames, &flags, &pos, &ts); + if (FAILED(res)) { + if (res != AUDCLNT_E_DEVICE_INVALIDATED) + return false; + } + + float* data = (float*)buffer; + for (int i = 0; i < frames * channels; ++i) { + capture_buffer.push_back(data[i]); + + if (capture_buffer.size() == ((sampleRate / 100) * channels)) { + ReorderingBuffer(release_capture_buffer, capture_buffer.data(), + channels, audio_multiplier); + source->UpdateFrame((const int16_t*)(release_capture_buffer.data()), + 480, 48000, channels); + capture_buffer.clear(); + } + } + + capture->ReleaseBuffer(frames); + } + + return true; + } else { + return false; + } +} + +void SystemModule::Initialize() { + ResetEvent(receiveSignal); + + ComPtr temp_client = InitClient( + process_id, activate_audio_interface_async, speakers, format, sampleRate); + + ComPtr temp_capture = + InitCapture(temp_client, receiveSignal); + + client = std::move(temp_client); + capture = std::move(temp_capture); +} + +void SystemModule::InitFormat(const WAVEFORMATEX* wfex, + speaker_layout& channels, + int& format, + uint32_t& sampleRate) { + DWORD layout = 0; + + if (wfex->wFormatTag == WAVE_FORMAT_EXTENSIBLE) { + WAVEFORMATEXTENSIBLE* ext = (WAVEFORMATEXTENSIBLE*)wfex; + layout = ext->dwChannelMask; + } + + /* WASAPI is always float */ + channels = ConvertSpeakerLayout(layout, wfex->nChannels); + format = 4; // float bytes + sampleRate = wfex->nSamplesPerSec; +} + +ComPtr SystemModule::InitCapture(IAudioClient* client, + HANDLE receiveSignal) { + ComPtr capture; + HRESULT res = client->GetService(IID_PPV_ARGS(capture.Assign())); + if (FAILED(res)) + throw HRError("Failed to create capture context", res); + + res = client->SetEventHandle(receiveSignal); + if (FAILED(res)) + throw HRError("Failed to set event handle", res); + + res = client->Start(); + if (FAILED(res)) + throw HRError("Failed to start capture client", res); + + return capture; +} + +////////////////////////// API + +int32_t SystemModule::Terminate() { + SetEvent(exitSignal); + return StopRecording(); +} + +int32_t SystemModule::StartRecording() { + SetEvent(initSignal); + return 0; +} + +void SystemModule::SetSystemAudioLevel(float level) { + audio_multiplier = level; +} + +float SystemModule::GetSystemAudioLevel() const { + return audio_multiplier; +} + +void SystemModule::SetRecordingSource(int id) { + const bool restart = process_id != id; + process_id = id; + + if (restart) { + SetEvent(restartSignal); + } +} + +bool SystemModule::Init() { + bool success; + try { + Initialize(); + success = true; + } catch (...) { + success = false; + } + + previouslyFailed = !success; + return success; +} + +int SystemModule::StopRecording() { + return 0; +} + +rtc::scoped_refptr SystemModule::CreateSource() { + if (!source) { + source = rtc::scoped_refptr(new SystemSource(this)); + } + auto result = source; + source->Release(); + return result; +} + +int32_t SystemModule::RecordingChannels() { + return GetChannels(speakers); +} + +void SystemModule::ResetSource() { + source = nullptr; +} + +std::vector SystemModule::EnumerateSystemSource() const { + return ms_fill_window_list(window_search_mode::INCLUDE_MINIMIZED); +} + +#endif \ No newline at end of file diff --git a/crates/libwebrtc-sys/src/lib.rs b/crates/libwebrtc-sys/src/lib.rs index 92739f1a68..6b75c84d25 100644 --- a/crates/libwebrtc-sys/src/lib.rs +++ b/crates/libwebrtc-sys/src/lib.rs @@ -190,16 +190,45 @@ impl TaskQueueFactory { } } +/// Representation of a custom audio source, +/// used to mix and send an audio device module. +pub struct AudioSource(UniquePtr); + unsafe impl Send for webrtc::TaskQueueFactory {} unsafe impl Sync for webrtc::TaskQueueFactory {} +/// Information about system audio source. +pub struct AudioSourceInfo { + /// Unique id of system audio source. + id: i64, + /// Title of system audio source. + title: String, +} + +impl AudioSourceInfo { + /// Returns a `id` of this [`AudioSourceInfo`]. + #[must_use] + pub fn id(&self) -> i64 { + self.id + } + + /// Returns a `title` of this [`AudioSourceInfo`]. + #[must_use] + pub fn title(&self) -> String { + self.title.clone() + } +} + /// Available audio devices manager that is responsible for driving input /// (microphone) and output (speaker) audio in WebRTC. /// /// Backed by WebRTC's [Audio Device Module]. /// /// [Audio Device Module]: https://tinyurl.com/doc-adm -pub struct AudioDeviceModule(UniquePtr); +pub struct AudioDeviceModule( + UniquePtr, + UniquePtr, +); impl AudioDeviceModule { /// Creates a new [`AudioDeviceModule`] for the given [`AudioLayer`]. @@ -211,24 +240,52 @@ impl AudioDeviceModule { audio_layer: AudioLayer, task_queue_factory: &mut TaskQueueFactory, ) -> anyhow::Result { - let ptr = webrtc::create_audio_device_module( + let adm = webrtc::create_custom_audio_device_module( worker_thread.0.pin_mut(), audio_layer, task_queue_factory.0.pin_mut(), ); + let manager = + webrtc::create_source_manager(&adm, worker_thread.0.pin_mut()); + + let ptr = webrtc::custom_audio_device_module_proxy_upcast( + adm, + worker_thread.0.pin_mut(), + ); + if ptr.is_null() { bail!("`null` pointer returned from `AudioDeviceModule::Create()`"); } - Ok(Self(ptr)) + Ok(Self(ptr, manager)) } /// Creates a new fake [`AudioDeviceModule`], that will not try to access /// real media devices, but will generate pulsed noise. pub fn create_fake(task_queue_factory: &mut TaskQueueFactory) -> Self { - Self(webrtc::create_fake_audio_device_module( - task_queue_factory.0.pin_mut(), - )) + Self( + webrtc::create_fake_audio_device_module( + task_queue_factory.0.pin_mut(), + ), + UniquePtr::null(), + ) + } + + /// Sets the system audio source. + pub fn set_system_audio_source(&mut self, id: i64) { + webrtc::set_system_audio_source(self.1.pin_mut(), id); + } + + /// Enumerates possible system audio sources. + #[must_use] + pub fn enumerate_system_audio_source(&self) -> Vec { + webrtc::enumerate_system_audio_source(&self.1) + .into_iter() + .map(|info| AudioSourceInfo { + id: webrtc::system_source_id(info), + title: webrtc::system_source_title(info).to_string(), + }) + .collect() } /// Initializes the current [`AudioDeviceModule`]. @@ -240,6 +297,44 @@ impl AudioDeviceModule { Ok(()) } + /// Creates a new [`AudioSource`] from microphone. + pub fn create_source_microphone(&mut self) -> AudioSource { + // Fake media. + if self.1.is_null() { + return AudioSource(UniquePtr::null()); + } + let result = webrtc::create_source_microphone(self.1.pin_mut()); + AudioSource(result) + } + + /// Creates a new [`AudioSource`] from microphone. + pub fn create_system_audio_source(&mut self) -> AudioSource { + // Fake media. + if self.1.is_null() { + return AudioSource(UniquePtr::null()); + } + let result = webrtc::create_system_audio_source(self.1.pin_mut()); + AudioSource(result) + } + + /// Adds [`AudioSource`] to [`webrtc::AudioSourceManager`]. + pub fn add_source(&mut self, source: &AudioSource) { + // Fake media. + if self.1.is_null() { + return; + } + webrtc::add_source(self.1.pin_mut(), &source.0); + } + + /// Removes [`AudioSource`] from [`webrtc::AudioSourceManager`]. + pub fn remove_source(&mut self, source: &AudioSource) { + // Fake media. + if self.1.is_null() { + return; + } + webrtc::remove_source(self.1.pin_mut(), &source.0); + } + /// Returns count of available audio playout devices. #[must_use] #[allow(clippy::cast_sign_loss)] @@ -471,11 +566,28 @@ impl AudioDeviceModule { Ok(volume) } + + /// Sets the volume of the system audio capture. + pub fn set_system_audio_source_volume(&mut self, level: f32) { + webrtc::set_system_audio_source_volume(self.1.pin_mut(), level); + } + + /// Returns the current volume of the system audio capture. + #[must_use] + pub fn system_audio_source_volume(&self) -> f32 { + webrtc::system_audio_source_volume(&self.1) + } } unsafe impl Send for webrtc::AudioDeviceModule {} unsafe impl Sync for webrtc::AudioDeviceModule {} +unsafe impl Send for webrtc::AudioSourceManager {} +unsafe impl Sync for webrtc::AudioSourceManager {} + +unsafe impl Send for webrtc::AudioSource {} +unsafe impl Sync for webrtc::AudioSource {} + /// Representation of The Audio Processing Module, providing a collection of /// voice processing components designed for real-time communications software. pub struct AudioProcessing(UniquePtr); diff --git a/crates/native/src/api.rs b/crates/native/src/api.rs index d013c4f309..0b7de38137 100644 --- a/crates/native/src/api.rs +++ b/crates/native/src/api.rs @@ -161,6 +161,14 @@ impl From for TrackKind { } } +/// Information about system audio source. +pub struct AudioSourceInfo { + /// Unique id of system audio source. + pub id: i64, + /// Title of system audio source. + pub title: String, +} + /// Fields of [`RtcStatsType::RtcInboundRtpStreamStats`] variant. pub enum RtcInboundRtpStreamMediaType { /// `audio` media type fields. @@ -1636,6 +1644,10 @@ pub struct AudioConstraints { /// changing device will affect all previously obtained audio /// tracks. pub device_id: Option, + + /// Identifier of the system audio source generating the content of the + /// [`MediaStreamTrack`]. + pub system_id: Option, } /// Representation of a single media track within a [`MediaStream`]. @@ -1865,6 +1877,11 @@ pub fn enumerate_devices() -> anyhow::Result> { WEBRTC.lock().unwrap().enumerate_devices() } +/// Enumerates possible system audio sources. +pub fn enumerate_system_audio_source() -> Vec { + WEBRTC.lock().unwrap().enumerate_system_audio_source() +} + /// Returns a list of all available displays that can be used for screen /// capturing. pub fn enumerate_displays() -> Vec { @@ -2188,6 +2205,16 @@ pub fn clone_track( WEBRTC.lock().unwrap().clone_track(track_id, kind) } +/// Sets the volume of the system audio capture. +pub fn set_system_audio_volume(level: f32) { + WEBRTC.lock().unwrap().set_system_audio_volume(level); +} + +/// Returns the current volume of the system audio capture. +pub fn system_audio_volume() -> f32 { + WEBRTC.lock().unwrap().system_audio_volume() +} + /// Registers an observer to the [`MediaStreamTrack`] events. pub fn register_track_observer( cb: StreamSink, diff --git a/crates/native/src/bridge_generated.io.rs b/crates/native/src/bridge_generated.io.rs index ea085face7..d6a6710f14 100644 --- a/crates/native/src/bridge_generated.io.rs +++ b/crates/native/src/bridge_generated.io.rs @@ -16,6 +16,11 @@ pub extern "C" fn wire_enumerate_devices(port_: i64) { wire_enumerate_devices_impl(port_) } +#[no_mangle] +pub extern "C" fn wire_enumerate_system_audio_source(port_: i64) { + wire_enumerate_system_audio_source_impl(port_) +} + #[no_mangle] pub extern "C" fn wire_enumerate_displays(port_: i64) { wire_enumerate_displays_impl(port_) @@ -270,6 +275,16 @@ pub extern "C" fn wire_clone_track( wire_clone_track_impl(port_, track_id, kind) } +#[no_mangle] +pub extern "C" fn wire_set_system_audio_volume(port_: i64, level: f32) { + wire_set_system_audio_volume_impl(port_, level) +} + +#[no_mangle] +pub extern "C" fn wire_system_audio_volume(port_: i64) { + wire_system_audio_volume_impl(port_) +} + #[no_mangle] pub extern "C" fn wire_register_track_observer( port_: i64, @@ -319,6 +334,11 @@ pub extern "C" fn new_box_autoadd_audio_constraints_0( support::new_leak_box_ptr(wire_AudioConstraints::new_with_null_ptr()) } +#[no_mangle] +pub extern "C" fn new_box_autoadd_i64_0(value: i64) -> *mut i64 { + support::new_leak_box_ptr(value) +} + #[no_mangle] pub extern "C" fn new_box_autoadd_media_stream_constraints_0( ) -> *mut wire_MediaStreamConstraints { @@ -383,6 +403,7 @@ impl Wire2Api for wire_AudioConstraints { fn wire2api(self) -> AudioConstraints { AudioConstraints { device_id: self.device_id.wire2api(), + system_id: self.system_id.wire2api(), } } } @@ -393,6 +414,11 @@ impl Wire2Api for *mut wire_AudioConstraints { Wire2Api::::wire2api(*wrap).into() } } +impl Wire2Api for *mut i64 { + fn wire2api(self) -> i64 { + unsafe { *support::box_from_leak_ptr(self) } + } +} impl Wire2Api for *mut wire_MediaStreamConstraints { fn wire2api(self) -> MediaStreamConstraints { let wrap = unsafe { support::box_from_leak_ptr(self) }; @@ -481,6 +507,7 @@ pub struct wire_StringList { #[derive(Clone)] pub struct wire_AudioConstraints { device_id: *mut wire_uint_8_list, + system_id: *mut i64, } #[repr(C)] @@ -546,6 +573,7 @@ impl NewWithNullPtr for wire_AudioConstraints { fn new_with_null_ptr() -> Self { Self { device_id: core::ptr::null_mut(), + system_id: core::ptr::null_mut(), } } } diff --git a/crates/native/src/bridge_generated.rs b/crates/native/src/bridge_generated.rs index b86eadea17..f1f717f761 100644 --- a/crates/native/src/bridge_generated.rs +++ b/crates/native/src/bridge_generated.rs @@ -50,6 +50,16 @@ fn wire_enumerate_devices_impl(port_: MessagePort) { move || move |task_callback| enumerate_devices(), ) } +fn wire_enumerate_system_audio_source_impl(port_: MessagePort) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap( + WrapInfo { + debug_name: "enumerate_system_audio_source", + port: Some(port_), + mode: FfiCallMode::Normal, + }, + move || move |task_callback| Ok(enumerate_system_audio_source()), + ) +} fn wire_enumerate_displays_impl(port_: MessagePort) { FLUTTER_RUST_BRIDGE_HANDLER.wrap( WrapInfo { @@ -608,6 +618,32 @@ fn wire_clone_track_impl( }, ) } +fn wire_set_system_audio_volume_impl( + port_: MessagePort, + level: impl Wire2Api + UnwindSafe, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap( + WrapInfo { + debug_name: "set_system_audio_volume", + port: Some(port_), + mode: FfiCallMode::Normal, + }, + move || { + let api_level = level.wire2api(); + move |task_callback| Ok(set_system_audio_volume(api_level)) + }, + ) +} +fn wire_system_audio_volume_impl(port_: MessagePort) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap( + WrapInfo { + debug_name: "system_audio_volume", + port: Some(port_), + mode: FfiCallMode::Normal, + }, + move || move |task_callback| Ok(system_audio_volume()), + ) +} fn wire_register_track_observer_impl( port_: MessagePort, track_id: impl Wire2Api + UnwindSafe, @@ -723,6 +759,11 @@ impl Wire2Api for i32 { } } } +impl Wire2Api for f32 { + fn wire2api(self) -> f32 { + self + } +} impl Wire2Api for i32 { fn wire2api(self) -> i32 { self @@ -801,6 +842,13 @@ impl Wire2Api for u8 { // Section: impl IntoDart +impl support::IntoDart for AudioSourceInfo { + fn into_dart(self) -> support::DartAbi { + vec![self.id.into_dart(), self.title.into_dart()].into_dart() + } +} +impl support::IntoDartExceptPrimitive for AudioSourceInfo {} + impl support::IntoDart for CandidateType { fn into_dart(self) -> support::DartAbi { match self { diff --git a/crates/native/src/devices.rs b/crates/native/src/devices.rs index 2575952b9a..064330d8f4 100644 --- a/crates/native/src/devices.rs +++ b/crates/native/src/devices.rs @@ -296,6 +296,24 @@ impl Webrtc { self.audio_device_module.set_microphone_volume(level) } + /// Returns the current volume of the system audio capture. + pub fn enumerate_system_audio_source( + &mut self, + ) -> Vec { + self.audio_device_module.enumerate_system_audio_source() + } + + /// Sets the volume of the system audio capture. + pub fn set_system_audio_volume(&mut self, level: f32) { + self.audio_device_module + .set_system_audio_source_volume(level); + } + + /// Returns the current volume of the system audio capture. + pub fn system_audio_volume(&mut self) -> f32 { + self.audio_device_module.system_audio_source_volume() + } + /// Indicates if the microphone is available to set volume. pub fn microphone_volume_is_available(&mut self) -> anyhow::Result { self.audio_device_module.microphone_volume_is_available() diff --git a/crates/native/src/lib.rs b/crates/native/src/lib.rs index 9d407c1850..0bf4a91e3d 100644 --- a/crates/native/src/lib.rs +++ b/crates/native/src/lib.rs @@ -55,7 +55,10 @@ struct Webrtc { video_device_info: VideoDeviceInfo, video_sources: HashMap>, video_tracks: Arc>, - audio_source: Option>, + audio_source: ( + Option>, + HashMap, + ), audio_tracks: Arc>, video_sinks: HashMap, ap: sys::AudioProcessing, @@ -114,7 +117,7 @@ impl Webrtc { peer_connection_factory, video_sources: HashMap::new(), video_tracks: Arc::new(DashMap::new()), - audio_source: None, + audio_source: (None, HashMap::new()), audio_tracks: Arc::new(DashMap::new()), peer_connections: HashMap::new(), video_sinks: HashMap::new(), diff --git a/crates/native/src/user_media.rs b/crates/native/src/user_media.rs index 0e9081aa01..a936d2ca15 100644 --- a/crates/native/src/user_media.rs +++ b/crates/native/src/user_media.rs @@ -53,10 +53,14 @@ impl Webrtc { .create_audio_track(src) .map_err(|err| api::GetMediaError::Audio(err.to_string())); if let Err(err) = track { - if Arc::get_mut(self.audio_source.as_mut().unwrap()) + if Arc::get_mut(self.audio_source.0.as_mut().unwrap()) .is_some() { - self.audio_source.take(); + for source in self.audio_source.1.values() { + self.audio_device_module.remove_source(source); + } + self.audio_source.1.clear(); + self.audio_source.0.take(); } return Err(err); } @@ -86,7 +90,11 @@ impl Webrtc { { if let MediaTrackSource::Local(src) = track.source { if Arc::strong_count(&src) == 2 { - self.audio_source.take(); + for source in self.audio_source.1.values() { + self.audio_device_module.remove_source(source); + } + self.audio_source.1.clear(); + self.audio_source.0.take(); // TODO: We should make `AudioDeviceModule` to stop // recording. }; @@ -294,16 +302,64 @@ impl Webrtc { if Some(&device_id) != self.audio_device_module.current_device_id.as_ref() { + if let Some(source) = self + .audio_device_module + .current_device_id + .as_ref() + .map(|id| self.audio_source.1.remove(id)) + .flatten() + { + self.audio_device_module.remove_source(&source); + } + self.audio_device_module - .set_recording_device(device_id, device_index)?; + .set_recording_device(device_id.clone(), device_index)?; + } + + let microphone_audio = + self.audio_device_module.create_source_microphone(); + self.audio_device_module.add_source(µphone_audio); + self.audio_source.1.insert(device_id, microphone_audio); + + if let Some(id) = caps.system_id { + let system_id = AudioDeviceId(id.to_string()); + if Some(&system_id) + != self.audio_device_module.current_system_id.as_ref() + { + self.audio_device_module.set_system_audio_source(id); + + if let Some(current_system_id) = + self.audio_device_module.current_system_id.take() + { + if let Some(source) = + self.audio_source.1.remove(¤t_system_id) + { + self.audio_device_module.remove_source(&source); + } + } + + self.audio_device_module.current_system_id = + Some(system_id.clone()); + let system_audio = + self.audio_device_module.create_system_audio_source(); + + self.audio_device_module.add_source(&system_audio); + self.audio_source.1.insert(system_id, system_audio); + } + } else if let Some(system_id) = + self.audio_device_module.current_system_id.take() + { + if let Some(source) = self.audio_source.1.remove(&system_id) { + self.audio_device_module.remove_source(&source); + } } - let src = if let Some(src) = self.audio_source.as_ref() { + let src = if let Some(src) = self.audio_source.0.as_ref() { Arc::clone(src) } else { let src = Arc::new(self.peer_connection_factory.create_audio_source()?); - self.audio_source.replace(Arc::clone(&src)); + self.audio_source.0.replace(Arc::clone(&src)); src }; @@ -609,6 +665,10 @@ pub struct AudioDeviceModule { /// [`None`] if the [`AudioDeviceModule`] was not used yet to record data /// from the audio input device. current_device_id: Option, + + /// ID of the system audio source currently used by this + /// [`sys::AudioDeviceModule`]. + current_system_id: Option, } impl AudioDeviceModule { @@ -633,6 +693,7 @@ impl AudioDeviceModule { let mut adm = Self { inner, current_device_id: None, + current_system_id: None, }; if adm.recording_devices() > 0 { adm.set_recording_device( @@ -656,9 +717,49 @@ impl AudioDeviceModule { Self { inner, current_device_id: None, + current_system_id: None, } } + /// Sets the system audio source. + pub fn set_system_audio_source(&mut self, id: i64) { + self.inner.set_system_audio_source(id); + } + + /// Enumerates possible system audio sources. + pub fn enumerate_system_audio_source( + &mut self, + ) -> Vec { + self.inner + .enumerate_system_audio_source() + .into_iter() + .map(|info| api::AudioSourceInfo { + id: info.id(), + title: info.title(), + }) + .collect() + } + + /// Creates a new [`sys::AudioSource`] from microphone. + pub fn create_source_microphone(&mut self) -> sys::AudioSource { + self.inner.create_source_microphone() + } + + /// Creates a new [`sys::AudioSource`] from microphone. + pub fn create_system_audio_source(&mut self) -> sys::AudioSource { + self.inner.create_system_audio_source() + } + + /// Adds [`sys::AudioSource`] to [`AudioDeviceModule`]. + pub fn add_source(&mut self, source: &sys::AudioSource) { + self.inner.add_source(source); + } + + /// Removes [`sys::AudioSource`] to [`AudioDeviceModule`]. + pub fn remove_source(&mut self, source: &sys::AudioSource) { + self.inner.remove_source(source); + } + /// Returns the `(label, id)` tuple for the given audio playout device /// `index`. /// @@ -791,6 +892,17 @@ impl AudioDeviceModule { self.inner.set_microphone_volume(volume as u32) } + /// Sets the volume of the system audio capture. + pub fn set_system_audio_source_volume(&mut self, level: f32) { + self.inner.set_system_audio_source_volume(level); + } + + /// Returns the current volume of the system audio capture. + #[must_use] + pub fn system_audio_source_volume(&self) -> f32 { + self.inner.system_audio_source_volume() + } + /// Indicates if the microphone is available to set volume. /// /// # Errors diff --git a/example/lib/src/get_sources.dart b/example/lib/src/get_sources.dart index 1122a8ced4..c304d2ae79 100644 --- a/example/lib/src/get_sources.dart +++ b/example/lib/src/get_sources.dart @@ -23,6 +23,7 @@ class _GetSourcesSampleState extends State { void _getSources() async { var mediaDeviceInfos = await enumerateDevices(); var mediaDisplayInfos = await enumerateDisplays(); + var systemAudioInfos = await enumerateSystemAudioSource(); setState(() { var devicesInfo = ''; for (var device in mediaDeviceInfos) { @@ -33,6 +34,10 @@ class _GetSourcesSampleState extends State { devicesInfo += 'Kind: ${MediaDeviceKind.videoinput}\nTitle: ${display.title.toString()}\nId: ${display.deviceId}\n\n'; } + for (var system in systemAudioInfos) { + devicesInfo += + 'Title: ${system.title.toString()}\nId: ${system.id}\n\n'; + } text = devicesInfo; }); } diff --git a/example/pubspec.lock b/example/pubspec.lock index b51ddf8cd3..0f5f6b9c97 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -28,7 +28,7 @@ packages: name: args url: "https://pub.dartlang.org" source: hosted - version: "2.3.2" + version: "2.4.0" async: dependency: transitive description: @@ -70,7 +70,7 @@ packages: name: build_daemon url: "https://pub.dartlang.org" source: hosted - version: "3.1.0" + version: "3.1.1" build_resolvers: dependency: transitive description: @@ -105,7 +105,7 @@ packages: name: built_value url: "https://pub.dartlang.org" source: hosted - version: "8.4.3" + version: "8.4.4" characters: dependency: transitive description: @@ -175,7 +175,7 @@ packages: name: dart_style url: "https://pub.dartlang.org" source: hosted - version: "2.2.4" + version: "2.2.5" fake_async: dependency: transitive description: @@ -417,14 +417,14 @@ packages: name: pubspec_parse url: "https://pub.dartlang.org" source: hosted - version: "1.2.1" + version: "1.2.2" puppeteer: dependency: transitive description: name: puppeteer url: "https://pub.dartlang.org" source: hosted - version: "2.22.0" + version: "2.24.0" shelf: dependency: transitive description: @@ -562,7 +562,7 @@ packages: name: web_socket_channel url: "https://pub.dartlang.org" source: hosted - version: "2.3.0" + version: "2.4.0" webdriver: dependency: transitive description: diff --git a/ios/Classes/controller/MediaStreamTrackController.swift b/ios/Classes/controller/MediaStreamTrackController.swift index be53e2b7ae..b8660f293e 100644 --- a/ios/Classes/controller/MediaStreamTrackController.swift +++ b/ios/Classes/controller/MediaStreamTrackController.swift @@ -1,3 +1,4 @@ +// swiftformat:disable all import Flutter /// Controller of a `MediaStreamTrack`. diff --git a/lib/src/api/bridge.g.dart b/lib/src/api/bridge.g.dart index 52cdf5ee9c..4dc6e5b23d 100644 --- a/lib/src/api/bridge.g.dart +++ b/lib/src/api/bridge.g.dart @@ -30,6 +30,11 @@ abstract class FlutterWebrtcNative { FlutterRustBridgeTaskConstMeta get kEnumerateDevicesConstMeta; + /// Enumerates possible system audio sources. + Future> enumerateSystemAudioSource({dynamic hint}); + + FlutterRustBridgeTaskConstMeta get kEnumerateSystemAudioSourceConstMeta; + /// Returns a list of all available displays that can be used for screen /// capturing. Future> enumerateDisplays({dynamic hint}); @@ -250,6 +255,16 @@ abstract class FlutterWebrtcNative { FlutterRustBridgeTaskConstMeta get kCloneTrackConstMeta; + /// Sets the volume of the system audio capture. + Future setSystemAudioVolume({required double level, dynamic hint}); + + FlutterRustBridgeTaskConstMeta get kSetSystemAudioVolumeConstMeta; + + /// Returns the current volume of the system audio capture. + Future systemAudioVolume({dynamic hint}); + + FlutterRustBridgeTaskConstMeta get kSystemAudioVolumeConstMeta; + /// Registers an observer to the [`MediaStreamTrack`] events. Stream registerTrackObserver( {required String trackId, required MediaType kind, dynamic hint}); @@ -296,8 +311,27 @@ class AudioConstraints { /// tracks. final String? deviceId; + /// Identifier of the system audio source generating the content of the + /// [`MediaStreamTrack`]. + final int? systemId; + AudioConstraints({ this.deviceId, + this.systemId, + }); +} + +/// Information about system audio source. +class AudioSourceInfo { + /// Unique id of system audio source. + final int id; + + /// Title of system audio source. + final String title; + + AudioSourceInfo({ + required this.id, + required this.title, }); } @@ -1779,6 +1813,23 @@ class FlutterWebrtcNativeImpl implements FlutterWebrtcNative { argNames: [], ); + Future> enumerateSystemAudioSource({dynamic hint}) { + return _platform.executeNormal(FlutterRustBridgeTask( + callFfi: (port_) => + _platform.inner.wire_enumerate_system_audio_source(port_), + parseSuccessData: _wire2api_list_audio_source_info, + constMeta: kEnumerateSystemAudioSourceConstMeta, + argValues: [], + hint: hint, + )); + } + + FlutterRustBridgeTaskConstMeta get kEnumerateSystemAudioSourceConstMeta => + const FlutterRustBridgeTaskConstMeta( + debugName: "enumerate_system_audio_source", + argNames: [], + ); + Future> enumerateDisplays({dynamic hint}) { return _platform.executeNormal(FlutterRustBridgeTask( callFfi: (port_) => _platform.inner.wire_enumerate_displays(port_), @@ -2369,6 +2420,40 @@ class FlutterWebrtcNativeImpl implements FlutterWebrtcNative { argNames: ["trackId", "kind"], ); + Future setSystemAudioVolume({required double level, dynamic hint}) { + var arg0 = api2wire_f32(level); + return _platform.executeNormal(FlutterRustBridgeTask( + callFfi: (port_) => + _platform.inner.wire_set_system_audio_volume(port_, arg0), + parseSuccessData: _wire2api_unit, + constMeta: kSetSystemAudioVolumeConstMeta, + argValues: [level], + hint: hint, + )); + } + + FlutterRustBridgeTaskConstMeta get kSetSystemAudioVolumeConstMeta => + const FlutterRustBridgeTaskConstMeta( + debugName: "set_system_audio_volume", + argNames: ["level"], + ); + + Future systemAudioVolume({dynamic hint}) { + return _platform.executeNormal(FlutterRustBridgeTask( + callFfi: (port_) => _platform.inner.wire_system_audio_volume(port_), + parseSuccessData: _wire2api_f32, + constMeta: kSystemAudioVolumeConstMeta, + argValues: [], + hint: hint, + )); + } + + FlutterRustBridgeTaskConstMeta get kSystemAudioVolumeConstMeta => + const FlutterRustBridgeTaskConstMeta( + debugName: "system_audio_volume", + argNames: [], + ); + Stream registerTrackObserver( {required String trackId, required MediaType kind, dynamic hint}) { var arg0 = _platform.api2wire_String(trackId); @@ -2455,6 +2540,16 @@ class FlutterWebrtcNativeImpl implements FlutterWebrtcNative { return raw as String; } + AudioSourceInfo _wire2api_audio_source_info(dynamic raw) { + final arr = raw as List; + if (arr.length != 2) + throw Exception('unexpected arr length: expect 2 but see ${arr.length}'); + return AudioSourceInfo( + id: _wire2api_i64(arr[0]), + title: _wire2api_String(arr[1]), + ); + } + bool _wire2api_bool(dynamic raw) { return raw as bool; } @@ -2524,6 +2619,10 @@ class FlutterWebrtcNativeImpl implements FlutterWebrtcNative { return CandidateType.values[raw]; } + double _wire2api_f32(dynamic raw) { + return raw as double; + } + double _wire2api_f64(dynamic raw) { return raw as double; } @@ -2594,6 +2693,10 @@ class FlutterWebrtcNativeImpl implements FlutterWebrtcNative { return IceRole.values[raw]; } + List _wire2api_list_audio_source_info(dynamic raw) { + return (raw as List).map(_wire2api_audio_source_info).toList(); + } + List _wire2api_list_media_device_info(dynamic raw) { return (raw as List).map(_wire2api_media_device_info).toList(); } @@ -3013,6 +3116,11 @@ int api2wire_bundle_policy(BundlePolicy raw) { return api2wire_i32(raw.index); } +@protected +double api2wire_f32(double raw) { + return raw; +} + @protected int api2wire_i32(int raw) { return raw; @@ -3079,6 +3187,11 @@ class FlutterWebrtcNativePlatform return ptr; } + @protected + ffi.Pointer api2wire_box_autoadd_i64(int raw) { + return inner.new_box_autoadd_i64_0(api2wire_i64(raw)); + } + @protected ffi.Pointer api2wire_box_autoadd_media_stream_constraints( @@ -3132,6 +3245,11 @@ class FlutterWebrtcNativePlatform : api2wire_box_autoadd_audio_constraints(raw); } + @protected + ffi.Pointer api2wire_opt_box_autoadd_i64(int? raw) { + return raw == null ? ffi.nullptr : api2wire_box_autoadd_i64(raw); + } + @protected ffi.Pointer api2wire_opt_box_autoadd_video_constraints( VideoConstraints? raw) { @@ -3159,6 +3277,7 @@ class FlutterWebrtcNativePlatform void _api_fill_to_wire_audio_constraints( AudioConstraints apiObj, wire_AudioConstraints wireObj) { wireObj.device_id = api2wire_opt_String(apiObj.deviceId); + wireObj.system_id = api2wire_opt_box_autoadd_i64(apiObj.systemId); } void _api_fill_to_wire_box_autoadd_audio_constraints( @@ -3362,6 +3481,20 @@ class FlutterWebrtcNativeWire implements FlutterRustBridgeWireBase { late final _wire_enumerate_devices = _wire_enumerate_devicesPtr.asFunction(); + void wire_enumerate_system_audio_source( + int port_, + ) { + return _wire_enumerate_system_audio_source( + port_, + ); + } + + late final _wire_enumerate_system_audio_sourcePtr = + _lookup>( + 'wire_enumerate_system_audio_source'); + late final _wire_enumerate_system_audio_source = + _wire_enumerate_system_audio_sourcePtr.asFunction(); + void wire_enumerate_displays( int port_, ) { @@ -3897,6 +4030,36 @@ class FlutterWebrtcNativeWire implements FlutterRustBridgeWireBase { late final _wire_clone_track = _wire_clone_trackPtr .asFunction, int)>(); + void wire_set_system_audio_volume( + int port_, + double level, + ) { + return _wire_set_system_audio_volume( + port_, + level, + ); + } + + late final _wire_set_system_audio_volumePtr = + _lookup>( + 'wire_set_system_audio_volume'); + late final _wire_set_system_audio_volume = + _wire_set_system_audio_volumePtr.asFunction(); + + void wire_system_audio_volume( + int port_, + ) { + return _wire_system_audio_volume( + port_, + ); + } + + late final _wire_system_audio_volumePtr = + _lookup>( + 'wire_system_audio_volume'); + late final _wire_system_audio_volume = + _wire_system_audio_volumePtr.asFunction(); + void wire_register_track_observer( int port_, ffi.Pointer track_id, @@ -3992,6 +4155,20 @@ class FlutterWebrtcNativeWire implements FlutterRustBridgeWireBase { _new_box_autoadd_audio_constraints_0Ptr .asFunction Function()>(); + ffi.Pointer new_box_autoadd_i64_0( + int value, + ) { + return _new_box_autoadd_i64_0( + value, + ); + } + + late final _new_box_autoadd_i64_0Ptr = + _lookup Function(ffi.Int64)>>( + 'new_box_autoadd_i64_0'); + late final _new_box_autoadd_i64_0 = _new_box_autoadd_i64_0Ptr + .asFunction Function(int)>(); + ffi.Pointer new_box_autoadd_media_stream_constraints_0() { return _new_box_autoadd_media_stream_constraints_0(); @@ -4115,6 +4292,8 @@ class wire_RtcConfiguration extends ffi.Struct { class wire_AudioConstraints extends ffi.Struct { external ffi.Pointer device_id; + + external ffi.Pointer system_id; } class wire_VideoConstraints extends ffi.Struct { diff --git a/lib/src/api/devices.dart b/lib/src/api/devices.dart index cb699e1c88..3efbcda991 100644 --- a/lib/src/api/devices.dart +++ b/lib/src/api/devices.dart @@ -137,6 +137,33 @@ Future> enumerateDisplays() async { } } +/// Returns list of [AudioSourceInfo]s for the currently available system audio source. +Future> enumerateSystemAudioSource() async { + if (isDesktop) { + return await api!.enumerateSystemAudioSource(); + } else { + return List.empty(); + } +} + +/// Returns the current volume of the system audio capture. +Future systemAudioVolume() async { + if (isDesktop) { + return await api!.systemAudioVolume(); + } else { + throw 'Not supported'; + } +} + +/// Sets the system audio capture volume according to the specified [volume]. +Future setSystemAudioVolume(double volume) async { + if (isDesktop) { + return await api!.setSystemAudioVolume(level: volume); + } else { + throw 'Not supported'; + } +} + /// Returns list of local audio and video [NativeMediaStreamTrack]s based on the /// provided [DeviceConstraints]. Future> getUserMedia( @@ -213,10 +240,12 @@ Future> _getUserMediaChannel( /// FFI-based implementation of a [getUserMedia] function. Future> _getUserMediaFFI( DeviceConstraints constraints) async { - var audioConstraints = constraints.audio.mandatory != null || - constraints.audio.optional != null - ? ffi.AudioConstraints(deviceId: constraints.audio.mandatory?.deviceId) - : null; + var audioConstraints = + constraints.audio.mandatory != null || constraints.audio.optional != null + ? ffi.AudioConstraints( + deviceId: constraints.audio.mandatory?.deviceId, + systemId: constraints.audio.mandatory?.systemId) + : null; var videoConstraints = constraints.video.mandatory != null || constraints.video.optional != null diff --git a/lib/src/model/constraints.dart b/lib/src/model/constraints.dart index 7685287cfb..f65b04bda4 100644 --- a/lib/src/model/constraints.dart +++ b/lib/src/model/constraints.dart @@ -63,6 +63,7 @@ abstract class DeviceMediaConstraints { /// [DeviceMediaConstraints] for audio devices. class AudioConstraints implements DeviceMediaConstraints { String? deviceId; + int? systemId; /// Converts this model to the [Map] expected by Flutter. @override diff --git a/lib/src/model/device.dart b/lib/src/model/device.dart index a3fc860fdb..b5bdaef4c5 100644 --- a/lib/src/model/device.dart +++ b/lib/src/model/device.dart @@ -1,5 +1,7 @@ import '/src/api/bridge.g.dart' as ffi; +export '/src/api/bridge.g.dart' show AudioSourceInfo; + /// Media device kind. enum MediaDeviceKind { /// Represents an audio input device (for example, a microphone).