From 67a7e48088f6344bceb4402c5d2d4bfe4e5f4056 Mon Sep 17 00:00:00 2001 From: dvonthenen Date: Fri, 1 Nov 2024 12:32:07 -0700 Subject: [PATCH] Initial AgentAPI for EA --- Deepgram.Dev.sln | 16 + Deepgram/AgentWebSocketClient.cs | 18 + Deepgram/ClientFactory.cs | 20 + Deepgram/Clients/Agent/v2/Websocket/Client.cs | 713 ++++++++++++++++++ .../Clients/Agent/v2/Websocket/Constants.cs | 15 + .../Agent/v2/Websocket/ResponseEvent.cs | 11 + .../Clients/Agent/v2/Websocket/UriSegments.cs | 16 + .../Interfaces/v2/IAgentWebSocketClient.cs | 165 ++++ Deepgram/Deepgram.csproj | 6 + Deepgram/Models/Agent/v2/WebSocket/Agent.cs | 28 + .../v2/WebSocket/AgentAudioDoneResponse.cs | 24 + .../WebSocket/AgentStartedSpeakingResponse.cs | 36 + .../v2/WebSocket/AgentThinkingResponse.cs | 28 + .../Models/Agent/v2/WebSocket/AgentType.cs | 36 + Deepgram/Models/Agent/v2/WebSocket/Audio.cs | 25 + .../Agent/v2/WebSocket/AudioResponse.cs | 30 + .../Agent/v2/WebSocket/CloseResponse.cs | 11 + Deepgram/Models/Agent/v2/WebSocket/Context.cs | 25 + .../Agent/v2/WebSocket/ControlMessage.cs | 24 + .../v2/WebSocket/ConversationTextResponse.cs | 32 + .../Agent/v2/WebSocket/ErrorResponse.cs | 11 + .../Models/Agent/v2/WebSocket/Function.cs | 40 + .../WebSocket/FunctionCallRequestResponse.cs | 26 + .../WebSocket/FunctionCallResponseSchema.cs | 31 + .../v2/WebSocket/FunctionCallingResponse.cs | 28 + Deepgram/Models/Agent/v2/WebSocket/Header.cs | 24 + .../v2/WebSocket/InjectAgentMessageSchema.cs | 27 + Deepgram/Models/Agent/v2/WebSocket/Input.cs | 24 + Deepgram/Models/Agent/v2/WebSocket/Item.cs | 24 + .../Agent/v2/WebSocket/KeepAliveSchema.cs | 23 + Deepgram/Models/Agent/v2/WebSocket/Listen.cs | 20 + .../Models/Agent/v2/WebSocket/OpenResponse.cs | 11 + Deepgram/Models/Agent/v2/WebSocket/Output.cs | 32 + .../Models/Agent/v2/WebSocket/Parameters.cs | 30 + .../Models/Agent/v2/WebSocket/Properties.cs | 21 + .../Models/Agent/v2/WebSocket/Provider.cs | 20 + .../WebSocket/SettingsConfigurationSchema.cs | 35 + Deepgram/Models/Agent/v2/WebSocket/Speak.cs | 20 + Deepgram/Models/Agent/v2/WebSocket/Think.cs | 34 + .../Agent/v2/WebSocket/UnhandledResponse.cs | 11 + .../v2/WebSocket/UpdateInstructionsSchema.cs | 27 + .../Agent/v2/WebSocket/UpdateSpeakSchema.cs | 27 + .../WebSocket/UserStartedSpeakingResponse.cs | 24 + .../Agent/v2/WebSocket/WelcomeResponse.cs | 28 + examples/agent/websocket/simple/Agent.csproj | 19 + examples/agent/websocket/simple/Program.cs | 210 ++++++ 46 files changed, 2106 insertions(+) create mode 100644 Deepgram/AgentWebSocketClient.cs create mode 100644 Deepgram/Clients/Agent/v2/Websocket/Client.cs create mode 100644 Deepgram/Clients/Agent/v2/Websocket/Constants.cs create mode 100644 Deepgram/Clients/Agent/v2/Websocket/ResponseEvent.cs create mode 100644 Deepgram/Clients/Agent/v2/Websocket/UriSegments.cs create mode 100644 Deepgram/Clients/Interfaces/v2/IAgentWebSocketClient.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Agent.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/AgentAudioDoneResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/AgentStartedSpeakingResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/AgentThinkingResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/AgentType.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Audio.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/AudioResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/CloseResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Context.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/ControlMessage.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/ConversationTextResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/ErrorResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Function.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/FunctionCallRequestResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/FunctionCallResponseSchema.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/FunctionCallingResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Header.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/InjectAgentMessageSchema.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Input.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Item.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/KeepAliveSchema.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Listen.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/OpenResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Output.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Parameters.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Properties.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Provider.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/SettingsConfigurationSchema.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Speak.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/Think.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/UnhandledResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/UpdateInstructionsSchema.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/UpdateSpeakSchema.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/UserStartedSpeakingResponse.cs create mode 100644 Deepgram/Models/Agent/v2/WebSocket/WelcomeResponse.cs create mode 100644 examples/agent/websocket/simple/Agent.csproj create mode 100644 examples/agent/websocket/simple/Program.cs diff --git a/Deepgram.Dev.sln b/Deepgram.Dev.sln index d0f59b96..08497041 100644 --- a/Deepgram.Dev.sln +++ b/Deepgram.Dev.sln @@ -177,6 +177,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Streaming", "tests\edge_cas EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Speak", "tests\edge_cases\tts_v1_client_example\Speak.csproj", "{AB053DDA-2487-476C-9793-A50C37F10CCA}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "agent", "agent", "{B7C828E2-1CDB-4F78-9AB7-CA2180795DF4}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "websocket", "websocket", "{B254E451-B210-4A01-8854-DA03AC2A1065}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "simple", "simple", "{FB6A2238-DD9A-4A47-B723-C173F2D2E31C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Agent", "examples\agent\websocket\simple\Agent.csproj", "{332347AC-E1D8-4B5D-A26F-50975AEF1F4F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -327,6 +335,10 @@ Global {AB053DDA-2487-476C-9793-A50C37F10CCA}.Debug|Any CPU.Build.0 = Debug|Any CPU {AB053DDA-2487-476C-9793-A50C37F10CCA}.Release|Any CPU.ActiveCfg = Release|Any CPU {AB053DDA-2487-476C-9793-A50C37F10CCA}.Release|Any CPU.Build.0 = Release|Any CPU + {332347AC-E1D8-4B5D-A26F-50975AEF1F4F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {332347AC-E1D8-4B5D-A26F-50975AEF1F4F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {332347AC-E1D8-4B5D-A26F-50975AEF1F4F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {332347AC-E1D8-4B5D-A26F-50975AEF1F4F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -412,6 +424,10 @@ Global {5CEEB2F0-F284-4BB3-8999-6E91151C0C06} = {1280E66D-A375-422A-ACB4-48F17E9C190E} {964A87B4-31F8-4D68-AE64-64E66C9959FD} = {0BF29CA2-1CD6-4FF0-BC7B-B33C6B41E9A1} {AB053DDA-2487-476C-9793-A50C37F10CCA} = {5CEEB2F0-F284-4BB3-8999-6E91151C0C06} + {B7C828E2-1CDB-4F78-9AB7-CA2180795DF4} = {C673DFD1-528A-4BAE-94E6-02EF058AC363} + {B254E451-B210-4A01-8854-DA03AC2A1065} = {B7C828E2-1CDB-4F78-9AB7-CA2180795DF4} + {FB6A2238-DD9A-4A47-B723-C173F2D2E31C} = {B254E451-B210-4A01-8854-DA03AC2A1065} + {332347AC-E1D8-4B5D-A26F-50975AEF1F4F} = {FB6A2238-DD9A-4A47-B723-C173F2D2E31C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8D4ABC6D-7126-4EE2-9303-43A954616B2A} diff --git a/Deepgram/AgentWebSocketClient.cs b/Deepgram/AgentWebSocketClient.cs new file mode 100644 index 00000000..eb84a76a --- /dev/null +++ b/Deepgram/AgentWebSocketClient.cs @@ -0,0 +1,18 @@ +// Copyright 2021-2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +using Deepgram.Clients.Agent.v2.WebSocket; +using Deepgram.Models.Authenticate.v1; + +namespace Deepgram; + +/// +/// Implements the latest supported version of the Agent Client. +/// +public class AgentWebSocketClient : Client +{ + public AgentWebSocketClient(string apiKey = "", DeepgramWsClientOptions? deepgramClientOptions = null) : base(apiKey, deepgramClientOptions) + { + } +} diff --git a/Deepgram/ClientFactory.cs b/Deepgram/ClientFactory.cs index 9eb46e2d..18c5bb65 100644 --- a/Deepgram/ClientFactory.cs +++ b/Deepgram/ClientFactory.cs @@ -15,6 +15,17 @@ namespace Deepgram; public static class ClientFactory { + /// + /// Create a new AgentWebSocketClient using the latest version + /// + /// + /// + /// + public static V2.IAgentWebSocketClient CreateAgentWebSocketClient(string apiKey = "", DeepgramWsClientOptions? options = null) + { + return new AgentWebSocketClient(apiKey, options); + } + /// /// Create a new AnalyzeClient /// @@ -108,6 +119,15 @@ public static V2.ISpeakWebSocketClient CreateSpeakWebSocketClient(string apiKey /// and the latest version will be renamed to v1. /// + /// + /// This method allows you to create an AgentClient with a specific version of the client. + /// TODO: this should be revisited at a later time. Opportunity to use reflection to get the type of the client + /// + public static object CreateAgentWebSocketClient(int version, string apiKey = "", DeepgramWsClientOptions? options = null) + { + return new AgentWebSocketClient(apiKey, options); + } + /// /// This method allows you to create an AnalyzeClient with a specific version of the client. /// TODO: this should be revisited at a later time. Opportunity to use reflection to get the type of the client diff --git a/Deepgram/Clients/Agent/v2/Websocket/Client.cs b/Deepgram/Clients/Agent/v2/Websocket/Client.cs new file mode 100644 index 00000000..09ff3f4b --- /dev/null +++ b/Deepgram/Clients/Agent/v2/Websocket/Client.cs @@ -0,0 +1,713 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +using Deepgram.Abstractions.v2; +using Deepgram.Models.Authenticate.v1; +using Deepgram.Models.Agent.v2.WebSocket; +using Common = Deepgram.Models.Common.v2.WebSocket; +using Deepgram.Clients.Interfaces.v2; + +namespace Deepgram.Clients.Agent.v2.WebSocket; + +/// +/// Implements version 2 of the Listen WebSocket Client. +/// +public class Client : AbstractWebSocketClient, IAgentWebSocketClient +{ + /// Required DeepgramApiKey + /// for HttpClient Configuration + public Client(string? apiKey = null, IDeepgramClientOptions? options = null) : base(apiKey, options) + { + Log.Verbose("AgentWSClient", "ENTER"); + Log.Debug("AgentWSClient", $"KeepAlive: {_deepgramClientOptions.KeepAlive}"); + Log.Verbose("AgentWSClient", "LEAVE"); + } + + #region Event Handlers + /// + /// Fires when an event is received from the Deepgram API + /// + private event EventHandler? _audioReceived; + private event EventHandler? _agentAudioDoneReceived; + private event EventHandler? _agentStartedSpeakingReceived; + private event EventHandler? _agentThinkingReceived; + private event EventHandler? _conversationTextReceived; + private event EventHandler? _functionCallingReceived; + private event EventHandler? _functionCallRequestReceived; + private event EventHandler? _userStartedSpeakingReceived; + private event EventHandler? _welcomeReceived; + #endregion + + /// + /// Connect to a Deepgram API Web Socket to begin transcribing audio + /// + /// Options to use when transcribing audio + /// The task object representing the asynchronous operation. + public async Task Connect(SettingsConfigurationSchema options, CancellationTokenSource? cancelToken = null, Dictionary? addons = null, + Dictionary? headers = null) + { + Log.Verbose("AgentWSClient.Connect", "ENTER"); + Log.Information("Connect", $"options:\n{JsonSerializer.Serialize(options, JsonSerializeOptions.DefaultOptions)}"); + Log.Debug("Connect", $"addons: {addons}"); + + try + { + var myURI = GetUri(_deepgramClientOptions); + Log.Debug("Connect", $"uri: {myURI}"); + bool bConnected = await base.Connect(myURI.ToString(), cancelToken, headers); + if (!bConnected) + { + Log.Warning("Connect", "Connect failed"); + Log.Verbose("AgentWSClient.Connect", "LEAVE"); + return false; + } + + // send the settings configuration + var bytes = Encoding.UTF8.GetBytes(options.ToString()); + await SendMessageImmediately(bytes); + + // keepalive thread + if (_deepgramClientOptions.KeepAlive) + { + Log.Debug("Connect", "Starting KeepAlive Thread..."); + StartKeepAliveBackgroundThread(); + } + + Log.Debug("Connect", "Connect Succeeded"); + Log.Verbose("AgentWSClient.Connect", "LEAVE"); + + return true; + } + catch (TaskCanceledException ex) + { + Log.Debug("Connect", "Connect cancelled."); + Log.Verbose("Connect", $"Connect cancelled. Info: {ex}"); + Log.Verbose("AgentWSClient.Connect", "LEAVE"); + + return false; + } + catch (Exception ex) + { + Log.Error("Connect", $"{ex.GetType()} thrown {ex.Message}"); + Log.Verbose("Connect", $"Exception: {ex}"); + Log.Verbose("AgentWSClient.Connect", "LEAVE"); + throw; + } + + void StartKeepAliveBackgroundThread() => Task.Run(async () => await ProcessKeepAlive()); + } + + #region Subscribe Event + /// + /// Subscribe to an Open event from the Deepgram API + /// + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + // Create a new event handler that wraps the original one + EventHandler wrappedHandler = (sender, args) => + { + // Cast the event arguments to the desired type + var castedArgs = new OpenResponse(); + castedArgs.Copy(args); + if (castedArgs != null) + { + // Invoke the original event handler with the casted arguments + eventHandler(sender, castedArgs); + } + }; + + // Pass the new event handler to the base Subscribe method + return await base.Subscribe(wrappedHandler); + } + + /// + /// Subscribe to a Audio event from the Deepgram API + /// + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _audioReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to a AgentAudioDone event from the Deepgram API + /// + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _agentAudioDoneReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to a AgentStartedSpeaking event from the Deepgram API + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _agentStartedSpeakingReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to an AgentThinking event from the Deepgram API + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _agentThinkingReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to a ConversationText event from the Deepgram API + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _conversationTextReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to a FunctionCalling event from the Deepgram API + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _functionCallingReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to a FunctionCallRequest event from the Deepgram API + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _functionCallRequestReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to a UserStartedSpeaking event from the Deepgram API + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _userStartedSpeakingReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to a Welcome event from the Deepgram API + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + await _mutexSubscribe.WaitAsync(); + try + { + _welcomeReceived += (sender, e) => eventHandler(sender, e); + } + finally + { + _mutexSubscribe.Release(); + } + return true; + } + + /// + /// Subscribe to an Close event from the Deepgram API + /// + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + // Create a new event handler that wraps the original one + EventHandler wrappedHandler = (sender, args) => + { + // Cast the event arguments to the desired type + var castedArgs = new CloseResponse(); + castedArgs.Copy(args); + if (castedArgs != null) + { + // Invoke the original event handler with the casted arguments + eventHandler(sender, castedArgs); + } + }; + + return await base.Subscribe(wrappedHandler); + } + + /// + /// Subscribe to an Error event from the Deepgram API + /// + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + // Create a new event handler that wraps the original one + EventHandler wrappedHandler = (sender, args) => + { + // Cast the event arguments to the desired type + var castedArgs = new ErrorResponse(); + castedArgs.Copy(args); + if (castedArgs != null) + { + // Invoke the original event handler with the casted arguments + eventHandler(sender, castedArgs); + } + }; + + return await base.Subscribe(wrappedHandler); + } + + /// + /// Subscribe to an Unhandled event from the Deepgram API + /// + /// + /// True if successful + public async Task Subscribe(EventHandler eventHandler) + { + // Create a new event handler that wraps the original one + EventHandler wrappedHandler = (sender, args) => + { + // Cast the event arguments to the desired type + var castedArgs = new UnhandledResponse(); + castedArgs.Copy(args); + if (castedArgs != null) + { + // Invoke the original event handler with the casted arguments + eventHandler(sender, castedArgs); + } + }; + + return await base.Subscribe(wrappedHandler); + } + #endregion + + #region Send Functions + /// + /// Sends a KeepAlive message to Deepgram + /// + public async Task SendKeepAlive() + { + ControlMessage message = new ControlMessage(AgentClientTypes.KeepAlive); + byte[] data = Encoding.ASCII.GetBytes(message.ToString()); + await SendMessageImmediately(data); + } + + /// + /// Sends a Close message to Deepgram + /// + public override async Task SendClose(bool nullByte = false, CancellationTokenSource? _cancellationToken = null) + { + if (_clientWebSocket == null || !IsConnected()) + { + Log.Warning("SendClose", "ClientWebSocket is null or not connected. Skipping..."); + return; + } + + // provide a cancellation token, or use the one in the class + var _cancelToken = _cancellationToken ?? _cancellationTokenSource; + + Log.Debug("SendClose", "Sending Close Message Immediately..."); + if (nullByte) + { + // send a close to Deepgram + await _mutexSend.WaitAsync(_cancelToken.Token); + try + { + await _clientWebSocket.SendAsync(new ArraySegment(new byte[1] { 0 }), WebSocketMessageType.Binary, true, _cancellationTokenSource.Token) + .ConfigureAwait(false); + } + finally + { + _mutexSend.Release(); + } + return; + } + + ControlMessage message = new ControlMessage(AgentClientTypes.Close); + byte[] data = Encoding.ASCII.GetBytes(message.ToString()); + await SendMessageImmediately(data); + } + #endregion + + internal async Task ProcessKeepAlive() + { + Log.Verbose("AgentWSClient.ProcessKeepAlive", "ENTER"); + + try + { + while (true) + { + Log.Verbose("ProcessKeepAlive", "Waiting for KeepAlive..."); + await Task.Delay(5000, _cancellationTokenSource.Token); + + if (_cancellationTokenSource.Token.IsCancellationRequested) + { + Log.Information("ProcessKeepAlive", "KeepAliveThread cancelled"); + break; + } + if (!IsConnected()) + { + Log.Debug("ProcessAutoFlush", "WebSocket is not connected. Exiting..."); + break; + } + + await SendKeepAlive(); + } + + Log.Verbose("ProcessKeepAlive", "Exit"); + Log.Verbose("AgentWSClient.ProcessKeepAlive", "LEAVE"); + } + catch (TaskCanceledException ex) + { + Log.Debug("ProcessKeepAlive", "KeepAliveThread cancelled."); + Log.Verbose("ProcessKeepAlive", $"KeepAliveThread cancelled. Info: {ex}"); + Log.Verbose("AgentWSClient.ProcessKeepAlive", "LEAVE"); + } + catch (Exception ex) + { + Log.Error("ProcessKeepAlive", $"{ex.GetType()} thrown {ex.Message}"); + Log.Verbose("ProcessKeepAlive", $"Exception: {ex}"); + Log.Verbose("AgentWSClient.ProcessKeepAlive", "LEAVE"); + } + } + + internal override void ProcessBinaryMessage(WebSocketReceiveResult result, MemoryStream ms) + { + try + { + Log.Debug("ProcessBinaryMessage", "Received WebSocketMessageType.Binary"); + + if (_audioReceived == null) + { + Log.Debug("ProcessBinaryMessage", "_audioReceived has no listeners"); + Log.Verbose("ProcessBinaryMessage", "LEAVE"); + return; + } + + var audioResponse = new AudioResponse() + { + Stream = ms + }; + + Log.Debug("ProcessBinaryMessage", "Invoking AudioResponse"); + InvokeParallel(_audioReceived, audioResponse); + } + catch (JsonException ex) + { + Log.Error("ProcessDataReceived", $"{ex.GetType()} thrown {ex.Message}"); + Log.Verbose("ProcessDataReceived", $"Exception: {ex}"); + Log.Verbose("SpeakWSClient.ProcessDataReceived", "LEAVE"); + } + catch (Exception ex) + { + Log.Error("ProcessDataReceived", $"{ex.GetType()} thrown {ex.Message}"); + Log.Verbose("ProcessDataReceived", $"Excepton: {ex}"); + Log.Verbose("SpeakWSClient.ProcessDataReceived", "LEAVE"); + } + } + + internal override void ProcessTextMessage(WebSocketReceiveResult result, MemoryStream ms) + { + Log.Verbose("AgentWSClient.ProcessTextMessage", "ENTER"); + + ms.Seek(0, SeekOrigin.Begin); + + var response = Encoding.UTF8.GetString(ms.ToArray()); + if (response == null) + { + Log.Warning("ProcessTextMessage", "Response is null"); + Log.Verbose("AgentWSClient.ProcessTextMessage", "LEAVE"); + return; + } + + try + { + Log.Verbose("ProcessTextMessage", $"raw response: {response}"); + var data = JsonDocument.Parse(response); + var val = Enum.Parse(typeof(AgentType), data.RootElement.GetProperty("type").GetString()!); + + Log.Verbose("ProcessTextMessage", $"Type: {val}"); + + switch (val) + { + case AgentType.Open: + case AgentType.Close: + case AgentType.Error: + Log.Debug("ProcessTextMessage", "Calling base.ProcessTextMessage..."); + base.ProcessTextMessage(result, ms); + break; + case AgentType.AgentAudioDone: + var agentAudioDoneResponse = data.Deserialize(); + if (_agentAudioDoneReceived == null) + { + Log.Debug("ProcessTextMessage", "_agentAudioDoneReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (agentAudioDoneResponse == null) + { + Log.Warning("ProcessTextMessage", "AgentAudioDoneResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking AgentAudioDoneResponse. event: {agentAudioDoneResponse}"); + InvokeParallel(_agentAudioDoneReceived, agentAudioDoneResponse); + break; + case AgentType.AgentStartedSpeaking: + var agentStartedSpeakingResponse = data.Deserialize(); + if (_agentStartedSpeakingReceived == null) + { + Log.Debug("ProcessTextMessage", "_agentStartedSpeakingReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (agentStartedSpeakingResponse == null) + { + Log.Warning("ProcessTextMessage", "AgentStartedSpeakingResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking AgentStartedSpeakingResponse. event: {agentStartedSpeakingResponse}"); + InvokeParallel(_agentStartedSpeakingReceived, agentStartedSpeakingResponse); + break; + case AgentType.AgentThinking: + var agentThinkingResponse = data.Deserialize(); + if (_agentThinkingReceived == null) + { + Log.Debug("ProcessTextMessage", "_agentThinkingReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (agentThinkingResponse == null) + { + Log.Warning("ProcessTextMessage", "AgentThinkingResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking AgentThinkingResponse. event: {agentThinkingResponse}"); + InvokeParallel(_agentThinkingReceived, agentThinkingResponse); + break; + case AgentType.ConversationText: + var conversationTextResponse = data.Deserialize(); + if (_conversationTextReceived == null) + { + Log.Debug("ProcessTextMessage", "_conversationTextReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (conversationTextResponse == null) + { + Log.Warning("ProcessTextMessage", "ConversationTextResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking ConversationTextResponse. event: {conversationTextResponse}"); + InvokeParallel(_conversationTextReceived, conversationTextResponse); + break; + case AgentType.FunctionCalling: + var functionCallingResponse = data.Deserialize(); + if (_functionCallingReceived == null) + { + Log.Debug("ProcessTextMessage", "_functionCallingReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (functionCallingResponse == null) + { + Log.Warning("ProcessTextMessage", "FunctionCallingResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking FunctionCallingResponse. event: {functionCallingResponse}"); + InvokeParallel(_functionCallingReceived, functionCallingResponse); + break; + case AgentType.FunctionCallRequest: + var functionCallRequestResponse = data.Deserialize(); + if (_functionCallRequestReceived == null) + { + Log.Debug("ProcessTextMessage", "_functionCallRequestReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (functionCallRequestResponse == null) + { + Log.Warning("ProcessTextMessage", "FunctionCallRequestResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking FunctionCallRequestResponse. event: {functionCallRequestResponse}"); + InvokeParallel(_functionCallRequestReceived, functionCallRequestResponse); + break; + case AgentType.UserStartedSpeaking: + var userStartedSpeakingResponse = data.Deserialize(); + if (_userStartedSpeakingReceived == null) + { + Log.Debug("ProcessTextMessage", "_userStartedSpeakingReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (userStartedSpeakingResponse == null) + { + Log.Warning("ProcessTextMessage", "UserStartedSpeakingResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking UserStartedSpeakingResponse. event: {userStartedSpeakingResponse}"); + InvokeParallel(_userStartedSpeakingReceived, userStartedSpeakingResponse); + break; + case AgentType.Welcome: + var welcomeResponse = data.Deserialize(); + if (_welcomeReceived == null) + { + Log.Debug("ProcessTextMessage", "_welcomeReceived has no listeners"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + if (welcomeResponse == null) + { + Log.Warning("ProcessTextMessage", "WelcomeResponse is invalid"); + Log.Verbose("ProcessTextMessage", "LEAVE"); + return; + } + + Log.Debug("ProcessTextMessage", $"Invoking WelcomeResponse. event: {welcomeResponse}"); + InvokeParallel(_welcomeReceived, welcomeResponse); + break; + default: + Log.Debug("ProcessTextMessage", "Calling base.ProcessTextMessage..."); + base.ProcessTextMessage(result, ms); + break; + } + + Log.Debug("ProcessTextMessage", "Succeeded"); + Log.Verbose("AgentWSClient.ProcessTextMessage", "LEAVE"); + } + catch (JsonException ex) + { + Log.Error("ProcessTextMessage", $"{ex.GetType()} thrown {ex.Message}"); + Log.Verbose("ProcessTextMessage", $"Exception: {ex}"); + Log.Verbose("AgentWSClient.ProcessTextMessage", "LEAVE"); + } + catch (Exception ex) + { + Log.Error("ProcessTextMessage", $"{ex.GetType()} thrown {ex.Message}"); + Log.Verbose("ProcessTextMessage", $"Exception: {ex}"); + Log.Verbose("AgentWSClient.ProcessTextMessage", "LEAVE"); + } + } + + #region Helpers + /// + /// Get the URI for the WebSocket connection + /// + internal static Uri GetUri(IDeepgramClientOptions options) + { + var baseAddress = options.BaseAddress; + + // if base address contains "api.deepgram.com", then replace with "agent.deepgram.com" + // which is the default URI for the Agent API. This will preserve the wss or ws prefix. + // If this is a custom URI, then the we don't need to modify anything because DeepgramWSClientOptions + // will attach the protocol to the URI and the URI will be used as is. + if (baseAddress.Contains("api.deepgram.com")) + { + Log.Debug("GetUri", "Replacing baseAddress with agent.deepgram.com"); + baseAddress = baseAddress.Replace("api.deepgram.com", UriSegments.AGENT_URI); + } + Log.Debug("GetUri", $"baseAddress: {baseAddress}"); + + // if the base address has an v1, v2, etc remove it from the URI + Regex regex = new Regex(@"\b(\/v[0-9]+)\b", RegexOptions.IgnoreCase); + if (regex.IsMatch(baseAddress)) + { + Log.Information("GetUri", $"BaseAddress contains API version: {baseAddress}"); + baseAddress = regex.Replace(baseAddress, ""); + Log.Debug("GetUri", $"BaseAddress: {baseAddress}"); + } + + return new Uri($"{baseAddress}/{UriSegments.AGENT}"); + } + #endregion +} diff --git a/Deepgram/Clients/Agent/v2/Websocket/Constants.cs b/Deepgram/Clients/Agent/v2/Websocket/Constants.cs new file mode 100644 index 00000000..3af39379 --- /dev/null +++ b/Deepgram/Clients/Agent/v2/Websocket/Constants.cs @@ -0,0 +1,15 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Clients.Agent.v2.WebSocket; + +/// +/// Headers of interest in the return values from the Deepgram Speak API. +/// +public static class Constants +{ + // Default flush period + public const int DefaultFlushPeriodInMs = 500; +} + diff --git a/Deepgram/Clients/Agent/v2/Websocket/ResponseEvent.cs b/Deepgram/Clients/Agent/v2/Websocket/ResponseEvent.cs new file mode 100644 index 00000000..a09c315c --- /dev/null +++ b/Deepgram/Clients/Agent/v2/Websocket/ResponseEvent.cs @@ -0,0 +1,11 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Clients.Agent.v2.WebSocket; + +public class ResponseEvent(T? response) : EventArgs +{ + public T? Response { get; set; } = response; +} + diff --git a/Deepgram/Clients/Agent/v2/Websocket/UriSegments.cs b/Deepgram/Clients/Agent/v2/Websocket/UriSegments.cs new file mode 100644 index 00000000..36ea03ff --- /dev/null +++ b/Deepgram/Clients/Agent/v2/Websocket/UriSegments.cs @@ -0,0 +1,16 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Clients.Agent.v2.WebSocket; + +public static class UriSegments +{ + // overriding the default uri segments which is typically api.deepgram.com + // this is special for agent api for some odd reason + public const string AGENT_URI = "agent.deepgram.com"; + + //using constants instead of inline value(magic strings) make consistence + //across SDK And Test Projects Simpler and Easier to change + public const string AGENT = "agent"; +} diff --git a/Deepgram/Clients/Interfaces/v2/IAgentWebSocketClient.cs b/Deepgram/Clients/Interfaces/v2/IAgentWebSocketClient.cs new file mode 100644 index 00000000..6fa238be --- /dev/null +++ b/Deepgram/Clients/Interfaces/v2/IAgentWebSocketClient.cs @@ -0,0 +1,165 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +using Deepgram.Models.Agent.v2.WebSocket; + +namespace Deepgram.Clients.Interfaces.v2; + +/// +/// Implements version 2 of the Agent Client. +/// +public interface IAgentWebSocketClient +{ + #region Connect and Disconnect + /// + /// Connects to the Deepgram WebSocket API + /// + public Task Connect(SettingsConfigurationSchema options, CancellationTokenSource? cancelToken = null, Dictionary? addons = null, + Dictionary? headers = null); + + /// + /// Disconnects from the Deepgram WebSocket API + /// + public Task Stop(CancellationTokenSource? cancelToken = null, bool nullByte = false); + #endregion + + #region Subscribe Event + /// + /// Subscribe to an Open event from the Deepgram API + /// + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to an Audio Binary event from the Deepgram API + /// + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a AgentAudioDone event from the Deepgram API + /// + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a AgentStartedSpeaking event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler) +; + /// + /// Subscribe to an AgentThinking event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a ConversationText event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a FunctionCalling event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a FunctionCallRequest event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a UserStartedSpeaking event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a Welcome event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to a Close event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to an Unhandled event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + + /// + /// Subscribe to an Error event from the Deepgram API + /// + /// True if successful + public Task Subscribe(EventHandler eventHandler); + #endregion + + #region Send Functions + /// + /// Sends a KeepAlive message to Deepgram + /// + public Task SendKeepAlive(); + + /// + /// Sends a Close message to Deepgram + /// + public Task SendClose(bool nullByte = false, CancellationTokenSource? _cancellationToken = null); + + /// + /// This method sends a binary message over the WebSocket connection. + /// + /// + /// The number of bytes from the data to send. Use `Constants.UseArrayLengthForSend` to send the entire array. + public void SendBinary(byte[] data, int length = Constants.UseArrayLengthForSend); + + /// + /// This method sends a text message over the WebSocket connection. + /// + /// + /// The number of bytes from the data to send. Use `Constants.UseArrayLengthForSend` to send the entire array. + public void SendMessage(byte[] data, int length = Constants.UseArrayLengthForSend); + + /// + /// This method sends a binary message over the WebSocket connection immediately without queueing. + /// + /// + /// The number of bytes from the data to send. Use `Constants.UseArrayLengthForSend` to send the entire array. + /// /// Provide a cancel token to be used for the send function or use the internal one + public Task SendBinaryImmediately(byte[] data, int length = Constants.UseArrayLengthForSend, CancellationTokenSource? _cancellationToken = null); + + /// + /// This method sends a text message over the WebSocket connection immediately without queueing. + /// + /// + /// The number of bytes from the data to send. Use `Constants.UseArrayLengthForSend` to send the entire array. + /// /// Provide a cancel token to be used for the send function or use the internal one + public Task SendMessageImmediately(byte[] data, int length = Constants.UseArrayLengthForSend, CancellationTokenSource? _cancellationToken = null); + #endregion + + #region Helpers + /// + /// Retrieves the connection state of the WebSocket + /// + /// Returns the connection state of the WebSocket + public WebSocketState State(); + + /// + /// Indicates whether the WebSocket is connected + /// + /// Returns true if the WebSocket is connected + public bool IsConnected(); + #endregion +} diff --git a/Deepgram/Deepgram.csproj b/Deepgram/Deepgram.csproj index 8a43f9ea..9bd8718a 100644 --- a/Deepgram/Deepgram.csproj +++ b/Deepgram/Deepgram.csproj @@ -123,6 +123,12 @@ + + + + + + diff --git a/Deepgram/Models/Agent/v2/WebSocket/Agent.cs b/Deepgram/Models/Agent/v2/WebSocket/Agent.cs new file mode 100644 index 00000000..4635b17b --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Agent.cs @@ -0,0 +1,28 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Agent +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("listen")] + public Listen Listen { get; set; } = new Listen(); + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("think")] + public Think Think { get; set; } = new Think(); + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("speak")] + public Speak Speak { get; set; } = new Speak(); + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/AgentAudioDoneResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/AgentAudioDoneResponse.cs new file mode 100644 index 00000000..14f32020 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/AgentAudioDoneResponse.cs @@ -0,0 +1,24 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record AgentAudioDoneResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.AgentAudioDone; + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/AgentStartedSpeakingResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/AgentStartedSpeakingResponse.cs new file mode 100644 index 00000000..829efe9e --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/AgentStartedSpeakingResponse.cs @@ -0,0 +1,36 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record AgentStartedSpeakingResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.AgentStartedSpeaking; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("total_latency")] + public decimal? TotalLatency { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("tts_latency")] + public decimal? TtsLatency { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("ttt_latency")] + public decimal? TttLatency { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/AgentThinkingResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/AgentThinkingResponse.cs new file mode 100644 index 00000000..79edcfc7 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/AgentThinkingResponse.cs @@ -0,0 +1,28 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record AgentThinkingResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.AgentThinking; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("content")] + public string? Content { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/AgentType.cs b/Deepgram/Models/Agent/v2/WebSocket/AgentType.cs new file mode 100644 index 00000000..0e7a139d --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/AgentType.cs @@ -0,0 +1,36 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +using Deepgram.Models.Common.v2.WebSocket; + +public enum AgentType +{ + Open = WebSocketType.Open, + Close = WebSocketType.Close, + Unhandled = WebSocketType.Unhandled, + Error = WebSocketType.Error, + Welcome, + ConversationText, + UserStartedSpeaking, + AgentThinking, + FunctionCallRequest, + FunctionCalling, + AgentStartedSpeaking, + AgentAudioDone, + Audio, +} + +public static class AgentClientTypes +{ + // user message types + public const string SettingsConfiguration = "SettingsConfiguration"; + public const string UpdateInstructions = "UpdateInstructions"; + public const string UpdateSpeak = "UpdateSpeak"; + public const string InjectAgentMessage = "InjectAgentMessage"; + public const string FunctionCallResponse = "FunctionCallResponse"; + public const string KeepAlive = "KeepAlive"; + public const string Close = "Close"; +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Audio.cs b/Deepgram/Models/Agent/v2/WebSocket/Audio.cs new file mode 100644 index 00000000..f1405cf7 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Audio.cs @@ -0,0 +1,25 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Audio +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("input")] + public Input Input { get; set; } = new Input(); + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("output")] + public Output Output { get; set; } = new Output(); + + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/AudioResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/AudioResponse.cs new file mode 100644 index 00000000..dd7377bb --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/AudioResponse.cs @@ -0,0 +1,30 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record AudioResponse : IDisposable +{ + /// + /// The type of speak response, defaults to Audio. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; set; } = AgentType.Audio; + + /// + /// A stream of the audio file + /// + public MemoryStream? Stream { get; set; } + + // NOTE: There isn't a ToString() function because this will cause an odd Exception to be thrown: + // InvalidOperationException: "Timeouts are not supported on this stream." + + public void Dispose() + { + Stream?.Dispose(); + Stream = null; + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/CloseResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/CloseResponse.cs new file mode 100644 index 00000000..3652b178 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/CloseResponse.cs @@ -0,0 +1,11 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +using Common = Deepgram.Models.Common.v2.WebSocket; + +public record CloseResponse : Common.CloseResponse +{ +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Context.cs b/Deepgram/Models/Agent/v2/WebSocket/Context.cs new file mode 100644 index 00000000..4bdf2d0f --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Context.cs @@ -0,0 +1,25 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Content +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("messages")] + public List? Messages { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("replay")] + public bool? Replay { get; set; } + + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/ControlMessage.cs b/Deepgram/Models/Agent/v2/WebSocket/ControlMessage.cs new file mode 100644 index 00000000..e24ee041 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/ControlMessage.cs @@ -0,0 +1,24 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public class ControlMessage(string text) +{ + /// + /// Gets or sets the type of control message. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; set; } = text; + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} + diff --git a/Deepgram/Models/Agent/v2/WebSocket/ConversationTextResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/ConversationTextResponse.cs new file mode 100644 index 00000000..5088a233 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/ConversationTextResponse.cs @@ -0,0 +1,32 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record ConversationTextResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.ConversationText; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("role")] + public string? Role { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("content")] + public string? Content { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/ErrorResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/ErrorResponse.cs new file mode 100644 index 00000000..11e562b6 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/ErrorResponse.cs @@ -0,0 +1,11 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +using Common = Deepgram.Models.Common.v2.WebSocket; + +public record ErrorResponse : Common.ErrorResponse +{ +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Function.cs b/Deepgram/Models/Agent/v2/WebSocket/Function.cs new file mode 100644 index 00000000..25920de7 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Function.cs @@ -0,0 +1,40 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Function +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("name")] + public string? Name { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("description")] + public string? Description { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("parameters")] + public Parameters? Parameters { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("headers")] + public List
? Headers { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("method")] + public string? Method { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/FunctionCallRequestResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/FunctionCallRequestResponse.cs new file mode 100644 index 00000000..0e67a4b8 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/FunctionCallRequestResponse.cs @@ -0,0 +1,26 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record FunctionCallRequestResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.FunctionCallRequest; + + // TODO: this needs to be defined + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/FunctionCallResponseSchema.cs b/Deepgram/Models/Agent/v2/WebSocket/FunctionCallResponseSchema.cs new file mode 100644 index 00000000..5b5b66ee --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/FunctionCallResponseSchema.cs @@ -0,0 +1,31 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public class FunctionCallResponseSchema +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; } = AgentClientTypes.FunctionCallResponse; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("function_call_id")] + public string? FunctionCallId { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("output")] + public string? Output { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/FunctionCallingResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/FunctionCallingResponse.cs new file mode 100644 index 00000000..59012439 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/FunctionCallingResponse.cs @@ -0,0 +1,28 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record FunctionCallingResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.FunctionCalling; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("session_id")] + public string? SessionId { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Header.cs b/Deepgram/Models/Agent/v2/WebSocket/Header.cs new file mode 100644 index 00000000..7e9413e2 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Header.cs @@ -0,0 +1,24 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Header +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("key")] + public string? Key { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("value")] + public string? Value { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/InjectAgentMessageSchema.cs b/Deepgram/Models/Agent/v2/WebSocket/InjectAgentMessageSchema.cs new file mode 100644 index 00000000..7133e015 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/InjectAgentMessageSchema.cs @@ -0,0 +1,27 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public class InjectAgentMessageSchema +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; } = AgentClientTypes.InjectAgentMessage; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("message")] + public string? Message { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Input.cs b/Deepgram/Models/Agent/v2/WebSocket/Input.cs new file mode 100644 index 00000000..8687537f --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Input.cs @@ -0,0 +1,24 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Input +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("encoding")] + public string Encoding { get; set; } = "linear16"; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sample_rate")] + public long? SampleRate { get; set; } = 24000; + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Item.cs b/Deepgram/Models/Agent/v2/WebSocket/Item.cs new file mode 100644 index 00000000..c68b5fb5 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Item.cs @@ -0,0 +1,24 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Item +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("description")] + public string? Description { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/KeepAliveSchema.cs b/Deepgram/Models/Agent/v2/WebSocket/KeepAliveSchema.cs new file mode 100644 index 00000000..d99f0fae --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/KeepAliveSchema.cs @@ -0,0 +1,23 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public class KeepAliveSchema +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; } = AgentClientTypes.KeepAlive; + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Listen.cs b/Deepgram/Models/Agent/v2/WebSocket/Listen.cs new file mode 100644 index 00000000..26e461e2 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Listen.cs @@ -0,0 +1,20 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Listen +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("model")] + public string Model { get; set; } = "nova-2"; + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/OpenResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/OpenResponse.cs new file mode 100644 index 00000000..2a6f06c6 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/OpenResponse.cs @@ -0,0 +1,11 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +using Common = Deepgram.Models.Common.v2.WebSocket; + +public record OpenResponse : Common.OpenResponse +{ +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Output.cs b/Deepgram/Models/Agent/v2/WebSocket/Output.cs new file mode 100644 index 00000000..1febfc7e --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Output.cs @@ -0,0 +1,32 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Output +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("encoding")] + public string Encoding { get; set; } = "linear16"; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sample_rate")] + public long? SampleRate { get; set; } = 24000; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("bitrate")] + public long? Bitrate { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("container")] + public string? Container { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Parameters.cs b/Deepgram/Models/Agent/v2/WebSocket/Parameters.cs new file mode 100644 index 00000000..92e02953 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Parameters.cs @@ -0,0 +1,30 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Parameters +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string Type { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("properties")] + public Properties? Properties { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("required")] + public List? ParametersRequired { get; set; } + + + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Properties.cs b/Deepgram/Models/Agent/v2/WebSocket/Properties.cs new file mode 100644 index 00000000..12e48595 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Properties.cs @@ -0,0 +1,21 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Properties +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("item")] + public Item? Item { get; set; } + + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Provider.cs b/Deepgram/Models/Agent/v2/WebSocket/Provider.cs new file mode 100644 index 00000000..4b4b0382 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Provider.cs @@ -0,0 +1,20 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Provider +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string Type { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/SettingsConfigurationSchema.cs b/Deepgram/Models/Agent/v2/WebSocket/SettingsConfigurationSchema.cs new file mode 100644 index 00000000..bd63d13b --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/SettingsConfigurationSchema.cs @@ -0,0 +1,35 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public class SettingsConfigurationSchema +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; } = AgentClientTypes.SettingsConfiguration; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("audio")] + public Audio Audio { get; set; } = new Audio(); + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("agent")] + public Agent Agent { get; set; } = new Agent(); + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("context")] + public Context Context { get; set; } = new Context(); + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Speak.cs b/Deepgram/Models/Agent/v2/WebSocket/Speak.cs new file mode 100644 index 00000000..86d7e722 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Speak.cs @@ -0,0 +1,20 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Speak +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("model")] + public string Model { get; set; } = "aura-asteria-en"; + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/Think.cs b/Deepgram/Models/Agent/v2/WebSocket/Think.cs new file mode 100644 index 00000000..16f23986 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/Think.cs @@ -0,0 +1,34 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record Think +{ + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("provider")] + public Provider? Provider { get; set; } = new Provider(); + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("model")] + public string? Model { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("instructions")] + public string? Instructions { get; set; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("functions")] + public List? Functions { get; set; } + + + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/UnhandledResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/UnhandledResponse.cs new file mode 100644 index 00000000..0dc6481e --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/UnhandledResponse.cs @@ -0,0 +1,11 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +using Common = Deepgram.Models.Common.v2.WebSocket; + +public record UnhandledResponse : Common.UnhandledResponse +{ +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/UpdateInstructionsSchema.cs b/Deepgram/Models/Agent/v2/WebSocket/UpdateInstructionsSchema.cs new file mode 100644 index 00000000..a7752479 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/UpdateInstructionsSchema.cs @@ -0,0 +1,27 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public class UpdateInstructionsSchema +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; } = AgentClientTypes.UpdateInstructions; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("instructions")] + public string? Instructions { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/UpdateSpeakSchema.cs b/Deepgram/Models/Agent/v2/WebSocket/UpdateSpeakSchema.cs new file mode 100644 index 00000000..78eb8ff2 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/UpdateSpeakSchema.cs @@ -0,0 +1,27 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public class UpdateSpeakSchema +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + public string? Type { get; } = AgentClientTypes.UpdateSpeak; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("model")] + public string? Model { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/UserStartedSpeakingResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/UserStartedSpeakingResponse.cs new file mode 100644 index 00000000..6aebbc90 --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/UserStartedSpeakingResponse.cs @@ -0,0 +1,24 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record UserStartedSpeakingResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.UserStartedSpeaking; + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/Deepgram/Models/Agent/v2/WebSocket/WelcomeResponse.cs b/Deepgram/Models/Agent/v2/WebSocket/WelcomeResponse.cs new file mode 100644 index 00000000..3d8f712c --- /dev/null +++ b/Deepgram/Models/Agent/v2/WebSocket/WelcomeResponse.cs @@ -0,0 +1,28 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +namespace Deepgram.Models.Agent.v2.WebSocket; + +public record WelcomeResponse +{ + /// + /// SettingsConfiguration event type. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("type")] + [JsonConverter(typeof(JsonStringEnumConverter))] + public AgentType? Type { get; } = AgentType.Welcome; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("session_id")] + public string? SessionId { get; set; } + + /// + /// Override ToString method to serialize the object + /// + public override string ToString() + { + return Regex.Unescape(JsonSerializer.Serialize(this, JsonSerializeOptions.DefaultOptions)); + } +} diff --git a/examples/agent/websocket/simple/Agent.csproj b/examples/agent/websocket/simple/Agent.csproj new file mode 100644 index 00000000..f7e99529 --- /dev/null +++ b/examples/agent/websocket/simple/Agent.csproj @@ -0,0 +1,19 @@ + + + + Exe + net6.0 + enable + enable + + + + + + + + + + + + diff --git a/examples/agent/websocket/simple/Program.cs b/examples/agent/websocket/simple/Program.cs new file mode 100644 index 00000000..a98cb4ec --- /dev/null +++ b/examples/agent/websocket/simple/Program.cs @@ -0,0 +1,210 @@ +// Copyright 2024 Deepgram .NET SDK contributors. All Rights Reserved. +// Use of this source code is governed by a MIT license that can be found in the LICENSE file. +// SPDX-License-Identifier: MIT + +using Deepgram.Logger; +using Deepgram.Microphone; +using Deepgram.Models.Authenticate.v1; +using Deepgram.Models.Agent.v2.WebSocket; + +namespace SampleApp +{ + class Program + { + static async Task Main(string[] args) + { + try + { + // Initialize Library with default logging + // Normal logging is "Info" level + Deepgram.Library.Initialize(); + // OR very chatty logging + //Deepgram.Library.Initialize(LogLevel.Verbose); // LogLevel.Default, LogLevel.Debug, LogLevel.Verbose + + // Initialize the microphone library + Deepgram.Microphone.Library.Initialize(); + + Console.WriteLine("\n\nPress any key to stop and exit...\n\n\n"); + + // Set "DEEPGRAM_API_KEY" environment variable to your Deepgram API Key + DeepgramWsClientOptions options = new DeepgramWsClientOptions(null, null, true); + var agentClient = ClientFactory.CreateAgentWebSocketClient(apiKey: "", options: options); + + // current time + var lastAudioTime = DateTime.Now; + var audioFileCount = 0; + + // Subscribe to the EventResponseReceived event + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e.Type} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e.Type} received"); + + // if the last audio response is more than 5 seconds ago, add a wav header + if (DateTime.Now.Subtract(lastAudioTime).TotalSeconds > 7) + { + audioFileCount = audioFileCount + 1; // increment the audio file count + + // delete the file if it exists + if (File.Exists($"output_{audioFileCount}.wav")) + { + File.Delete($"output_{audioFileCount}.wav"); + } + + using (BinaryWriter writer = new BinaryWriter(File.Open($"output_{audioFileCount}.wav", FileMode.Append))) + { + Console.WriteLine("Adding WAV header to output.wav"); + byte[] wavHeader = new byte[44]; + int sampleRate = 48000; + short bitsPerSample = 16; + short channels = 1; + int byteRate = sampleRate * channels * (bitsPerSample / 8); + short blockAlign = (short)(channels * (bitsPerSample / 8)); + + wavHeader[0] = 0x52; // R + wavHeader[1] = 0x49; // I + wavHeader[2] = 0x46; // F + wavHeader[3] = 0x46; // F + wavHeader[4] = 0x00; // Placeholder for file size (will be updated later) + wavHeader[5] = 0x00; // Placeholder for file size (will be updated later) + wavHeader[6] = 0x00; // Placeholder for file size (will be updated later) + wavHeader[7] = 0x00; // Placeholder for file size (will be updated later) + wavHeader[8] = 0x57; // W + wavHeader[9] = 0x41; // A + wavHeader[10] = 0x56; // V + wavHeader[11] = 0x45; // E + wavHeader[12] = 0x66; // f + wavHeader[13] = 0x6D; // m + wavHeader[14] = 0x74; // t + wavHeader[15] = 0x20; // Space + wavHeader[16] = 0x10; // Subchunk1Size (16 for PCM) + wavHeader[17] = 0x00; // Subchunk1Size + wavHeader[18] = 0x00; // Subchunk1Size + wavHeader[19] = 0x00; // Subchunk1Size + wavHeader[20] = 0x01; // AudioFormat (1 for PCM) + wavHeader[21] = 0x00; // AudioFormat + wavHeader[22] = (byte)channels; // NumChannels + wavHeader[23] = 0x00; // NumChannels + wavHeader[24] = (byte)(sampleRate & 0xFF); // SampleRate + wavHeader[25] = (byte)((sampleRate >> 8) & 0xFF); // SampleRate + wavHeader[26] = (byte)((sampleRate >> 16) & 0xFF); // SampleRate + wavHeader[27] = (byte)((sampleRate >> 24) & 0xFF); // SampleRate + wavHeader[28] = (byte)(byteRate & 0xFF); // ByteRate + wavHeader[29] = (byte)((byteRate >> 8) & 0xFF); // ByteRate + wavHeader[30] = (byte)((byteRate >> 16) & 0xFF); // ByteRate + wavHeader[31] = (byte)((byteRate >> 24) & 0xFF); // ByteRate + wavHeader[32] = (byte)blockAlign; // BlockAlign + wavHeader[33] = 0x00; // BlockAlign + wavHeader[34] = (byte)bitsPerSample; // BitsPerSample + wavHeader[35] = 0x00; // BitsPerSample + wavHeader[36] = 0x64; // d + wavHeader[37] = 0x61; // a + wavHeader[38] = 0x74; // t + wavHeader[39] = 0x61; // a + wavHeader[40] = 0x00; // Placeholder for data chunk size (will be updated later) + wavHeader[41] = 0x00; // Placeholder for data chunk size (will be updated later) + wavHeader[42] = 0x00; // Placeholder for data chunk size (will be updated later) + wavHeader[43] = 0x00; // Placeholder for data chunk size (will be updated later) + + writer.Write(wavHeader); + } + } + + if (e.Stream != null) + { + using (BinaryWriter writer = new BinaryWriter(File.Open($"output_{audioFileCount}.wav", FileMode.Append))) + { + writer.Write(e.Stream.ToArray()); + } + } + + // record the last audio time + lastAudioTime = DateTime.Now; + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received"); + })); + await agentClient.Subscribe(new EventHandler((sender, e) => + { + Console.WriteLine($"----> {e} received. Error: {e.Message}"); + })); + + // Start the connection + var settingsConfiguration = new SettingsConfigurationSchema(); + settingsConfiguration.Agent.Think.Provider.Type = "open_ai"; + settingsConfiguration.Agent.Think.Model = "gpt-4o-mini"; + settingsConfiguration.Agent.Think.Instructions = "You are a helpful AI assistant."; + settingsConfiguration.Audio.Output.SampleRate = 48000; + settingsConfiguration.Audio.Input.SampleRate = 16000; + + bool bConnected = await agentClient.Connect(settingsConfiguration); + if (!bConnected) + { + Console.WriteLine("Failed to connect to Deepgram WebSocket server."); + return; + } + + // Microphone streaming + var microphone = new Microphone(agentClient.SendBinary); + microphone.Start(); + + // Wait for the user to press a key + Console.ReadKey(); + + // Stop the microphone + microphone.Stop(); + + // Stop the connection + await agentClient.Stop(); + + // Terminate Libraries + Deepgram.Microphone.Library.Terminate(); + Deepgram.Library.Terminate(); + } + catch (Exception ex) + { + Console.WriteLine($"Exception: {ex.Message}"); + } + } + } +}