Realtime: Launch PR

bifrost/app/changelog/changes/20250407-realtime-launch/metadata.json

@@ -0,0 +1,4 @@
+{
+  "title": "Helicone Now Supports OpenAI's Realtime API",
+  "description": "Integrate OpenAI's Realtime API with Helicone's standard one-line integration to monitor performance, analyze interactions, and gain insights into your real-time conversations."
+}

bifrost/app/changelog/changes/20250407-realtime-launch/src.mdx

@@ -0,0 +1,39 @@
+We're thrilled to announce that Helicone now supports logging of OpenAI's Realtime API, enabling low-latency, multi-modal conversational experiences.
+
+### Seamless Integration with Helicone
+
+Integrating OpenAI's Realtime API with Helicone is as simple as ever. Following our standard one-line integration approach, you can immediately start monitoring performance, analyzing interactions, and gaining valuable insights into your real-time conversations.
+
+### How it Works
+
+Connect to the Realtime API through Helicone using your preferred provider (OpenAI or Azure). Helicone acts as a proxy, allowing you to leverage our observability features without changing your core application logic.
+
+### Example: Connecting via WebSocket (OpenAI Provider)
+
+```typescript
+// Simply swap with the following url:
+const url =
+  "wss://api.helicone.ai/v1/gateway/oai/realtime?model=gpt-4o-realtime-preview-2024-12-17";
+
+const ws = new WebSocket(url, {
+  headers: {
+    // Your OpenAI Key
+    Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+    // Your Helicone Key
+    "Helicone-Auth": `Bearer ${process.env.HELICONE_API_KEY}`,
+    // Optional Helicone properties for enhanced analytics
+    "Helicone-Session-Id": `session_123`,
+    "Helicone-User-Id": "user_123",
+  },
+});
+```
+
+### Key Benefits
+
+- **Effortless Setup**: Integrate with the standard Helicone proxy URL and API key header.
+- **Real-time Monitoring**: Track latency, token usage, and other critical metrics for your real-time sessions.
+- **Session Analysis**: Utilize Helicone headers like `Helicone-Session-Id` and `Helicone-User-Id` to group and analyze conversations.
+- **Multi-modal Support**: Monitor both text and audio interactions.
+- **Provider Flexibility**: Works seamlessly with both OpenAI and Azure endpoints.
+
+Get started today by updating your WebSocket connection URL and adding your Helicone API key header. For more details, check out the full [OpenAI Realtime Integration documentation](https://docs.helicone.ai/integrations/openai/realtime).

bifrost/app/llm-cost/ModelPriceCalculator.tsx

@@ -1,26 +1,26 @@
 "use client";
 
-import { useState, useEffect, useMemo, useRef } from "react";
+import { ModelCostCardSkeleton } from "@/components/skeletons/ModelCostCardSkeleton";
+import { TableSkeleton } from "@/components/skeletons/TableSkeleton";
+import { Button } from "@/components/ui/button";
+import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { ThemedTextDropDown } from "@/components/ui/themedTextDropDown";
 import {
   Calculator,
-  Twitter,
-  ChevronUp,
   ChevronDown,
+  ChevronUp,
+  Twitter,
   XCircle,
 } from "lucide-react";
-import { costOf, costOfPrompt } from "../../packages/cost"; // Ensure the path is correct
-import { providers } from "../../packages/cost/providers/mappings"; // Ensure the path is correct
 import Image from "next/image";
-import { Label } from "@/components/ui/label";
-import { Input } from "@/components/ui/input";
-import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
-import { usePathname } from "next/navigation";
 import Link from "next/link";
+import { usePathname } from "next/navigation";
+import { useEffect, useMemo, useState } from "react";
+import { costOf, costOfPrompt } from "../../packages/cost"; // Ensure the path is correct
+import { providers } from "../../packages/cost/providers/mappings"; // Ensure the path is correct
 import CalculatorInfo, { formatProviderName } from "./CalculatorInfo";
-import { Button } from "@/components/ui/button";
-import { ThemedTextDropDown } from "@/components/ui/themedTextDropDown";
-import { ModelCostCardSkeleton } from "@/components/skeletons/ModelCostCardSkeleton";
-import { TableSkeleton } from "@/components/skeletons/TableSkeleton";
 import { ProviderWithModels } from "./utils";
 
 // Define and export the CostData type
@@ -223,6 +223,8 @@ export default function ModelPriceCalculator({
             promptCacheWriteTokens: 0,
             promptCacheReadTokens: 0,
             completionTokens: outputTokensNum,
+            completionAudioTokens: 0,
+            promptAudioTokens: 0,
           });
 
           if (costDetails) {

bifrost/app/llm-cost/utils.ts

@@ -32,6 +32,8 @@ export const getInitialCostData = (): CostData[] => {
         promptCacheWriteTokens: 0,
         promptCacheReadTokens: 0,
         completionTokens: outputTokensNum,
+        promptAudioTokens: 0,
+        completionAudioTokens: 0,
       });
 
       if (costDetails) {

bifrost/lib/clients/jawnTypes/private.ts

@@ -392,6 +392,9 @@ export interface paths {
      */
     get: operations["GetSubscriptionData"];
   };
+  "/v1/audio/convert-to-wav": {
+    post: operations["ConvertToWav"];
+  };
 }
 
 export type webhooks = Record<string, never>;
@@ -736,6 +739,7 @@ Json: JsonObject;
       arguments: components["schemas"]["Record_string.any_"];
     };
     Message: {
+      deleted?: boolean;
       contentArray?: components["schemas"]["Message"][];
       /** Format: double */
       idx?: number;
@@ -922,6 +926,10 @@ Json: JsonObject;
       prompt_cache_read_tokens: number | null;
       /** Format: double */
       completion_tokens: number | null;
+      /** Format: double */
+      prompt_audio_tokens: number | null;
+      /** Format: double */
+      completion_audio_tokens: number | null;
       prompt_id: string | null;
       feedback_created_at?: string | null;
       feedback_id?: string | null;
@@ -13996,6 +14004,13 @@ Json: JsonObject;
     "Record_string.stripe.Stripe.Discount_": {
       [key: string]: components["schemas"]["stripe.Stripe.Discount"];
     };
+    ConvertToWavResponse: {
+      data: string | null;
+      error: string | null;
+    };
+    ConvertToWavRequestBody: {
+      audioData: string;
+    };
   };
   responses: {
   };
@@ -16444,4 +16459,19 @@ export interface operations {
       };
     };
   };
+  ConvertToWav: {
+    requestBody: {
+      content: {
+        "application/json": components["schemas"]["ConvertToWavRequestBody"];
+      };
+    };
+    responses: {
+      /** @description Ok */
+      200: {
+        content: {
+          "application/json": components["schemas"]["ConvertToWavResponse"];
+        };
+      };
+    };
+  };
 }

bifrost/lib/clients/jawnTypes/public.ts

@@ -985,6 +985,7 @@ export interface components {
       arguments: components["schemas"]["Record_string.any_"];
     };
     Message: {
+      deleted?: boolean;
       contentArray?: components["schemas"]["Message"][];
       /** Format: double */
       idx?: number;
@@ -1171,6 +1172,10 @@ export interface components {
       prompt_cache_read_tokens: number | null;
       /** Format: double */
       completion_tokens: number | null;
+      /** Format: double */
+      prompt_audio_tokens: number | null;
+      /** Format: double */
+      completion_audio_tokens: number | null;
       prompt_id: string | null;
       feedback_created_at?: string | null;
       feedback_id?: string | null;

bifrost/packages/cost/costCalc.ts

@@ -6,7 +6,9 @@ export function modelCost(modelRow: {
   sum_prompt_tokens: number;
   prompt_cache_write_tokens: number;
   prompt_cache_read_tokens: number;
+  prompt_audio_tokens: number;
   sum_completion_tokens: number;
+  completion_audio_tokens: number;
   sum_tokens: number;
   per_call?: number;
   per_image?: number;
@@ -18,7 +20,9 @@ export function modelCost(modelRow: {
       promptTokens: modelRow.sum_prompt_tokens,
       promptCacheWriteTokens: modelRow.prompt_cache_write_tokens,
       promptCacheReadTokens: modelRow.prompt_cache_read_tokens,
+      promptAudioTokens: modelRow.prompt_audio_tokens,
       completionTokens: modelRow.sum_completion_tokens,
+      completionAudioTokens: modelRow.completion_audio_tokens,
       perCall: modelRow.per_call,
       images: modelRow.per_image,
     }) ?? 0

bifrost/packages/cost/index.ts

@@ -56,7 +56,9 @@ export function costOfPrompt({
   promptTokens,
   promptCacheWriteTokens,
   promptCacheReadTokens,
+  promptAudioTokens,
   completionTokens,
+  completionAudioTokens,
   images = 1,
   perCall = 1,
 }: {
@@ -65,7 +67,9 @@ export function costOfPrompt({
   promptTokens: number;
   promptCacheWriteTokens: number;
   promptCacheReadTokens: number;
+  promptAudioTokens: number;
   completionTokens: number;
+  completionAudioTokens: number;
   images?: number;
   perCall?: number;
 }) {
@@ -93,9 +97,23 @@ export function costOfPrompt({
     totalCost += promptCacheReadTokens * cost.prompt_token;
   }
 
+  // Add cost for prompt audio tokens if applicable
+  if (cost.prompt_audio_token && promptAudioTokens > 0) {
+    totalCost += promptAudioTokens * cost.prompt_audio_token;
+  } else if (promptAudioTokens > 0) {
+    totalCost += promptAudioTokens * cost.prompt_token;
+  }
+
   // Add cost for completion tokens
   totalCost += completionTokens * cost.completion_token;
 
+  // Add cost for completion audio tokens if applicable
+  if (cost.completion_audio_token && completionAudioTokens > 0) {
+    totalCost += completionAudioTokens * cost.completion_audio_token;
+  } else if (completionAudioTokens > 0) {
+    totalCost += completionAudioTokens * cost.completion_token;
+  }
+
   // Add cost for images and per-call fees
   const imageCost = images * (cost.per_image ?? 0);
   const perCallCost = perCall * (cost.per_call ?? 0);
@@ -112,28 +130,63 @@ function caseForCost(costs: ModelRow[], table: string, multiple: number) {
       const costPerMultiple = {
         prompt: Math.round(cost.cost.prompt_token * multiple),
         completion: Math.round(cost.cost.completion_token * multiple),
+        prompt_audio: Math.round(
+          (cost.cost.prompt_audio_token ?? cost.cost.prompt_token) * multiple
+        ),
+        completion_audio: Math.round(
+          (cost.cost.completion_audio_token ?? cost.cost.completion_token) *
+            multiple
+        ),
+        prompt_cache_write: Math.round(
+          (cost.cost.prompt_cache_write_token ?? cost.cost.prompt_token) *
+            multiple
+        ),
+        prompt_cache_read: Math.round(
+          (cost.cost.prompt_cache_read_token ?? cost.cost.prompt_token) *
+            multiple
+        ),
         image: Math.round((cost.cost.per_image ?? 0) * multiple),
         per_call: Math.round((cost.cost.per_call ?? 0) * multiple),
       };
 
-      const costs = [];
+      const costParts = [];
       if (costPerMultiple.prompt > 0) {
-        costs.push(`${costPerMultiple.prompt} * ${table}.prompt_tokens`);
+        costParts.push(`${costPerMultiple.prompt} * ${table}.prompt_tokens`);
       }
       if (costPerMultiple.completion > 0) {
-        costs.push(
+        costParts.push(
           `${costPerMultiple.completion} * ${table}.completion_tokens`
         );
       }
+      if (costPerMultiple.prompt_audio > 0) {
+        costParts.push(
+          `${costPerMultiple.prompt_audio} * ${table}.prompt_audio_tokens`
+        );
+      }
+      if (costPerMultiple.completion_audio > 0) {
+        costParts.push(
+          `${costPerMultiple.completion_audio} * ${table}.completion_audio_tokens`
+        );
+      }
+      if (costPerMultiple.prompt_cache_write > 0) {
+        costParts.push(
+          `${costPerMultiple.prompt_cache_write} * ${table}.prompt_cache_write_tokens`
+        );
+      }
+      if (costPerMultiple.prompt_cache_read > 0) {
+        costParts.push(
+          `${costPerMultiple.prompt_cache_read} * ${table}.prompt_cache_read_tokens`
+        );
+      }
       if (costPerMultiple.image > 0) {
-        costs.push(`${costPerMultiple.image}`);
+        costParts.push(`${costPerMultiple.image}`); // Assuming image cost is per image, not per token
       }
       if (costPerMultiple.per_call > 0) {
-        costs.push(`${costPerMultiple.per_call}`);
+        costParts.push(`${costPerMultiple.per_call}`); // Assuming per_call cost is per call
       }
 
-      if (costs.length > 0) {
-        const costString = costs.join(" + ");
+      if (costParts.length > 0) {
+        const costString = costParts.join(" + ");
         if (cost.model.operator === "equals") {
           return `WHEN (${table}.model ILIKE '${cost.model.value}') THEN ${costString}`;
         } else if (cost.model.operator === "startsWith") {
@@ -144,7 +197,7 @@ function caseForCost(costs: ModelRow[], table: string, multiple: number) {
           throw new Error("Unknown operator");
         }
       } else {
-        return ``;
+        return ``; // Return empty string if no costs apply for this model
       }
     })
     .join("\n")}

bifrost/packages/cost/interfaces/Cost.ts

@@ -39,6 +39,8 @@ export interface ModelRow {
     per_call?: number;
     prompt_cache_write_token?: number;
     prompt_cache_read_token?: number;
+    prompt_audio_token?: number;
+    completion_audio_token?: number;
   };
   showInPlayground?: boolean;
   targetUrl?: string;

bifrost/packages/cost/providers/azure/index.ts

@@ -95,4 +95,29 @@ export const costs: ModelRow[] = [
       completion_token: 0.00003,
     },
   },
+  {
+    model: {
+      operator: "includes",
+      value: "gpt-4o-realtime",
+    },
+    cost: {
+      prompt_token: 0.000005,
+      completion_token: 0.00002,
+      prompt_audio_token: 0.00004,
+      completion_audio_token: 0.00008,
+      prompt_cache_read_token: 0.0000025,
+    },
+  },
+  {
+    model: {
+      operator: "includes",
+      value: "gpt-4o-mini-realtime",
+    },
+    cost: {
+      prompt_token: 0.00000015,
+      completion_token: 0.0000006,
+      prompt_audio_token: 0.00001,
+      completion_audio_token: 0.00002,
+    },
+  },
 ];