-
-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
LLM performance metric tracking (#2825)
* WIP performance metric tracking * fix: patch UI trying to .toFixed() null metric Anthropic tracking migraiton cleanup logs * Apipie implmentation, not tested * Cleanup Anthropic notes, Add support for AzureOpenAI tracking * bedrock token metric tracking * Cohere support * feat: improve default stream handler to track for provider who are actually OpenAI compliant in usage reporting add deepseek support * feat: Add FireworksAI tracking reporting fix: improve handler when usage:null is reported (why?) * Add token reporting for GenericOpenAI * token reporting for koboldcpp + lmstudio * lint * support Groq token tracking * HF token tracking * token tracking for togetherai * LiteLLM token tracking * linting + Mitral token tracking support * XAI token metric reporting * native provider runner * LocalAI token tracking * Novita token tracking * OpenRouter token tracking * Apipie stream metrics * textwebgenui token tracking * perplexity token reporting * ollama token reporting * lint * put back comment * Rip out LC ollama wrapper and use official library * patch images with new ollama lib * improve ollama offline message * fix image handling in ollama llm provider * lint * NVIDIA NIM token tracking * update openai compatbility responses * UI/UX show/hide metrics on click for user preference * update bedrock client --------- Co-authored-by: shatfield4 <[email protected]>
- Loading branch information
1 parent
15abc3f
commit dd7c467
Showing
42 changed files
with
1,770 additions
and
566 deletions.
There are no files selected for viewing
118 changes: 118 additions & 0 deletions
118
...WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/RenderMetrics/index.jsx
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,118 @@ | ||
| import { numberWithCommas } from "@/utils/numbers"; | ||
| import React, { useEffect, useState, useContext } from "react"; | ||
| const MetricsContext = React.createContext(); | ||
| const SHOW_METRICS_KEY = "anythingllm_show_chat_metrics"; | ||
| const SHOW_METRICS_EVENT = "anythingllm_show_metrics_change"; | ||
|
|
||
| /** | ||
| * @param {number} duration - duration in milliseconds | ||
| * @returns {string} | ||
| */ | ||
| function formatDuration(duration) { | ||
| try { | ||
| return duration < 1 | ||
| ? `${(duration * 1000).toFixed(0)}ms` | ||
| : `${duration.toFixed(3)}s`; | ||
| } catch { | ||
| return ""; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Format the output TPS to a string | ||
| * @param {number} outputTps - output TPS | ||
| * @returns {string} | ||
| */ | ||
| function formatTps(outputTps) { | ||
| try { | ||
| return outputTps < 1000 | ||
| ? outputTps.toFixed(2) | ||
| : numberWithCommas(outputTps.toFixed(0)); | ||
| } catch { | ||
| return ""; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Get the show metrics setting from localStorage `anythingllm_show_chat_metrics` key | ||
| * @returns {boolean} | ||
| */ | ||
| function getAutoShowMetrics() { | ||
| return window?.localStorage?.getItem(SHOW_METRICS_KEY) === "true"; | ||
| } | ||
|
|
||
| /** | ||
| * Toggle the show metrics setting in localStorage `anythingllm_show_chat_metrics` key | ||
| * @returns {void} | ||
| */ | ||
| function toggleAutoShowMetrics() { | ||
| const currentValue = getAutoShowMetrics() || false; | ||
| window?.localStorage?.setItem(SHOW_METRICS_KEY, !currentValue); | ||
| window.dispatchEvent( | ||
| new CustomEvent(SHOW_METRICS_EVENT, { | ||
| detail: { showMetricsAutomatically: !currentValue }, | ||
| }) | ||
| ); | ||
| return !currentValue; | ||
| } | ||
|
|
||
| /** | ||
| * Provider for the metrics context that controls the visibility of the metrics | ||
| * per-chat based on the user's preference. | ||
| * @param {React.ReactNode} children | ||
| * @returns {React.ReactNode} | ||
| */ | ||
| export function MetricsProvider({ children }) { | ||
| const [showMetricsAutomatically, setShowMetricsAutomatically] = | ||
| useState(getAutoShowMetrics()); | ||
|
|
||
| useEffect(() => { | ||
| function handleShowingMetricsEvent(e) { | ||
| if (!e?.detail?.hasOwnProperty("showMetricsAutomatically")) return; | ||
| setShowMetricsAutomatically(e.detail.showMetricsAutomatically); | ||
| } | ||
| console.log("Adding event listener for metrics visibility"); | ||
| window.addEventListener(SHOW_METRICS_EVENT, handleShowingMetricsEvent); | ||
| return () => | ||
| window.removeEventListener(SHOW_METRICS_EVENT, handleShowingMetricsEvent); | ||
| }, []); | ||
|
|
||
| return ( | ||
| <MetricsContext.Provider | ||
| value={{ showMetricsAutomatically, setShowMetricsAutomatically }} | ||
| > | ||
| {children} | ||
| </MetricsContext.Provider> | ||
| ); | ||
| } | ||
|
|
||
| /** | ||
| * Render the metrics for a given chat, if available | ||
| * @param {metrics: {duration:number, outputTps: number}} props | ||
| * @returns | ||
| */ | ||
| export default function RenderMetrics({ metrics = {} }) { | ||
| // Inherit the showMetricsAutomatically state from the MetricsProvider so the state is shared across all chats | ||
| const { showMetricsAutomatically, setShowMetricsAutomatically } = | ||
| useContext(MetricsContext); | ||
| if (!metrics?.duration || !metrics?.outputTps) return null; | ||
|
|
||
| return ( | ||
| <button | ||
| type="button" | ||
| onClick={() => setShowMetricsAutomatically(toggleAutoShowMetrics())} | ||
| data-tooltip-id="metrics-visibility" | ||
| data-tooltip-content={ | ||
| showMetricsAutomatically | ||
| ? "Click to only show metrics when hovering" | ||
| : "Click to show metrics as soon as they are available" | ||
| } | ||
| className={`border-none flex justify-end items-center gap-x-[8px] ${showMetricsAutomatically ? "opacity-100" : "opacity-0"} md:group-hover:opacity-100 transition-all duration-300`} | ||
| > | ||
| <p className="cursor-pointer text-xs font-mono text-theme-text-secondary opacity-50"> | ||
| {formatDuration(metrics.duration)} ({formatTps(metrics.outputTps)}{" "} | ||
| tok/s) | ||
| </p> | ||
| </button> | ||
| ); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.