From aa8af7778c42cda27468b1e326343059bd42c362 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman <105607645+karthikscale3@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:38:40 -0700 Subject: [PATCH] Release 3.0.8 (#281) * Pagination bug * Bug fix * Update package * Bugfixes * Add migration file * adding ability to add to dataset from traces tab * fixing z-index bug * fixing llm parsing, disable button for non llm traces * cleaning up api providers page * more bug fixes, auditing models * adding api key auth to create project api key (#270) * Improvements to prompt playground (#272) * fixing static text area, fixing overlap * creating custo ui for playground taces * fetching traces on dialog open * fixes * fixes * fixes --------- Co-authored-by: Karthik Kalyanaraman * fix * fix * support for o1-preview and o1-mini (#275) * fixing live prompt bug (#277) * DSPy enhancements (#280) * DSPy experiments * LiteLLM support * eval charts * experiment metrics * empty state * minor fix * fix * clearing invite form fields on send (#279) * Track project creation count metrics (#271) * adding posthog * bug fixes, adding new env vars * adding posthog api key to dockerfile * updating read me * removing sensitive data * removing more data * adding sign up count by team * minor * update * fixes --------- Co-authored-by: Karthik Kalyanaraman * minor --------- Co-authored-by: dylan Co-authored-by: dylanzuber-scale3 <116033320+dylanzuber-scale3@users.noreply.github.com> --- .env | 6 +- Dockerfile | 2 +- README.md | 20 +- app/(protected)/layout.tsx | 13 +- .../[experiment_id]/page-client.tsx | 667 ++++++++++++++++++ .../dspy-experiments/[experiment_id]/page.tsx | 24 + .../dspy-experiments/layout-client.tsx | 180 +++++ .../[project_id]/dspy-experiments/layout.tsx | 11 + .../dspy-experiments/page-client.tsx | 3 + .../[project_id]/dspy-experiments/page.tsx | 24 + app/(protected)/projects/page-client.tsx | 12 + .../settings/members/page-client.tsx | 3 + app/api/metrics/latency/trace/route.ts | 20 +- app/api/metrics/usage/cost/inference/route.ts | 11 +- app/api/metrics/usage/trace/route.ts | 20 +- app/api/project/route.ts | 8 + components/charts/dspy-eval-chart.tsx | 88 +++ components/charts/inference-chart.tsx | 35 +- components/project/metrics.tsx | 7 +- components/project/project-type-dropdown.tsx | 2 +- components/shared/hover-cell.tsx | 85 +++ components/shared/nav.tsx | 7 + components/shared/posthog.tsx | 42 ++ components/shared/tabs.tsx | 21 +- components/shared/vendor-metadata.tsx | 20 + components/traces/trace_graph.tsx | 7 +- lib/auth/options.ts | 4 +- lib/dspy_trace_util.ts | 331 +++++++++ lib/middleware/app.ts | 10 +- lib/services/posthog.ts | 37 + lib/services/trace_service.ts | 29 +- lib/utils.ts | 42 +- package-lock.json | 44 +- package.json | 2 + public/litellm.png | Bin 0 -> 9748 bytes 35 files changed, 1779 insertions(+), 58 deletions(-) create mode 100644 app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page-client.tsx create mode 100644 app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page.tsx create mode 100644 app/(protected)/project/[project_id]/dspy-experiments/layout-client.tsx create mode 100644 app/(protected)/project/[project_id]/dspy-experiments/layout.tsx create mode 100644 app/(protected)/project/[project_id]/dspy-experiments/page-client.tsx create mode 100644 app/(protected)/project/[project_id]/dspy-experiments/page.tsx create mode 100644 components/charts/dspy-eval-chart.tsx create mode 100644 components/shared/posthog.tsx create mode 100644 lib/dspy_trace_util.ts create mode 100644 lib/services/posthog.ts create mode 100644 public/litellm.png diff --git a/.env b/.env index 02821c9f..a37ffcf3 100644 --- a/.env +++ b/.env @@ -31,4 +31,8 @@ NEXT_PUBLIC_ENABLE_ADMIN_LOGIN="true" # Azure AD Variables AZURE_AD_CLIENT_ID="" AZURE_AD_CLIENT_SECRET="" -AZURE_AD_TENANT_ID="" \ No newline at end of file +AZURE_AD_TENANT_ID="" + +POSTHOG_HOST="https://us.i.posthog.com" + +TELEMETRY_ENABLED="true" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 801090c8..1dcc6c41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ WORKDIR /app ARG LANGTRACE_VERSION -RUN NEXT_PUBLIC_ENABLE_ADMIN_LOGIN=true NEXT_PUBLIC_LANGTRACE_VERSION=$LANGTRACE_VERSION npm run build +RUN POSTHOG_API_KEY=$POSTHOG_API_KEY NEXT_PUBLIC_ENABLE_ADMIN_LOGIN=true NEXT_PUBLIC_LANGTRACE_VERSION=$LANGTRACE_VERSION npm run build # Final release image FROM node:21.6-bookworm AS production diff --git a/README.md b/README.md index 920b9395..2c18bc01 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Langtrace.init({ api_key: }) OR ```typescript -import * as Langtrace from '@langtrase/typescript-sdk'; // Must precede any llm module imports +import * as Langtrace from "@langtrase/typescript-sdk"; // Must precede any llm module imports LangTrace.init(); // LANGTRACE_API_KEY as an ENVIRONMENT variable ``` @@ -119,6 +119,23 @@ docker compose down -v `-v` flag is used to delete volumes +## Telemetry + +Langtrace collects basic, non-sensitive usage data from self-hosted instances by default, which is sent to a central server (via PostHog). + +The following telemetry data is collected by us: +- Project name and type +- Team name + +This data helps us to: + +- Understand how the platform is being used to improve key features. +- Monitor overall usage for internal analysis and reporting. + +No sensitive information is gathered, and the data is not shared with third parties. + +If you prefer to disable telemetry, you can do so by setting TELEMETRY_ENABLED=false in your configuration. + --- ## Supported integrations @@ -138,6 +155,7 @@ Langtrace automatically captures traces from the following vendors: | Langchain | Framework | :x: | :white_check_mark: | | LlamaIndex | Framework | :white_check_mark: | :white_check_mark: | | Langgraph | Framework | :x: | :white_check_mark: | +| LiteLLM | Framework | :x: | :white_check_mark: | | DSPy | Framework | :x: | :white_check_mark: | | CrewAI | Framework | :x: | :white_check_mark: | | Ollama | Framework | :x: | :white_check_mark: | diff --git a/app/(protected)/layout.tsx b/app/(protected)/layout.tsx index 622e9cf9..f6c783a1 100644 --- a/app/(protected)/layout.tsx +++ b/app/(protected)/layout.tsx @@ -6,6 +6,7 @@ import { getServerSession } from "next-auth"; import { redirect } from "next/navigation"; import { Suspense } from "react"; import { PageSkeleton } from "./projects/page-client"; +import CustomPostHogProvider from "@/components/shared/posthog"; export default async function Layout({ children, @@ -19,11 +20,13 @@ export default async function Layout({ return ( }> -
-
- - {children} -
+ +
+
+ + {children} +
+ ); } diff --git a/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page-client.tsx b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page-client.tsx new file mode 100644 index 00000000..bf60f79d --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page-client.tsx @@ -0,0 +1,667 @@ +"use client"; +import { + DspyEvalChart, + DspyEvalChartData, +} from "@/components/charts/dspy-eval-chart"; +import { + AverageCostInferenceChart, + CountInferenceChart, +} from "@/components/charts/inference-chart"; +import { TableSkeleton } from "@/components/project/traces/table-skeleton"; +import { TraceSheet } from "@/components/project/traces/trace-sheet"; +import { GenericHoverCell } from "@/components/shared/hover-cell"; +import RowSkeleton from "@/components/shared/row-skeleton"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; +import { Separator } from "@/components/ui/separator"; +import { Skeleton } from "@/components/ui/skeleton"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { PAGE_SIZE } from "@/lib/constants"; +import { DspyTrace, processDspyTrace } from "@/lib/dspy_trace_util"; +import { correctTimestampFormat } from "@/lib/trace_utils"; +import { formatDateTime } from "@/lib/utils"; +import { ResetIcon } from "@radix-ui/react-icons"; +import { + ColumnDef, + flexRender, + getCoreRowModel, + useReactTable, + VisibilityState, +} from "@tanstack/react-table"; +import { ChevronDown, RefreshCwIcon } from "lucide-react"; +import { usePathname } from "next/navigation"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { useBottomScrollListener } from "react-bottom-scroll-listener"; +import { useQuery } from "react-query"; +import { toast } from "sonner"; + +export function PageClient({ email }: { email: string }) { + const pathname = usePathname(); + const project_id = pathname.split("/")[2]; + let experimentName = pathname.split("/")[4]; + // replace dashes with spaces + experimentName = experimentName.replace(/-/g, " "); + + const [data, setData] = useState([]); + const [page, setPage] = useState(1); + const [description, setDescription] = useState(""); + const [totalPages, setTotalPages] = useState(1); + const [openDropdown, setOpenDropdown] = useState(false); + const [openSheet, setOpenSheet] = useState(false); + const [selectedTrace, setSelectedTrace] = useState(null); + const [showBottomLoader, setShowBottomLoader] = useState(false); + const [enableFetch, setEnableFetch] = useState(true); + const [chartData, setChartData] = useState([]); + const [showEvalChart, setShowEvalChart] = useState(false); + + useEffect(() => { + const handleFocusChange = () => { + setPage(1); + setEnableFetch(true); + }; + + window.addEventListener("focus", handleFocusChange); + + return () => { + window.removeEventListener("focus", handleFocusChange); + }; + }, []); + + // Table state + const [tableState, setTableState] = useState({ + pagination: { + pageIndex: 0, + pageSize: 100, + }, + }); + const [columnVisibility, setColumnVisibility] = useState({}); + + const fetchTracesCall = useCallback( + async (pageNum: number) => { + const apiEndpoint = "/api/traces"; + const body = { + page: pageNum, + pageSize: PAGE_SIZE, + projectId: project_id, + filters: { + filters: [ + { + key: "experiment", + operation: "EQUALS", + value: experimentName, + type: "attribute", + }, + ], + operation: "OR", + }, + group: true, + }; + + const response = await fetch(apiEndpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to fetch traces"); + } + return await response.json(); + }, + [project_id] + ); + + useEffect(() => { + if (typeof window !== "undefined") { + try { + const storedState = window.localStorage.getItem( + "preferences.traces.table-view" + ); + if (storedState) { + const parsedState = JSON.parse(storedState); + setTableState((prevState: any) => ({ + ...prevState, + ...parsedState, + pagination: { + ...prevState.pagination, + ...parsedState.pagination, + }, + })); + if (parsedState.columnVisibility) { + setColumnVisibility(parsedState.columnVisibility); + } + } + } catch (error) { + console.error("Error parsing stored table state:", error); + } + } + }, []); + + const fetchTraces = useQuery({ + queryKey: ["fetch-experiments-query", page, experimentName], + queryFn: () => fetchTracesCall(page), + onSuccess: (data) => { + const newData = data?.traces?.result || []; + const metadata = data?.traces?.metadata || {}; + setTotalPages(parseInt(metadata?.total_pages) || 1); + if (parseInt(metadata?.page) <= parseInt(metadata?.total_pages)) { + setPage(parseInt(metadata?.page) + 1); + } + + const transformedNewData: DspyTrace[] = newData.map((trace: any) => { + return processDspyTrace(trace); + }); + + setDescription( + transformedNewData.length > 0 + ? transformedNewData[0].experiment_description + : "" + ); + + if (page === 1) { + setData(transformedNewData); + } else { + setData((prevData) => [...prevData, ...transformedNewData]); + } + + // construct chart data + const chartData: DspyEvalChartData[] = []; + for (const trace of transformedNewData) { + if (trace.evaluated_score) { + chartData.push({ + timestamp: formatDateTime( + correctTimestampFormat(trace.start_time.toString()), + true + ), + score: trace.evaluated_score, + runId: trace.run_id, + }); + } + } + // reverse the chart data to show the latest last + chartData.reverse(); + if (page === 1) { + setChartData(chartData); + } else { + setChartData((prevData) => [...chartData, ...prevData]); + } + + setEnableFetch(false); + setShowBottomLoader(false); + }, + onError: (error) => { + setEnableFetch(false); + setShowBottomLoader(false); + toast.error("Failed to fetch traces", { + description: error instanceof Error ? error.message : String(error), + }); + }, + refetchOnWindowFocus: false, + enabled: enableFetch, + }); + + const scrollableDivRef: any = useBottomScrollListener(() => { + if (fetchTraces.isRefetching) { + return; + } + if (page <= totalPages) { + setShowBottomLoader(true); + fetchTraces.refetch(); + } + }); + + const columns: ColumnDef[] = [ + { + accessorKey: "run_id", + enableResizing: true, + header: "Run ID", + cell: ({ row }) => { + const id = row.getValue("run_id") as string; + return ( +
+ {id} +
+ ); + }, + }, + { + accessorKey: "start_time", + enableResizing: true, + header: "Start Time", + cell: ({ row }) => { + const starttime = row.getValue("start_time") as string; + return ( +
+ {formatDateTime(correctTimestampFormat(starttime), true)} +
+ ); + }, + }, + { + accessorKey: "status", + header: "Status", + cell: ({ row }) => { + const status = row.getValue("status") as string; + return ( +
+ +

+ {status} +

+
+ ); + }, + }, + { + accessorKey: "namespace", + header: "Namespace", + cell: ({ row }) => { + const namespace = row.getValue("namespace") as string; + return ( +
+

{namespace}

+
+ ); + }, + }, + { + accessorKey: "vendors", + header: "Vendors", + cell: ({ row }) => { + const vendors = row.getValue("vendors") as string[]; + return ( +
+ {vendors && + vendors.map((vendor, i) => ( + + {vendor} + + ))} +
+ ); + }, + }, + { + accessorKey: "models", + header: "Models", + cell: ({ row }) => { + const models = row.getValue("models") as string[]; + return ( +
+ {models && + models.map((model, i) => ( + + {model} + + ))} +
+ ); + }, + }, + { + accessorKey: "result", + header: "Result", + cell: ({ row }) => { + const result = row.getValue("result") as any; + return ; + }, + }, + { + accessorKey: "checkpoint", + header: "Checkpoint State", + cell: ({ row }) => { + const result = row.getValue("checkpoint") as any; + return ; + }, + }, + { + accessorKey: "input_tokens", + header: "Input Tokens", + cell: ({ row }) => { + const count = row.getValue("input_tokens") as number; + return

{count}

; + }, + }, + { + accessorKey: "output_tokens", + header: "Output Tokens", + cell: ({ row }) => { + const count = row.getValue("output_tokens") as number; + return

{count}

; + }, + }, + { + accessorKey: "total_tokens", + header: "Total Tokens", + cell: ({ row }) => { + const count = row.getValue("total_tokens") as number; + return

{count}

; + }, + }, + { + accessorKey: "input_cost", + header: "Input Cost", + cell: ({ row }) => { + const cost = row.getValue("input_cost") as number; + if (!cost) { + return null; + } + return ( +

+ {cost.toFixed(6) !== "0.000000" ? `\$${cost.toFixed(6)}` : ""} +

+ ); + }, + }, + { + accessorKey: "output_cost", + header: "Output Cost", + cell: ({ row }) => { + const cost = row.getValue("output_cost") as number; + if (!cost) { + return null; + } + return ( +

+ {cost.toFixed(6) !== "0.000000" ? `\$${cost.toFixed(6)}` : ""} +

+ ); + }, + }, + { + accessorKey: "total_cost", + header: "Total Cost", + cell: ({ row }) => { + const cost = row.getValue("total_cost") as number; + if (!cost) { + return null; + } + return ( +

+ {cost.toFixed(6) !== "0.000000" ? `\$${cost.toFixed(6)}` : ""} +

+ ); + }, + }, + { + accessorKey: "total_duration", + header: "Total Duration", + cell: ({ row }) => { + const duration = row.getValue("total_duration") as number; + if (!duration) { + return null; + } + return

{duration}ms

; + }, + }, + ]; + + const table = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + initialState: { + ...tableState, + pagination: tableState.pagination, + columnVisibility, + }, + state: { + ...tableState, + pagination: tableState.pagination, + columnVisibility, + }, + onStateChange: (newState: any) => { + setTableState((prevState: any) => ({ + ...newState, + pagination: newState.pagination || prevState.pagination, + })); + const currState = table.getState(); + localStorage.setItem( + "preferences.traces.table-view", + JSON.stringify(currState) + ); + }, + onColumnVisibilityChange: (newVisibility) => { + setColumnVisibility(newVisibility); + const currState = table.getState(); + localStorage.setItem( + "preferences.traces.table-view", + JSON.stringify(currState) + ); + }, + enableColumnResizing: true, + columnResizeMode: "onChange", + manualPagination: true, // Add this if you're handling pagination yourself + }); + + const columnSizeVars = useMemo(() => { + const headers = table.getFlatHeaders(); + const colSizes: { [key: string]: number } = {}; + for (let i = 0; i < headers.length; i++) { + const header = headers[i]!; + colSizes[`--header-${header.id}-size`] = header.getSize(); + colSizes[`--col-${header.column.id}-size`] = header.column.getSize(); + } + return colSizes; + }, [table.getState().columnSizingInfo, table.getState().columnSizing]); + + return ( +
+
+
+ +
+

+ {experimentName} +

+ {description && ( +

+ {description} +

+ )} +
+
+
+ + + + + + {table + .getAllColumns() + .filter((column) => column.getCanHide()) + .map((column) => { + return ( + setOpenDropdown(true)} + onCheckedChange={(value) => + column.toggleVisibility(!!value) + } + > + {column.columnDef.header?.toString()} + + ); + })} + + + +
+
+ + + Eval Chart + Inference Metrics + + + {chartData.length > 0 ? ( + + ) : ( +
+

+ No evaluation data. Run dspy Evaluate() to see scores. +

+
+ )} + + +
+ + +
+ + +
+ {fetchTraces.isLoading && } + {!fetchTraces.isLoading && data && data.length > 0 && ( + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext() + )} +
header.column.resetSize()} + onMouseDown={header.getResizeHandler()} + onTouchStart={header.getResizeHandler()} + className={`bg-muted-foreground resizer ${ + header.column.getIsResizing() ? "isResizing" : "" + }`} + /> + + ))} + + ))} + + + {fetchTraces.isFetching && ( + + {table.getFlatHeaders().map((header) => ( + + + + ))} + + )} + {table.getRowModel().rows.map((row) => ( + { + setSelectedTrace(data[row.index] as DspyTrace); + setOpenSheet(true); + }} + > + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext() + )} + + ))} + + ))} + +
+ )} + {selectedTrace !== null && ( + + )} + {showBottomLoader && ( +
+ + {Array.from({ length: 2 }).map((_, index) => ( + + ))} +
+ )} +
+
+ ); +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page.tsx b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page.tsx new file mode 100644 index 00000000..c0dee8b7 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/[experiment_id]/page.tsx @@ -0,0 +1,24 @@ +import { authOptions } from "@/lib/auth/options"; +import { Metadata } from "next"; +import { getServerSession } from "next-auth"; +import { redirect } from "next/navigation"; +import { PageClient } from "./page-client"; + +export const metadata: Metadata = { + title: "Langtrace | Experiment", + description: "DSPy experiment.", +}; + +export default async function Page() { + const session = await getServerSession(authOptions); + if (!session || !session.user) { + redirect("/login"); + } + const user = session?.user; + + return ( + <> + + + ); +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/layout-client.tsx b/app/(protected)/project/[project_id]/dspy-experiments/layout-client.tsx new file mode 100644 index 00000000..eecf093e --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/layout-client.tsx @@ -0,0 +1,180 @@ +"use client"; +import { SetupInstructions } from "@/components/shared/setup-instructions"; +import Tabs from "@/components/shared/tabs"; +import { Skeleton } from "@/components/ui/skeleton"; +import { PAGE_SIZE } from "@/lib/constants"; +import { processDspyTrace } from "@/lib/dspy_trace_util"; +import { CircularProgress } from "@mui/material"; +import { usePathname } from "next/navigation"; +import { useCallback, useState } from "react"; +import { useBottomScrollListener } from "react-bottom-scroll-listener"; +import { useQuery } from "react-query"; +import { toast } from "sonner"; + +export default function LayoutClient({ + children, +}: { + children: React.ReactNode; +}) { + const pathname = usePathname(); + const project_id = pathname.split("/")[2]; + const href = `/project/${project_id}/dspy-experiments`; + + const [page, setPage] = useState(1); + const [totalPages, setTotalPages] = useState(1); + const [showBottomLoader, setShowBottomLoader] = useState(false); + const [experiments, setExperiments] = useState([]); + const [navLinks, setNavLinks] = useState([]); + + const fetchTracesCall = useCallback( + async (pageNum: number) => { + const apiEndpoint = "/api/traces"; + const body = { + page: pageNum, + pageSize: PAGE_SIZE, + projectId: project_id, + filters: { + filters: [ + { + key: "experiment", + operation: "NOT_EQUALS", + value: "", + type: "attribute", + }, + ], + operation: "OR", + }, + group: true, + }; + + const response = await fetch(apiEndpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const error = await response.json(); + throw new Error(error?.message || "Failed to fetch traces"); + } + return await response.json(); + }, + [project_id] + ); + + const fetchTraces = useQuery({ + queryKey: ["fetch-experiments-query", page], + queryFn: () => fetchTracesCall(page), + onSuccess: (data) => { + const newData = data?.traces?.result || []; + const metadata = data?.traces?.metadata || {}; + + const transformedNewData = newData.map((trace: any) => { + return processDspyTrace(trace); + }); + + const exps: string[] = []; + const navs: any[] = []; + for (const trace of transformedNewData) { + if ( + trace.experiment_name !== "" && + !exps.includes(trace.experiment_name) + ) { + exps.push(trace.experiment_name); + navs.push({ + name: trace.experiment_name, + value: + trace.experiment_name !== "" ? trace.experiment_name : "default", + href: `${href}/${trace.experiment_name}`, + }); + } + } + + // dedupe experiments from current and new data + const newExperiments = Array.from(new Set([...experiments, ...exps])); + setExperiments(newExperiments); + + // dedupe navLinks from current and new data + const newNavLinks = []; + for (const nav of navs) { + if (!navLinks.find((n) => n.value === nav.value)) { + newNavLinks.push(nav); + } + } + setNavLinks([...navLinks, ...newNavLinks]); + + // route to first experiment if no experiment is selected + if (navLinks.length > 0) { + const experimentId = navLinks[0].value; + if (!pathname.includes(experimentId)) { + window.location.href = `${href}/${experimentId}`; + } + } + + setTotalPages(parseInt(metadata?.total_pages) || 1); + if (parseInt(metadata?.page) <= parseInt(metadata?.total_pages)) { + setPage(parseInt(metadata?.page) + 1); + } + setShowBottomLoader(false); + }, + onError: (error) => { + setShowBottomLoader(false); + toast.error("Failed to fetch traces", { + description: error instanceof Error ? error.message : String(error), + }); + }, + }); + + const scrollableDivRef: any = useBottomScrollListener(() => { + if (fetchTraces.isRefetching) { + return; + } + if (page <= totalPages) { + setShowBottomLoader(true); + fetchTraces.refetch(); + } + }); + + return ( +
+
+

Experiments

+
+ {!fetchTraces.isLoading && experiments.length > 0 && ( +
+ +
+ {children} +
+
+ )} + {!fetchTraces.isLoading && experiments.length === 0 && ( +
+
+ +

+ Looking for new experiments... +

+
+ +
+ )} + {fetchTraces.isLoading && ( +
+
+ + + + +
+ +
+ )} +
+ ); +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/layout.tsx b/app/(protected)/project/[project_id]/dspy-experiments/layout.tsx new file mode 100644 index 00000000..bddf5627 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/layout.tsx @@ -0,0 +1,11 @@ +import { Metadata } from "next"; +import LayoutClient from "./layout-client"; + +export const metadata: Metadata = { + title: "Langtrace | Experiments", + description: "Manage your DSPy experiments.", +}; + +export default function Layout({ children }: { children: React.ReactNode }) { + return ; +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/page-client.tsx b/app/(protected)/project/[project_id]/dspy-experiments/page-client.tsx new file mode 100644 index 00000000..4d0ace06 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/page-client.tsx @@ -0,0 +1,3 @@ +export function PageClient({ email }: { email: string }) { + return
; +} diff --git a/app/(protected)/project/[project_id]/dspy-experiments/page.tsx b/app/(protected)/project/[project_id]/dspy-experiments/page.tsx new file mode 100644 index 00000000..8d08f1e1 --- /dev/null +++ b/app/(protected)/project/[project_id]/dspy-experiments/page.tsx @@ -0,0 +1,24 @@ +import { authOptions } from "@/lib/auth/options"; +import { Metadata } from "next"; +import { getServerSession } from "next-auth"; +import { redirect } from "next/navigation"; +import { PageClient } from "./page-client"; + +export const metadata: Metadata = { + title: "Langtrace | Traces", + description: "View all traces for a project.", +}; + +export default async function Page() { + const session = await getServerSession(authOptions); + if (!session || !session.user) { + redirect("/login"); + } + const user = session?.user; + + return ( + <> + + + ); +} diff --git a/app/(protected)/projects/page-client.tsx b/app/(protected)/projects/page-client.tsx index a7451b47..d7b39a7a 100644 --- a/app/(protected)/projects/page-client.tsx +++ b/app/(protected)/projects/page-client.tsx @@ -194,6 +194,18 @@ function ProjectCard({ {project.name} )} + {project.type === "dspy" && ( + + DSPy Logo + {project.name} + + )} {project.type === "default" && ( {project.name} diff --git a/app/(protected)/settings/members/page-client.tsx b/app/(protected)/settings/members/page-client.tsx index ea27aa3a..aaab18a2 100644 --- a/app/(protected)/settings/members/page-client.tsx +++ b/app/(protected)/settings/members/page-client.tsx @@ -62,6 +62,7 @@ export function InviteMember({ user }: { user: any }) { const MemberDetailsForm = useForm({ resolver: zodResolver(MemberDetailsFormSchema), }); + const { reset } = MemberDetailsForm; const sendMemberInvitation = async (data: FieldValues) => { try { @@ -92,6 +93,8 @@ export function InviteMember({ user }: { user: any }) { description: `Please have the invited person sign up with their ${data.email} email`, }); setOpen(false); + MemberDetailsForm.setValue("email", ""); + MemberDetailsForm.setValue("name", ""); } catch (error) { toast.error( "An error occurred while inviting your team member. Please try again later." diff --git a/app/api/metrics/latency/trace/route.ts b/app/api/metrics/latency/trace/route.ts index 036a2ad6..98883416 100644 --- a/app/api/metrics/latency/trace/route.ts +++ b/app/api/metrics/latency/trace/route.ts @@ -15,9 +15,22 @@ export async function GET(req: NextRequest) { const lastNHours = parseInt( req.nextUrl.searchParams.get("lastNHours") || "168" ); - const userId = req.nextUrl.searchParams.get("userId") || ""; - const model = req.nextUrl.searchParams.get("model") || ""; - const inference = req.nextUrl.searchParams.get("inference") || ""; + const userId = + req.nextUrl.searchParams.get("userId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("userId") as string); + const model = + req.nextUrl.searchParams.get("model") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("model") as string); + const inference = + req.nextUrl.searchParams.get("inference") === "undefined" + ? "false" + : (req.nextUrl.searchParams.get("inference") as string); + const experimentId = + req.nextUrl.searchParams.get("experimentId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("experimentId") as string); if (!projectId) { return NextResponse.json( @@ -34,6 +47,7 @@ export async function GET(req: NextRequest) { projectId, lastNHours, userId, + experimentId, model, inference === "true" ); diff --git a/app/api/metrics/usage/cost/inference/route.ts b/app/api/metrics/usage/cost/inference/route.ts index 480f1076..c855221a 100644 --- a/app/api/metrics/usage/cost/inference/route.ts +++ b/app/api/metrics/usage/cost/inference/route.ts @@ -13,9 +13,18 @@ export async function GET(req: NextRequest) { try { const projectId = req.nextUrl.searchParams.get("projectId") as string; + const experimentId = req.nextUrl.searchParams.get("experimentId") as string; + const experimentFilter: { [key: string]: string } = {}; + + if (experimentId) { + experimentFilter["experiment"] = experimentId; + } const traceService = new TraceService(); - const cost = await traceService.GetInferenceCostPerProject(projectId); + const cost = await traceService.GetInferenceCostPerProject( + projectId, + experimentFilter + ); const inferenceCount = await traceService.GetTotalTracesPerProject( projectId, true diff --git a/app/api/metrics/usage/trace/route.ts b/app/api/metrics/usage/trace/route.ts index 5282b1e8..21b79503 100644 --- a/app/api/metrics/usage/trace/route.ts +++ b/app/api/metrics/usage/trace/route.ts @@ -15,9 +15,22 @@ export async function GET(req: NextRequest) { const lastNHours = parseInt( req.nextUrl.searchParams.get("lastNHours") || "168" ); - const userId = req.nextUrl.searchParams.get("userId") || ""; - const model = req.nextUrl.searchParams.get("model") || ""; - const inference = req.nextUrl.searchParams.get("inference") || ""; + const userId = + req.nextUrl.searchParams.get("userId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("userId") as string); + const model = + req.nextUrl.searchParams.get("model") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("model") as string); + const inference = + req.nextUrl.searchParams.get("inference") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("inference") as string); + const experimentId = + req.nextUrl.searchParams.get("experimentId") === "undefined" + ? undefined + : (req.nextUrl.searchParams.get("experimentId") as string); if (!projectId) { return NextResponse.json( @@ -33,6 +46,7 @@ export async function GET(req: NextRequest) { projectId, lastNHours, userId, + experimentId, model, inference === "true" ); diff --git a/app/api/project/route.ts b/app/api/project/route.ts index 97ee506d..27b8deb2 100644 --- a/app/api/project/route.ts +++ b/app/api/project/route.ts @@ -1,6 +1,7 @@ import { authOptions } from "@/lib/auth/options"; import { DEFAULT_TESTS } from "@/lib/constants"; import prisma from "@/lib/prisma"; +import { captureEvent } from "@/lib/services/posthog"; import { authApiKey } from "@/lib/utils"; import { getServerSession } from "next-auth"; import { redirect } from "next/navigation"; @@ -87,6 +88,13 @@ export async function POST(req: NextRequest) { }, }); } + + const session = await getServerSession(authOptions); + const userEmail = session?.user?.email ?? "anonymous"; + await captureEvent(project.id, "project_created", { + project_name: project.name, + project_type: projectType, + }); } const { apiKeyHash, ...projectWithoutApiKeyHash } = project; diff --git a/components/charts/dspy-eval-chart.tsx b/components/charts/dspy-eval-chart.tsx new file mode 100644 index 00000000..d2e2e90a --- /dev/null +++ b/components/charts/dspy-eval-chart.tsx @@ -0,0 +1,88 @@ +"use client"; + +import { CartesianGrid, Line, LineChart, XAxis, YAxis } from "recharts"; + +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { + ChartConfig, + ChartContainer, + ChartTooltip, + ChartTooltipContent, +} from "@/components/ui/chart"; +import LargeChartSkeleton from "./large-chart-skeleton"; + +export interface DspyEvalChartData { + timestamp: string; + score: number; + runId: string; +} + +const chartConfig = { + score: { + label: "Score", + color: "hsl(var(--chart-1))", + }, + runId: { + label: "Run ID", + color: "hsl(var(--chart-2))", + }, +} satisfies ChartConfig; + +export function DspyEvalChart({ + data, + isLoading, +}: { + data: DspyEvalChartData[]; + isLoading: boolean; +}) { + if (isLoading) { + return ; + } + return ( + + + Evaluated scores across runs + + + + + + + + } /> + + + + + + ); +} diff --git a/components/charts/inference-chart.tsx b/components/charts/inference-chart.tsx index 176089e4..91382637 100644 --- a/components/charts/inference-chart.tsx +++ b/components/charts/inference-chart.tsx @@ -12,10 +12,12 @@ export function CountInferenceChart({ projectId, lastNHours = 168, userId, + experimentId, model, }: { projectId: string; lastNHours?: number; + experimentId?: string; userId?: string; model?: string; }) { @@ -32,9 +34,11 @@ export function CountInferenceChart({ model, ], queryFn: async () => { - const response = await fetch( - `/api/metrics/usage/trace?projectId=${projectId}&lastNHours=${lastNHours}&userId=${userId}&model=${model}&inference=true` - ); + let url = `/api/metrics/usage/trace?projectId=${projectId}&lastNHours=${lastNHours}&userId=${userId}&model=${model}&inference=true`; + if (experimentId) { + url += `&experimentId=${experimentId}`; + } + const response = await fetch(url); if (!response.ok) { const error = await response.json(); throw new Error(error?.message || "Failed to fetch inference count"); @@ -98,31 +102,24 @@ export function CountInferenceChart({ export function AverageCostInferenceChart({ projectId, - lastNHours = 168, - userId, - model, + experimentId, }: { projectId: string; - lastNHours?: number; - userId?: string; - model?: string; + experimentId?: string; }) { const { data: costUsage, isLoading: costUsageLoading, error: costUsageError, } = useQuery({ - queryKey: [ - "fetch-metrics-inference-cost", - projectId, - lastNHours, - userId, - model, - ], + queryKey: ["fetch-metrics-inference-cost", projectId, experimentId], queryFn: async () => { - const response = await fetch( - `/api/metrics/usage/cost/inference?projectId=${projectId}` - ); + let url = `/api/metrics/usage/cost/inference?projectId=${projectId}`; + if (experimentId) { + url += `&experimentId=${experimentId}`; + } + const response = await fetch(url); + if (!response.ok) { const error = await response.json(); throw new Error( diff --git a/components/project/metrics.tsx b/components/project/metrics.tsx index e91720a5..df4ab842 100644 --- a/components/project/metrics.tsx +++ b/components/project/metrics.tsx @@ -98,12 +98,7 @@ export default function Metrics({ email }: { email: string }) { projectId={project_id} lastNHours={lastNHours} /> - +
diff --git a/components/project/project-type-dropdown.tsx b/components/project/project-type-dropdown.tsx index 9df65238..cc264b28 100644 --- a/components/project/project-type-dropdown.tsx +++ b/components/project/project-type-dropdown.tsx @@ -30,7 +30,7 @@ const projectTypes = [ { value: "dspy", label: "DSPy", - comingSoon: true, + comingSoon: false, }, { value: "langgraph", diff --git a/components/shared/hover-cell.tsx b/components/shared/hover-cell.tsx index b475f435..4eb5d882 100644 --- a/components/shared/hover-cell.tsx +++ b/components/shared/hover-cell.tsx @@ -129,3 +129,88 @@ export function HoverCell({ return null; } } + +export function GenericHoverCell({ + value, + className, + expand = false, +}: { + value: any; + className?: string; + expand?: boolean; +}) { + try { + const [expandedView, setExpandedView] = useState(expand); + const content = safeStringify(value); + + if (!content || content.length === 0) { + return null; + } + + const copyToClipboard = (e: any) => { + e.stopPropagation(); + navigator.clipboard.writeText(content); + toast.success("Copied to clipboard"); + }; + + return ( + + +
+ + {!expandedView && ( + { + e.stopPropagation(); + setExpandedView(!expandedView); + }} + /> + )} + {expandedView && ( + { + e.stopPropagation(); + setExpandedView(!expandedView); + }} + /> + )} +
+
+ + e.stopPropagation()} + > +
+ +
+
+ + + ); + } catch (e) { + return null; + } +} diff --git a/components/shared/nav.tsx b/components/shared/nav.tsx index 0292da78..603b5a83 100644 --- a/components/shared/nav.tsx +++ b/components/shared/nav.tsx @@ -55,6 +55,13 @@ const ProjectNavLinks = (id: string, type = "default") => { href: `/project/${id}/crewai-dash`, }); } + if (type == "dspy") { + // add to the second position + result.splice(1, 0, { + name: "Experiments", + href: `/project/${id}/dspy-experiments`, + }); + } return result; }; diff --git a/components/shared/posthog.tsx b/components/shared/posthog.tsx new file mode 100644 index 00000000..a6599297 --- /dev/null +++ b/components/shared/posthog.tsx @@ -0,0 +1,42 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import posthog from "posthog-js"; +import { PostHogProvider } from "posthog-js/react"; +import { Session } from "next-auth"; + +interface CustomPostHogProviderProps { + children: React.ReactNode; + session: Session | null; +} + +export default function CustomPostHogProvider({ + children, + session, +}: CustomPostHogProviderProps) { + const [telemetryEnabled, setTelemetryEnabled] = useState(false); + + useEffect(() => { + async function initializePostHog() { + const enabled = process.env.TELEMETRY_ENABLED === "true"; + setTelemetryEnabled(enabled); + + if (enabled && typeof window !== "undefined") { + posthog.init(process.env.POSTHOG_API_KEY!, { + api_host: "https://app.posthog.com", + loaded: (posthog) => { + if (process.env.NODE_ENV === "development") posthog.debug(); + }, + }); + } + } + + initializePostHog(); + }, [session]); + + if (!telemetryEnabled) { + return <>{children}; + } + + return {children}; +} diff --git a/components/shared/tabs.tsx b/components/shared/tabs.tsx index b9827562..cad73b6d 100644 --- a/components/shared/tabs.tsx +++ b/components/shared/tabs.tsx @@ -1,6 +1,7 @@ "use client"; import Link from "next/link"; import { usePathname } from "next/navigation"; +import { Skeleton } from "../ui/skeleton"; export interface Tab { name: string; @@ -8,24 +9,36 @@ export interface Tab { href: string; } -export default function Tabs({ tabs }: { tabs: Tab[] }) { +export default function Tabs({ + tabs, + paginationLoading = false, + scrollableDivRef, +}: { + tabs: Tab[]; + paginationLoading?: boolean; + scrollableDivRef?: React.RefObject; +}) { const pathname = usePathname(); return ( -
+
{tabs.map((tab, idx) => ( {tab.name} ))} + {paginationLoading && }
); } diff --git a/components/shared/vendor-metadata.tsx b/components/shared/vendor-metadata.tsx index a56a2855..769ba083 100644 --- a/components/shared/vendor-metadata.tsx +++ b/components/shared/vendor-metadata.tsx @@ -62,6 +62,10 @@ export function vendorBadgeColor(vendor: string) { return "bg-red-500"; } + if (vendor.includes("litellm")) { + return "bg-blue-500"; + } + return "bg-gray-500"; } @@ -451,6 +455,22 @@ export function VendorLogo({ ); } + if (vendor.includes("litellm")) { + const color = vendorColor("litellm"); + return ( + LiteLLM Logo + ); + } + return (
= ({ serviceName.includes("llamaindex") ) color = "bg-indigo-500"; - else if ( - span.name.includes("vercel") || - serviceName.includes("vercel") - ) + else if (span.name.includes("vercel") || serviceName.includes("vercel")) color = "bg-gray-500"; else if ( span.name.includes("embedchain") || serviceName.includes("embedchain") ) color = "bg-slate-500"; + else if (span.name.includes("litellm") || serviceName.includes("litellm")) + color = "bg-blue-500"; const fillColor = color.replace("bg-", "fill-"); const vendor = getVendorFromSpan(span as any); diff --git a/lib/auth/options.ts b/lib/auth/options.ts index 0c578ce2..383be218 100644 --- a/lib/auth/options.ts +++ b/lib/auth/options.ts @@ -1,9 +1,9 @@ import prisma from "@/lib/prisma"; import { PrismaAdapter } from "@next-auth/prisma-adapter"; import { type NextAuthOptions } from "next-auth"; +import AzureADProvider from "next-auth/providers/azure-ad"; import CredentialsProvider from "next-auth/providers/credentials"; import GoogleProvider from "next-auth/providers/google"; -import AzureADProvider from 'next-auth/providers/azure-ad'; export const authOptions: NextAuthOptions = { providers: [ @@ -64,7 +64,7 @@ export const authOptions: NextAuthOptions = { clientSecret: process.env.AZURE_AD_CLIENT_SECRET as string, tenantId: process.env.AZURE_AD_TENANT_ID as string, allowDangerousEmailAccountLinking: true, - }) + }), ], adapter: PrismaAdapter(prisma), session: { strategy: "jwt" }, diff --git a/lib/dspy_trace_util.ts b/lib/dspy_trace_util.ts new file mode 100644 index 00000000..4bfb0ddb --- /dev/null +++ b/lib/dspy_trace_util.ts @@ -0,0 +1,331 @@ +import { calculateTotalTime, convertTracesToHierarchy } from "./trace_utils"; +import { calculatePriceFromUsage } from "./utils"; + +export interface DspyTrace { + id: string; + run_id: string; + experiment_name: string; + experiment_description: string; + status: string; + namespace: string; + user_ids: string[]; + prompt_ids: string[]; + prompt_versions: string[]; + models: string[]; + vendors: string[]; + inputs: Record[]; + outputs: Record[]; + input_tokens: number; + output_tokens: number; + total_tokens: number; + input_cost: number; + output_cost: number; + total_cost: number; + start_time: number; + total_duration: number; + all_events: any[]; + sorted_trace: any[]; + trace_hierarchy: any[]; + raw_attributes: any; + result: any; + checkpoint: any; + evaluated_score?: number; +} + +export function processDspyTrace(trace: any): DspyTrace { + const traceHierarchy = convertTracesToHierarchy(trace); + const totalTime = calculateTotalTime(trace); + const startTime = trace[0].start_time; + let tokenCounts: any = {}; + let models: string[] = []; + let vendors: string[] = []; + let userIds: string[] = []; + let promptIds: string[] = []; + let promptVersions: string[] = []; + let messages: Record[] = []; + let allEvents: any[] = []; + let attributes: any = {}; + let cost = { total: 0, input: 0, output: 0 }; + let experiment_name = "default"; + let experiment_description = ""; + let run_id = "unspecified"; + let spanResult: any = {}; + let checkpoint: any = {}; + let evaluated_score: number | undefined; + // set status to ERROR if any span has an error + let status = "success"; + for (const span of trace) { + if (span.status_code === "ERROR") { + status = "error"; + break; + } + } + + for (const span of trace) { + if (span.attributes) { + // parse the attributes of the span + attributes = JSON.parse(span.attributes); + let vendor = ""; + + let resultContent = ""; + if (attributes["dspy.signature.result"]) { + resultContent = attributes["dspy.signature.result"]; + } else if (attributes["dspy.evaluate.result"]) { + resultContent = attributes["dspy.evaluate.result"]; + evaluated_score = attributes["dspy.evaluate.result"]; + } + + if (resultContent) { + try { + spanResult = JSON.parse(resultContent); + } catch (e) { + spanResult = resultContent; + } + } + + if (attributes["dspy.checkpoint"]) { + checkpoint = JSON.parse(attributes["dspy.checkpoint"]); + } + + // get the service name from the attributes + if (attributes["langtrace.service.name"]) { + vendor = attributes["langtrace.service.name"].toLowerCase(); + if (!vendors.includes(vendor)) vendors.push(vendor); + } + + // get the experiment name from the attributes + if (attributes["experiment"]) { + experiment_name = attributes["experiment"] + .toLowerCase() + .replace(/\s/g, "-"); + } + + // get the run_id from the attributes + if (attributes["run_id"]) { + run_id = attributes["run_id"].toLowerCase().replace(/\s/g, "-"); + } + + // get the experiment description from the attributes + if (attributes["description"]) { + experiment_description = attributes["description"]; + } + + // get the user_id, prompt_id, prompt_version, and model from the attributes + if (attributes["user_id"]) { + userIds.push(attributes["user_id"]); + } + if (attributes["prompt_id"]) { + promptIds.push(attributes["prompt_id"]); + } + if (attributes["prompt_version"]) { + promptVersions.push(attributes["prompt_version"]); + } + + let model = ""; + if ( + attributes["gen_ai.response.model"] || + attributes["llm.model"] || + attributes["gen_ai.request.model"] + ) { + model = + attributes["gen_ai.response.model"] || + attributes["llm.model"] || + attributes["gen_ai.request.model"]; + models.push(model); + } + // TODO(Karthik): This logic is for handling old traces that were not compatible with the gen_ai conventions. + if (attributes["llm.prompts"] && attributes["llm.responses"]) { + const message = { + prompt: attributes["llm.prompts"], + response: attributes["llm.responses"], + }; + messages.push(message); + } + + if ( + attributes["gen_ai.usage.prompt_tokens"] && + attributes["gen_ai.usage.completion_tokens"] + ) { + tokenCounts = { + input_tokens: tokenCounts.prompt_tokens + ? tokenCounts.prompt_tokens + + attributes["gen_ai.usage.prompt_tokens"] + : attributes["gen_ai.usage.prompt_tokens"], + output_tokens: tokenCounts.completion_tokens + ? tokenCounts.completion_tokens + + attributes["gen_ai.usage.completion_tokens"] + : attributes["gen_ai.usage.completion_tokens"], + total_tokens: tokenCounts.total_tokens + ? tokenCounts.total_tokens + + attributes["gen_ai.usage.prompt_tokens"] + + attributes["gen_ai.usage.completion_tokens"] + : attributes["gen_ai.usage.prompt_tokens"] + + attributes["gen_ai.usage.completion_tokens"], + }; + + // calculate the cost of the current span + const currentcost = calculatePriceFromUsage(vendor, model, tokenCounts); + + // add the cost of the current span to the total cost + cost.total += currentcost.total; + cost.input += currentcost.input; + cost.output += currentcost.output; + } else if ( + attributes["gen_ai.usage.input_tokens"] && + attributes["gen_ai.usage.output_tokens"] + ) { + tokenCounts = { + input_tokens: tokenCounts.prompt_tokens + ? tokenCounts.prompt_tokens + + attributes["gen_ai.usage.input_tokens"] + : attributes["gen_ai.usage.input_tokens"], + output_tokens: tokenCounts.completion_tokens + ? tokenCounts.completion_tokens + + attributes["gen_ai.usage.output_tokens"] + : attributes["gen_ai.usage.output_tokens"], + total_tokens: tokenCounts.total_tokens + ? tokenCounts.total_tokens + + attributes["gen_ai.usage.input_tokens"] + + attributes["gen_ai.usage.output_tokens"] + : attributes["gen_ai.usage.input_tokens"] + + attributes["gen_ai.usage.output_tokens"], + }; + const currentcost = calculatePriceFromUsage(vendor, model, tokenCounts); + // add the cost of the current span to the total cost + cost.total += currentcost.total; + cost.input += currentcost.input; + cost.output += currentcost.output; + } else if (attributes["llm.token.counts"]) { + // TODO(Karthik): This logic is for handling old traces that were not compatible with the gen_ai conventions. + const currentcounts = JSON.parse(attributes["llm.token.counts"]); + tokenCounts = { + input_tokens: tokenCounts.input_tokens + ? tokenCounts.input_tokens + currentcounts.input_tokens + : currentcounts.input_tokens, + output_tokens: tokenCounts.output_tokens + ? tokenCounts.output_tokens + currentcounts.output_tokens + : currentcounts.output_tokens, + total_tokens: tokenCounts.total_tokens + ? tokenCounts.total_tokens + currentcounts.total_tokens + : currentcounts.total_tokens, + }; + + // calculate the cost of the current span + const currentcost = calculatePriceFromUsage( + vendor, + model, + currentcounts + ); + // add the cost of the current span to the total cost + cost.total += currentcost.total; + cost.input += currentcost.input; + cost.output += currentcost.output; + } + } + + // TODO(Karthik): This logic is for handling old traces that were not compatible with the gen_ai conventions. + const message: Record = { + prompts: [], + responses: [], + }; + + if (attributes["llm.prompts"]) { + message.prompts.push(attributes["llm.prompts"]); + } + + if (attributes["llm.responses"]) { + message.responses.push(attributes["llm.responses"]); + } + + if (message.prompts.length > 0 || message.responses.length > 0) { + messages.push(message); + } + + if (span.events && span.events !== "[]") { + const events = JSON.parse(span.events); + const inputs = []; + const outputs = []; + allEvents.push(events); + + // find event with name 'gen_ai.content.prompt' + const promptEvent = events.find( + (event: any) => event.name === "gen_ai.content.prompt" + ); + if ( + promptEvent && + promptEvent["attributes"] && + promptEvent["attributes"]["gen_ai.prompt"] + ) { + inputs.push(promptEvent["attributes"]["gen_ai.prompt"]); + } + + // find event with name 'gen_ai.content.completion' + const responseEvent = events.find( + (event: any) => event.name === "gen_ai.content.completion" + ); + if ( + responseEvent && + responseEvent["attributes"] && + responseEvent["attributes"]["gen_ai.completion"] + ) { + outputs.push(responseEvent["attributes"]["gen_ai.completion"]); + } + + const message: Record = { + prompts: [], + responses: [], + }; + if (inputs.length > 0) { + message.prompts.push(...inputs); + } + if (outputs.length > 0) { + message.responses.push(...outputs); + } + if (message.prompts.length > 0 || message.responses.length > 0) { + messages.push(message); + } + } + } + + // Sort the trace based on start_time, then end_time + trace.sort((a: any, b: any) => { + if (a.start_time === b.start_time) { + return a.end_time < b.end_time ? 1 : -1; + } + return a.start_time < b.start_time ? -1 : 1; + }); + + // construct the response object + const result: DspyTrace = { + id: trace[0]?.trace_id, + run_id: run_id, + experiment_name: experiment_name, + experiment_description: experiment_description, + status: status, + namespace: traceHierarchy[0].name, + user_ids: userIds, + prompt_ids: promptIds, + prompt_versions: promptVersions, + models: models, + vendors: vendors, + inputs: messages, + outputs: messages, + all_events: allEvents, + input_tokens: tokenCounts.input_tokens, + output_tokens: tokenCounts.output_tokens, + total_tokens: tokenCounts.total_tokens, + input_cost: cost.input, + output_cost: cost.output, + total_cost: cost.total, + total_duration: totalTime, + start_time: startTime, + sorted_trace: trace, + trace_hierarchy: traceHierarchy, + raw_attributes: attributes, + result: spanResult, + checkpoint: checkpoint, + evaluated_score: evaluated_score, + }; + + return result; +} diff --git a/lib/middleware/app.ts b/lib/middleware/app.ts index 5f5eb498..14fac36e 100644 --- a/lib/middleware/app.ts +++ b/lib/middleware/app.ts @@ -1,6 +1,7 @@ import { User } from "@prisma/client"; import { getToken } from "next-auth/jwt"; import { NextRequest, NextResponse } from "next/server"; +import { captureEvent } from "../services/posthog"; export default async function AppMiddleware(req: NextRequest) { const path = req.nextUrl.pathname; @@ -36,9 +37,10 @@ export default async function AppMiddleware(req: NextRequest) { const response = await userReq.json(); const user = response.data; if (user) { + let teamName: string | undefined; if (!user.teamId) { // create a team - await fetch(`${process.env.NEXT_PUBLIC_HOST}/api/team`, { + const team = await fetch(`${process.env.NEXT_PUBLIC_HOST}/api/team`, { method: "POST", headers: { "Content-Type": "application/json", @@ -49,6 +51,12 @@ export default async function AppMiddleware(req: NextRequest) { role: "owner", status: "active", }), + }).then((res) => res.json()); + teamName = team?.data?.name; + } + if (teamName) { + await captureEvent(user.id, "team_created", { + team_name: teamName, }); } } diff --git a/lib/services/posthog.ts b/lib/services/posthog.ts new file mode 100644 index 00000000..9f4f6c39 --- /dev/null +++ b/lib/services/posthog.ts @@ -0,0 +1,37 @@ +import { PostHog } from "posthog-node"; + +let posthogClient: PostHog | null = null; + +export function getPostHogClient(): PostHog { + if (!posthogClient) { + posthogClient = new PostHog(process.env.POSTHOG_API_KEY!, { + host: process.env.POSTHOG_HOST, + flushAt: 1, + flushInterval: 0, + }); + } + return posthogClient; +} + +export async function captureEvent( + distinctId: string, + eventName: string, + properties: Record = {} +): Promise { + try { + if (process.env.TELEMETRY_ENABLED !== "false") { + const client = getPostHogClient(); + await client.capture({ + distinctId, + event: eventName, + properties: { + ...properties, + is_self_hosted: true, + }, + }); + await client.shutdown(); + } + } catch (error) { + console.error("Error capturing telemetry events:", error); + } +} diff --git a/lib/services/trace_service.ts b/lib/services/trace_service.ts index 1caf851f..4e1014ea 100644 --- a/lib/services/trace_service.ts +++ b/lib/services/trace_service.ts @@ -18,6 +18,7 @@ export interface ITraceService { project_id: string, lastNHours?: number, userId?: string, + experimentId?: string, model?: string, inference?: boolean ) => Promise; @@ -96,7 +97,10 @@ export interface ITraceService { GetUsersInProject: (project_id: string) => Promise; GetPromptsInProject: (project_id: string) => Promise; GetModelsInProject: (project_id: string) => Promise; - GetInferenceCostPerProject: (project_id: string) => Promise; + GetInferenceCostPerProject: ( + project_id: string, + attribute_filters?: { [key: string]: string } + ) => Promise; } export class TraceService implements ITraceService { @@ -107,7 +111,10 @@ export class TraceService implements ITraceService { this.queryBuilderService = new QueryBuilderService(); } - async GetInferenceCostPerProject(project_id: string): Promise { + async GetInferenceCostPerProject( + project_id: string, + attribute_filters: { [key: string]: string } = {} + ): Promise { try { const tableExists = await this.client.checkTableExists(project_id); if (!tableExists) { @@ -120,6 +127,17 @@ export class TraceService implements ITraceService { ), ]; + if (Object.keys(attribute_filters).length > 0) { + Object.keys(attribute_filters).forEach((key) => { + conditions.push( + sql.eq( + `JSONExtractString(attributes, '${key}')`, + attribute_filters[key] + ) + ); + }); + } + const query = sql .select([ `IF( @@ -407,6 +425,7 @@ export class TraceService implements ITraceService { project_id: string, lastNHours = 168, userId?: string, + experimentId?: string, model?: string, inference = false ): Promise { @@ -425,6 +444,12 @@ export class TraceService implements ITraceService { ); } + if (experimentId) { + conditions.push( + sql.eq("JSONExtractString(attributes, 'experiment')", experimentId) + ); + } + if (model) { conditions.push( sql.eq("JSONExtractString(attributes, 'llm.model')", model) diff --git a/lib/utils.ts b/lib/utils.ts index a1ee9de5..ab299edb 100644 --- a/lib/utils.ts +++ b/lib/utils.ts @@ -518,10 +518,46 @@ export function calculatePriceFromUsage( if (!model) return { total: 0, input: 0, output: 0 }; let costTable: CostTableEntry | undefined = undefined; - if (vendor === "openai") { + if (vendor === "litellm") { + let correctModel = model; + if (model.includes("gpt") || model.includes("o1")) { + if (model.includes("gpt-4o-mini")) { + correctModel = "gpt-4o-mini"; + } else if (model.includes("gpt-4o")) { + correctModel = "gpt-4o"; + } else if (model.includes("gpt-4")) { + correctModel = "gpt-4"; + } else if (model.includes("o1-preview")) { + correctModel = "o1-preview"; + } else if (model.includes("o1-mini")) { + correctModel = "o1-mini"; + } + costTable = OPENAI_PRICING[correctModel]; + } else if (model.includes("claude")) { + let cmodel = ""; + if (model.includes("opus")) { + cmodel = "claude-3-opus"; + } else if (model.includes("sonnet")) { + cmodel = "claude-3-sonnet"; + } else if (model.includes("haiku")) { + cmodel = "claude-3-haiku"; + } else if (model.includes("claude-2.1")) { + cmodel = "claude-2.1"; + } else if (model.includes("claude-2.0")) { + cmodel = "claude-2.0"; + } else if (model.includes("instant")) { + cmodel = "claude-instant"; + } else { + return 0; + } + costTable = ANTHROPIC_PRICING[cmodel]; + } else if (model.includes("command")) { + costTable = COHERE_PRICING[model]; + } + } else if (vendor === "openai") { // check if model is present as key in OPENAI_PRICING let correctModel = model; - if (!OPENAI_PRICING.hasOwnProperty(model)) { + if (model.includes("gpt") || model.includes("o1")) { if (model.includes("gpt-4o-mini")) { correctModel = "gpt-4o-mini"; } else if (model.includes("gpt-4o")) { @@ -757,6 +793,8 @@ export function getVendorFromSpan(span: Span): string { serviceName.includes("embedchain") ) { vendor = "embedchain"; + } else if (span.name.includes("litellm") || serviceName.includes("litellm")) { + vendor = "litellm"; } return vendor; } diff --git a/package-lock.json b/package-lock.json index 20b8e6a6..fb73a2a7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -76,6 +76,8 @@ "npm": "^10.8.2", "openai": "^4.40.0", "pdf-parse": "^1.1.1", + "posthog-js": "^1.161.3", + "posthog-node": "^4.2.0", "pretty-print-json": "^3.0.0", "prism": "^1.0.0", "prismjs": "^1.29.0", @@ -5778,8 +5780,9 @@ } }, "node_modules/axios": { - "version": "1.7.2", - "license": "MIT", + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -8130,6 +8133,11 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fflate": { + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz", + "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==" + }, "node_modules/file-entry-cache": { "version": "6.0.1", "dev": true, @@ -14922,6 +14930,28 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/posthog-js": { + "version": "1.161.3", + "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.161.3.tgz", + "integrity": "sha512-TQ77jtLemkUJUyJAPrwGay6tLqcAmXEM1IJgXOx5Tr4UohiTx8JTznzrCuh/SdwPIrbcSM1r2YPwb72XwTC3wg==", + "dependencies": { + "fflate": "^0.4.8", + "preact": "^10.19.3", + "web-vitals": "^4.0.1" + } + }, + "node_modules/posthog-node": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-4.2.0.tgz", + "integrity": "sha512-hgyCYMyzMvuF3qWMw6JvS8gT55v7Mtp5wKWcnDrw+nu39D0Tk9BXD7I0LOBp0lGlHEPaXCEVYUtviNKrhMALGA==", + "dependencies": { + "axios": "^1.7.4", + "rusha": "^0.8.14" + }, + "engines": { + "node": ">=15.0.0" + } + }, "node_modules/preact": { "version": "10.23.1", "license": "MIT", @@ -15911,6 +15941,11 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/rusha": { + "version": "0.8.14", + "resolved": "https://registry.npmjs.org/rusha/-/rusha-0.8.14.tgz", + "integrity": "sha512-cLgakCUf6PedEu15t8kbsjnwIFFR2D4RfL+W3iWFJ4iac7z4B0ZI8fxy4R3J956kAI68HclCFGL8MPoUVC3qVA==" + }, "node_modules/rw": { "version": "1.3.3", "license": "BSD-3-Clause" @@ -17459,6 +17494,11 @@ "node": ">= 8" } }, + "node_modules/web-vitals": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-4.2.3.tgz", + "integrity": "sha512-/CFAm1mNxSmOj6i0Co+iGFJ58OS4NRGVP+AWS/l509uIK5a1bSoIVaHz/ZumpHTfHSZBpgrJ+wjfpAOrTHok5Q==" + }, "node_modules/webidl-conversions": { "version": "3.0.1", "license": "BSD-2-Clause" diff --git a/package.json b/package.json index 2ea3aac9..e433b162 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,8 @@ "npm": "^10.8.2", "openai": "^4.40.0", "pdf-parse": "^1.1.1", + "posthog-js": "^1.161.3", + "posthog-node": "^4.2.0", "pretty-print-json": "^3.0.0", "prism": "^1.0.0", "prismjs": "^1.29.0", diff --git a/public/litellm.png b/public/litellm.png new file mode 100644 index 0000000000000000000000000000000000000000..79468969fa41889d1a0d5058d6390613b3a1013c GIT binary patch literal 9748 zcmbWcWmFu^7A`!v1sEKHySoz@+@0X=5?~-Oz#zfh6C}7xa0rAzu)#G*@Zjzq+%9>~ z`OZ1(yZ>&jwR=Cky>|8L?phV4p(c-s{u&(s0AMOA$Y{Ort{0L+MSi*Fz?7c>fLG!U z($X4=($Z8KZZ5VCPBs95LR5+#ivHU{!W^Sdi4yj3h()2xh;YnsMWN&H3Y@`liHNB5 zeGRd?ll_eJlZs*hIBN-nR9H&)PXg}3f-bm*{EL;NCDmQbql`A~ zgWSjGX+G$KF$X{azo)F#7>v~u0f1FB(Y&Vs$S}JIh^f={*21xKs{q$~wX!(+b+ocr z`tPkl@449`jQ|>2Ypy+efG>^GIXPph5wFW1fp5^g;i&=dAdw&3+)B==)Pm+--$hcT z?(=E|j4M88+V0mfL?nS?x7iB;L!iSOVLr zVkTJWIaECoZCCdeJp+gF{OP>midDJF*@%GLG;efAYbeXj49Rewq$z;FM2R| zh4VGf{Z0v}$Z913c?f7uW#%jch{28R;UFme2^u^Z74lvn_94N@c%+O$0+7N=lKKXZ z1~foUvBP<=72-D#KyJGw@0-YQ@bh!r0K7W&D4Ycqsj0GcVQBXRE(~t~^$8~%fs|-F z6Em~ndF4X-&?_7*-4%(JBw9g_^qdYE7a@|mhpGUxI^llR6e=Wo*SQ_b>za`7?RU*6 zN)G{2;DLc6XkI0#(LLfe!dMWgM=Du%t$<1-FdDzf`+!B#a%mf5lv$LkCXCqbo(~q* zdIMkay_GAZo)zz1cyHFgOb-L`*2$v5hw_b2-bLkAzGRh z4)pHij^w`N9Rv60H%Py{_{>!aegZLm@t*Ih+{ms@&P!EyGySM9tj7TSI*EC>yks!E zT9SYmcDQ3a0buu=tE;OMC&s&qfUB17{^tR&(F{0o@*#k+P*+Oi#TFTd7;x1{3KQhu z14B)Na}=VqTSaaSq(nu`r^4+CqT?d5&4rt7CRRbIhxuu|QtCvnCDaa7HD{eey9z|e zgCj#I?G$z;X$T=*r7#v-T1DDLIf2EI#VgdpXmP%jx8%?Nv0NW_>WE_aepQd4qMP}65<^qk}& z4OXH8o!BSx!AyL`^~7_9-lEH*rJ|&wiX!kY6nYzaD|)+Ps$y&Ue65dK>AyIN@s#vZ za+p6bR3xzv9&{*H(U~6<2AS->02fsryB@#8cxa+AvqdU`Z>xcvLgD z(6r27<}vjy5LvZbkyeXV-m0vRBQEB(SO%S{@=poXoNcL}RT*}CE%wAeWBUr;J7Cx^ z*xNZQ8OpG;vN!6M=$6e;R0&l1JF7Yq9U&gUA1NJe{LU6hi!a79Vl=XA9)-)!6v>q% zjyI2AP0;5l0QXGeoOsCZ-|d}yC_}FIm&YHAhp%8>ahB_qwWV zrhR6zvZAVH0q9IDU?4#1^xIj}NN?fQg3vx`&M7YJKOy0 z_KtWzEsxv1pD*5f%W|j7Ykt>lA!S=+)Z?RL{ehmKmtdVywoy$>vv0I-$~}8HA#HSB z^83z^)wL7u*^RV*d3C`t(J_Ra0PlHU8}Fm@eG*6hRB_LRIS!Y%F7eldCu)>G)T762 zbJfi7%;H_Xomh$Uis!6&cG#(q+A_EcUKDo}cFaB3KKrIJItmJ>v2Kbo7z6g8~pW}zxo|XZn6!Zs%5{;as1wNZ}A7gu9XOsWNEx{!u)y26IS`;uQ zTjs+eClUc)Hr?Y8;&erZD*y-MjmdARhFIPiPEFWbLbM?^eBQFw7Ki5|b-9R!kfMzC5ZKJH&H`$R-)Ks$;9LT{3cQ@X_GSSLAxU z`s;0`t)fjboOkH;q3mztN(fbsfb&t&ve}cOrB&H8oLG1+W3q0HTLRk-os_q_QQtO+ z1`GXiABG;VC-F)s*~XcGgbky$*>%=^^<8;gE4J&r&Agd*=_4892NO=)hwSc&!e#Cm zI~hMsgueTlp0;LRFkjSfKbvmqH}lH;QQ`bT*9T zBxC~{@z@UD(XS{$=|3xxD-SDmux)X){i&F&+3LXJP&S%rTy(zeBj_RMA`sNOt6pqY zveH>Aub$q}ovq2QVK%t%JWBpXG;BO1P*qr&RC&OL&FNz7((d1T$4Nj!AjtB_daX;M z&0gly5%hg+U#=t0BlDGD3iz(Up!W7Eb(Q^cI(6E{D9Y&KJ7_7j_UOSScI92;feYu- zlShQB{LW}vHaq_@KYzvZHpDt><-d;@>WK0$TGQM7Tt;!N#P}eYUvhUsn4*x{&}+@eLIJ z!LKG};c{K^IEix!BUdXoRgBkPcXDIVy;AVY_s84Noh42gPS7>MbJ>~QwDQ5{h^O=W zif;ja0oSZ&6`ixnE4yv;XER&Q7ZOvjZZF{zPzUF2-nGS!d|&@XF0Q!Pz33yxp7O0; zW{v7fSNjs>2niYf`yR6S8UQkBMk+GwfE=Ka1jy@rB(VwhduHU7X}3$EKFzK5OJL#^ z1njRq`$8xmTAGny?ar?ENSD(L@IYB4s!@bItG-v?^V>3<%$WIU7EzqyA2KqM(4XQ4 zud~EAA1_E0bWC7lI{|PjEPf%hvk~zxJBnx-!yFFJta zTWLkb7yj1D&Bg}oZtnt#xGP?H5umv$7`Oufg!F$2PEm{Q^dg|5E%n z;r}@5|JRXQnD2i*|3~xx_tbH>ag%m&e#r;{{dcwg?fk#We>;kD{+;;$G~!=f{zv-K zW)Qk4=YN(Agid(5%=xm4WDYXw+AsX&8T)6@zFe7J=r4aEwDPRo9@`i0sVF0<4TU>2 z!En`{CV4t;_e#Q445ZVihU=q|t0PE~NR$sW$9I6CAjxRLPZ3Je%%IPyP5)9#;YoHV z4vG9FB@q)8{5DNMH%$|nDNh3z6ITj1auB>+^P%*)In#GOv(3O$A<>z2xINdC;;S#1IK}=h8Pc^hK0c#U^uH%t7NOFH#n;l zLA*h<5~!}!uCHAYn}9+nLh$~GP4cTMtEfPDW7I5UvKPluxcERyxMLh4JR$f{01Ln_ZZxZ0^()B(}w+po>tA-et#EzY&sOa3GJ8K*jEy z)#X{wy0EeooMG6q&Yxp+hGEb5I*dI_NwSkV>vlc4;GjEUMAC9yRCkvJFX-^>y6Sn}^bn!!=@`Fd!mLGZhr><1QE~#sL*|c5VK> zTJm*F+4;0n+Ac1y7ZyqxPPVZ8fL&|9MkbrPxT9zBlQW>OHta2$JSqrxKY`L$<9*AB z!(`ni<1l_Bsez(;3!W1^swhhZ`S`(QP)Y5OxLGs)!t#9O?)^13wuM%o_DhA)OS*9j z9pVb#4sOs$23D?J{#IC5HYgPEG@Qp#c-)Rc8Q?}UKoP%If{~x!{`#t1i%)RDq0vr; zD6cTVE-qH4D-XI0k^2N7-$X-v=NN%(_{dJ(f8~tPsSF=SX|B;a8C?3={E>25YYuE( z%!wZKVGNJ4L+aP~nuWCc;aq824!`8LfE8T$UIDi8_dgbErv(dE>kDtni(+j95_bF2 zSs9-J^@<52+}+&tHgt=0@HG+MNQ6{qbfKN)5(<4e_+2}t&oK`V*TqILJ6W>g-HFtS zU*j+&!)d+%hiGd@S>4RKhC2*NAtrjrge5$jRh|%KRDr!*Kf13`D81X|5r!?+NgSxH zy6+BZ(PRRMKF-gKhq7LY7+03ddaJS;l2hEI5slmD)8j;PvxkXud`IcKVi3*34JneW zbDii=**k+OpuW~Glvh%3^}&KJgHp!rWp~Dz@}8$N5tFBW(`QaqSvjeoqOnk{{ggoX#a>{hJg%k{A8dssl!LoCRVI$IV?p9wFLnUVWASH zlH(QY8lK-BZna77 zMJ?=&uQmB=qQVKQ+~x0x-gyZ`yNK`RSK84Emsm8l~9nTGk*#G5lzNz2@@fdhoIu zeX=CBb8qsAWW}kGN}y4k!;Ir|%e4sgE*}VAPkY^76RG?)IsRlO&|<1ER~DsBVlrITw*}MiQ^fpgNV;fY`kg+LAlJmaJs(U$u|Czfq!V^1f(yUVN{H-UAPTJma)s2B zd}kEvio=z1jw2t={n?oxf|6{7rfGCKgOhgu)bBS7yYO+V($dO! z=xPLowFMO5U7UO(pz{n(^Z=OO$ATNR*6>dxU6;JM@+@H6Jz;7`zg9S&sLpc{W@#_`w+ZJLmQpXAUH>)~j_W?hiK|$Dtp!8&xMlhsj3+ zGE7KArLDwaNQ=A?sZvvk#zan1fToh7ssQ-5?gL&6WvlTC%xH~D4DwY?^7^W%kkRWf z@HH}hX0d^Ypi_s9H(&7Y=ZC@;efNrE)4fhU|Yw0_iE&Z6Wcvk+lH;0qm z13BCVU@N2w{%Bow{}_}~yBIWUSy>YOxrX=M2lygCw%SKX$;3<}J45hnNLl~TQ*(`Z zuH3w08%<|T2nthXR3nU8F%a;?H|`00bJ9fM;qk2?>?jVOARr_*s!-EZm&L22Oln+% zVihu-|Dc5vJ>+EXnhC@s2r{}S<$_Ak9igN}wffM29u)wV>A0qtnP+?bsK+ah$&0`@ z7_Kev_S!kpR0w|?*s{uPn%S33?Xo4ykZyHW5VW&XD~u89#UZow4uTpjr@s{P;$3bk zUTeWWg+ZEyo)9)|w6yw$1j-V>Zd{f#|E*Hd z43z*wYF9Q>^d9rb1W9Cr9E~XJ!CrrBU}MOiC+<2K1L@n);l1AZb)UI+WCbXvTKcP= z(}iZqQK9&fYe3~JF?%PsPcn$Hsu+*gem?a;wH(p#$`QzI;n%LLW_8QS+(u@npcEM0 z$NBk9O~YkC487Fx%;!+9>$B3+epMsx0MkYq`3zx4fkvmFN&R#L6zM5Zv=QQ|<@y~K zygA0T#D$+C^1hybbvlWczY%}=344AHaXk6*axh`|_YdsI2n~uaOZc>tqr!m;q(M}W zz|zAfK=jfKlEuN7LDB_Ea^MK#BmD`1R=N$)&~OaYu_11vqSJh&jl}K6+ZqpZC5~Nm zW8BQJKK@X{u;i&8PMa~J)_(L!QLoKs>2d$^zGB%|XMc3d$M1{zL}x_s174Hs6EJJk za?Xh#NJ@%QF^!xwh@Le6s<2bWgZGb*7(|4;)s*v5^c1>iBaZC{!lq#paeCY)Bdh_% zty>D7CBrsYf@V`(lvPYKi4&Yaz?K!%A80VE%Bd3*Ej>P`_@@dH<#76C_x-)#fn)FPB+Sz-7@9b zJfrIfZ9W^T?O0~$hy7tCT+l)z20OUbpyXRi8`9T$>;dQM2cMGU_8`T-@h>*yXgtpR z2oCnYQ5H%^wCFTcIAn=?5zY#>==R%zlp!O$>2Gf zo9MRx;D~qk89`SuUY*cqyPwv)2LtqjXxO&lLy5Dn5 zc?EM4it|C^j*O0$w$^^dMo2nh%SuwNNZhe)-=Z-GW6i8h9{xr??KKEu0q^j_v!3vQiO)#Y;A_^~V_tW6DdZ5x!TQ z!C|bSyKfna&1Z}zi!AcIFGx?f1bt;P8Apx4N_}N-8MmG=GaK`dj=Um0>lr)}eb-tS ziEyd#2?;Gm3x3{M_9O9CQYHv8G{`N5>Kjtqmk`Nul7LzS{kyMy=6Agnld81~8$SnH z)bH21pU76~-?N@$EOC)Usqu=74#Jl0O0>Vd6|=Q`zRNkT`L z^Y=%RT&MX^`FqIc{<(&|%Nrlx)J5>GFN7SXBS{N7Scrl@4==QMPv@_ABO}|9qtJ9k z1(|U8t>UbEJO$i&z<1n$8$H__Qz^b@05Wn&XTa7pZ10bS#c$2SH~NUYL4o%|;j6g| z&P1$Ip}nv0nvGv!?fPx$YTI1Gwl~%#I8uu9 ziMs4PWIoejq=FngacO<7PL+{g-K6T(lSUQ`PWUxD;8&xX6I3equuUka7A-F+*}iDD zB{6q*t2Uzch(1xrKgc8&*&xJK5Fj=c&09FN7Bsw+aySg%R#HXF*!vC(> z-F6ICQPY}-n~5O$6d6#Y%}BNA6@blV(x^*u`~!QXA5Z#j=;((BUY(Vmz}zJ>3kyDk zm)=~4%A`JZ~{z~{4li}zt+>B23L#beH zN(Q4!GX>6G6rjsU#FCXP<8CrnP_vYM19K1~ym{?nJ~ z&h0v$v<0Py=X_oXxQ?5`pB3@I9Y5dJ<*5>ZTU+K>s`az^!Pb(*FhTm99J#oN!E@C6 zxzzI95pS#USN5k!%zfFyq8<+Z3JO&9*qqQ@e~^*^H3muD=U*8P6jJ)M~sx_oPk}IfOGVk=Ub|Gf#Dg*m_-iLcN zw@lWkS-Gw?rW6OAb{fC$ycnECD#h9a#mx|(o<9AFHg^R0;E%SBJmzgUn`;_40g(|i zfQ+`mqQR%Z0cP+j%m6*-u@1hjwnkd^N3`60?x#;$1tGA0^EI820G}bcUdt(7-jvF% z{bvlSMUVcULbU-kY#61|UXN)ZO-;~r2Av!VF+>N}R0Y-d}VBvuFS7Rc|p115M5Z^y73B39-4Jw1SWAup$q@BgT)d;2eo@)ASkBIbBk zSJSNk0aPP!+{%ZAaA!IXYtc=)Ar{C90XD0kTxd@2>{$0+!@!rl)dh5*)CpU)?KeM=VLj~6xa5b3+2IA#L( zMC2N-U+prrnEc3B)8Ns|J0o$G^t)83@{1&A-a8zd?b@p?#u!$EFqoFcgEkib_*3;0 zF%&P!S&Aj#6{@$*o8c~nVT!U^O-bg1;Wm5u?awWBJN@Ke^mr3&9ns&BC-c)_*qPv+9#a8^XxAWdKpB5|6-uk*`?ZkF+&+4*H92lqC_1tw^&;7oOn0{Di z+FQWpIU_{g2Cb2X>Aprg_sz4(9|nYN+*hc2Kg{M=PUfG134EHieAZXdTbSOj6#{2EisVT5j$P!rhq5!_?sOKofJ4_3CtY zE@(?96mv!T7+*ZHxh=VBouqn;-Fc#_-_?Mvxge3#T0&z#dO5)C^P9R2Uk?q|K_XPd zlKQNd&8~#0X?2h1`~kU_Cu~UPptJuKys`&;m)Y-c@^^P5FzQIy=LA)zw*@9YJo~>A z91NYE7Y#^gqz=IapGS4n=Ufcqw4VQdBjolq@v9i&89s=D`iS1ueB>Un>xZnUMPTo) zgmDj28pM-N1sq&dbl}Q#&v^LY-y-p5K@g!RQs9q>$8q#v&6s2;`~Fv4U38$QU5pp% zER(D3TqVen4y~J#%ciS-)fYYn(4}IC1CO4YLh`(b`aV=@?}TL@>^zNZ)!z|0dc9mj zSFwNeJ5}7V$g)Yw>KMK^nyc}_ppbR3djEJ<++5b)r>Z>HD_qdp>Fq>OAI659#aY41 zs6D&7G72LFJjQ~I6#Hd4Q{d_CiI^i@vs{o%x%q;rQ~#guT6NwBrbCYl_VQ`tEMMi1 zzp{=!JvkV;eYlyENj~GN{4oiMWI^6wRFk)~#PuH>>?``*H<91Y8%fuZlkjen2!A#G zJ@f98S7anP3zMnBJT|#VB*Ohym>x$K(FsJIXsSBfRZ$okE6jI{a9^Kz;V!Ulf2qts z#LirX^O_&*HhA9|;)gOpaA0|DQV-R~rvN13?1J_64H87Q<*>=Ro|1EE|gQEO<6wvBg R=4HDDP?S}Zsg^Pi{y!_j0}ucJ literal 0 HcmV?d00001