Scale3-Labs
diff --git a/‎.github/resources/slack-payloads/slack-message-template.json‎
Lines changed: 22 additions & 0 deletions b/‎.github/resources/slack-payloads/slack-message-template.json‎
Lines changed: 22 additions & 0 deletions
diff --git a/‎.github/workflows/release_langtrace.yaml‎
Lines changed: 42 additions & 0 deletions b/‎.github/workflows/release_langtrace.yaml‎
Lines changed: 42 additions & 0 deletions
diff --git a/‎Dockerfile‎
Lines changed: 3 additions & 3 deletions b/‎Dockerfile‎
Lines changed: 3 additions & 3 deletions
diff --git a/‎README.md‎
Lines changed: 4 additions & 2 deletions b/‎README.md‎
Lines changed: 4 additions & 2 deletions
diff --git a/‎app/(protected)/project/[project_id]/annotations/page-client.tsx‎
Lines changed: 2 additions & 2 deletions b/‎app/(protected)/project/[project_id]/annotations/page-client.tsx‎
Lines changed: 2 additions & 2 deletions
diff --git a/‎app/(protected)/project/[project_id]/prompts/[prompt_id]/page.tsx‎
Lines changed: 28 additions & 3 deletions b/‎app/(protected)/project/[project_id]/prompts/[prompt_id]/page.tsx‎
Lines changed: 28 additions & 3 deletions
diff --git a/‎app/api/metrics/score/route.ts‎
Lines changed: 34 additions & 9 deletions b/‎app/api/metrics/score/route.ts‎
Lines changed: 34 additions & 9 deletions
diff --git a/‎app/api/traces/route.ts‎
Lines changed: 36 additions & 0 deletions b/‎app/api/traces/route.ts‎
Lines changed: 36 additions & 0 deletions
@@ -0,0 +1,22 @@
+{
+	"attachments": [
+		{
+			"pretext": "{{ env.PRE_TEXT_MESSAGE }}",
+			"fallback": "{{ env.FALLBACK_MESSAGE }}",
+			"color": "{{ env.COLOR }}",
+			"author_name": "{{ github.workflow }}",
+			"author_link": "{{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}/actions/runs/{{ env.GITHUB_RUN_ID }}",
+			"title": "{{ env.GITHUB_REPOSITORY }}",
+			"title_link": "{{ env.GITHUB_SERVER_URL }}/{{ env.GITHUB_REPOSITORY }}",
+			"fields": [
+				{
+					"title": "Release Tag",
+					"short": true,
+					"value": "{{ env.RELEASETAG }}"
+				}
+			],
+			"footer": "deployed by: {{ github.actor }}",
+			"footer_icon": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png"
+		}
+	]
+}
@@ -64,3 +64,45 @@ jobs:
           tag: ${{ needs.generate-version.outputs.new_version }}
           generateReleaseNotes: true
           draft: false
+
+  post-release:
+    name: Post Release Actions
+
+    runs-on: ubuntu-latest
+    if: ${{ always() }}
+    needs:
+      - generate-version
+      - build-docker
+      - publish
+
+    steps:
+      - name: Checkout main branch
+        uses: actions/[email protected]
+        with:
+          ref: main
+
+      - name: Slack - Success Message
+        uses: DSdatsme/slack-github-action@env_support
+        if: ${{ success() && needs.build-docker.result == 'success' && needs.publish.result == 'success' }}
+        with:
+          channel-id: ${{ vars.SLACK_CHANNEL_ID }}
+          payload-file-path: ./.github/resources/slack-payloads/slack-message-template.json
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RELEASETAG: ${{ needs.generate-version.outputs.new_version }}
+          PRE_TEXT_MESSAGE: "Workflow Passed! :successkid:"
+          FALLBACK_MESSAGE: "Workflow Passed!"
+          COLOR: "good"
+
+      - name: Slack - Failure Message
+        uses: DSdatsme/slack-github-action@env_support
+        if: ${{ failure() || needs.build-docker.result != 'success' || needs.publish.result != 'success' }}
+        with:
+          channel-id: ${{ vars.SLACK_CHANNEL_ID }}
+          payload-file-path: ./.github/resources/slack-payloads/slack-message-template.json
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RELEASETAG: ${{ needs.generate-version.outputs.new_version }}
+          PRE_TEXT_MESSAGE: "<!channel> Workflow Failed! :x:"
+          FALLBACK_MESSAGE: "Workflow Failed!"
+          COLOR: "danger"
@@ -1,5 +1,5 @@
 # Debian based node 21.6 image
-FROM node:21.6-bookworm as development
+FROM node:21.6-bookworm AS development
 
 LABEL maintainer="Langtrace AI <[email protected]>"
 LABEL version="1.0"
@@ -20,14 +20,14 @@ CMD [ "/bin/sh", "-c", "npm run dev" ]
 
 
 # Intermediate image for building the application
-FROM development as builder
+FROM development AS builder
 
 WORKDIR /app
 
 RUN NEXT_PUBLIC_ENABLE_ADMIN_LOGIN=true npm run build
 
 # Final release image
-FROM node:21.6-bookworm as production
+FROM node:21.6-bookworm AS production
 
 WORKDIR /app
 
 
@@ -59,7 +59,7 @@ Langtrace.init({ api_key: <your_api_key> })
 OR
 
 ```typescript
-import * as Langtrace from "@langtrase/typescript-sdk"; // Must precede any llm module imports
+import * as Langtrace from '@langtrase/typescript-sdk'; // Must precede any llm module imports
 LangTrace.init(); // LANGTRACE_API_KEY as an ENVIRONMENT variable
 ```
 
@@ -131,14 +131,16 @@ Langtrace automatically captures traces from the following vendors:
 | Anthropic    | LLM             | :white_check_mark: | :white_check_mark:              |
 | Azure OpenAI | LLM             | :white_check_mark: | :white_check_mark:              |
 | Cohere       | LLM             | :white_check_mark: | :white_check_mark:              |
-| Groq         | LLM             | :x:                | :white_check_mark:              |
+| Groq         | LLM             | :white_check_mark: | :white_check_mark:              |
 | Perplexity   | LLM             | :white_check_mark: | :white_check_mark:              |
+| Gemini       | LLM             | :x:                | :white_check_mark:              |
 | Langchain    | Framework       | :x:                | :white_check_mark:              |
 | LlamaIndex   | Framework       | :white_check_mark: | :white_check_mark:              |
 | Langgraph    | Framework       | :x:                | :white_check_mark:              |
 | DSPy         | Framework       | :x:                | :white_check_mark:              |
 | CrewAI       | Framework       | :x:                | :white_check_mark:              |
 | Ollama       | Framework       | :x:                | :white_check_mark:              |
+| VertexAI     | Framework       | :x:                | :white_check_mark:              |
 | Pinecone     | Vector Database | :white_check_mark: | :white_check_mark:              |
 | ChromaDB     | Vector Database | :white_check_mark: | :white_check_mark:              |
 | QDrant       | Vector Database | :white_check_mark: | :white_check_mark:              |
 
@@ -1,11 +1,11 @@
 "use client";
 
+import { ChartTabs } from "@/components/annotations/chart-tabs";
 import { CreateTest } from "@/components/annotations/create-test";
 import { EditTest } from "@/components/annotations/edit-test";
 import EvaluationTable, {
   EvaluationTableSkeleton,
 } from "@/components/annotations/evaluation-table";
-import { EvalChart } from "@/components/charts/eval-chart";
 import LargeChartSkeleton from "@/components/charts/large-chart-skeleton";
 import { AddtoDataset } from "@/components/shared/add-to-dataset";
 import { Button } from "@/components/ui/button";
@@ -85,7 +85,7 @@ export default function PageClient({ email }: { email: string }) {
         <PageSkeleton />
       ) : tests?.tests?.length > 0 ? (
         <div className="flex flex-col gap-12 top-[16rem] w-full md:px-24 px-12 mb-24">
-          <EvalChart projectId={projectId} tests={tests.tests} />
+          <ChartTabs projectId={projectId} tests={tests.tests} />
           <div className="flex flex-col gap-2">
             <AddtoDataset
               projectId={projectId}
 
@@ -21,6 +21,7 @@ export default function Page() {
   const router = useRouter();
   const [prompts, setPrompts] = useState<Prompt[]>([]);
   const [selectedPrompt, setSelectedPrompt] = useState<Prompt>();
+  const [prettyJson, setPrettyJson] = useState<boolean>(false);
   const [createDialogOpen, setCreateDialogOpen] = useState<boolean>(false);
   const [live, setLive] = useState<boolean>(false);
   const queryClient = useQueryClient();
@@ -50,6 +51,15 @@ export default function Page() {
     },
   });
 
+  function isJsonString(str: string) {
+    try {
+      JSON.parse(str);
+      return true;
+    } catch (e) {
+      return false;
+    }
+  }
+
   if (promptsLoading) return <PageLoading />;
 
   if (!selectedPrompt)
@@ -217,10 +227,25 @@ export default function Page() {
             </div>
             <div className="flex flex-col gap-2">
               <Label>Prompt</Label>
-              <p className="p-2 rounded-md border-2 border-muted">
-                {selectedPrompt.value}
-              </p>
+              <div className="flex items-center gap-2">
+                <Switch
+                  checked={prettyJson}
+                  onCheckedChange={setPrettyJson}
+                  disabled={!isJsonString(selectedPrompt.value)}
+                />
+                <Label>JSON</Label>
+              </div>
+              <div className="p-2 rounded-md border-2 border-muted">
+                {prettyJson && isJsonString(selectedPrompt.value) ? (
+                  <pre>
+                    {JSON.stringify(JSON.parse(selectedPrompt.value), null, 2)}
+                  </pre>
+                ) : (
+                  selectedPrompt.value
+                )}
+              </div>
             </div>
+
             <div className="flex flex-col gap-2">
               <Label>Variables</Label>
               <div className="flex flex-wrap gap-2 p-4 border-2 border-muted rounded-md">
 
@@ -88,31 +88,56 @@ export async function POST(req: NextRequest) {
         }
 
         if (!dateScoreMap[date][`${testId}-${evaluation.Test?.name}`]) {
-          dateScoreMap[date][`${testId}-${evaluation.Test?.name}`] = 0;
+          dateScoreMap[date][`${testId}-${evaluation.Test?.name}`] = [0, 0];
         }
 
-        dateScoreMap[date][`${testId}-${evaluation.Test?.name}`] +=
-          evaluation.ltUserScore || 0;
+        const total =
+          dateScoreMap[date][`${testId}-${evaluation.Test?.name}`][0] +
+            evaluation.ltUserScore || 0;
+        dateScoreMap[date][`${testId}-${evaluation.Test?.name}`][0] = total;
+        dateScoreMap[date][`${testId}-${evaluation.Test?.name}`][1] += 1;
       });
     }
 
-    const chartData = Object.entries(dateScoreMap).map(
+    const metricsChartData = Object.entries(dateScoreMap).map(
       ([date, scoresByTestId]) => {
         const entry: any = { date };
-        Object.entries(scoresByTestId as any).forEach(([testId, score]) => {
-          entry[testId] = score;
-        });
+        Object.entries(scoresByTestId as any).forEach(
+          ([testId, scores]: any) => {
+            entry[testId] = scores[0];
+          }
+        );
         return entry;
       }
     );
 
-    chartData.sort(
+    // calculate score % test wise and not date wise
+    // const scoresChartData: any = {
+    //   "test-1": 10,
+    // };
+    const scoresChartData: any = {};
+    Object.entries(dateScoreMap).map(([date, scoresByTestId]) => {
+      Object.entries(scoresByTestId as any).forEach(([testId, scores]: any) => {
+        if (!scoresChartData[testId]) {
+          scoresChartData[testId] = 0;
+        }
+        if (scores[1] === 0) {
+          scoresChartData[testId] = 0;
+        }
+        scoresChartData[testId] = scores[0] / scores[1];
+      });
+    });
+
+    metricsChartData.sort(
       (a, b) =>
         new Date(a.date as string).getTime() -
         new Date(b.date as string).getTime()
     );
 
-    return NextResponse.json(chartData);
+    return NextResponse.json({
+      metrics: metricsChartData,
+      scores: scoresChartData,
+    });
   } catch (error) {
     return NextResponse.json(
       {
 
@@ -11,6 +11,8 @@ export async function POST(req: NextRequest) {
     const session = await getServerSession(authOptions);
     const apiKey = req.headers.get("x-api-key");
     const { page, pageSize, projectId, filters } = await req.json();
+
+    // check if user is logged in or has an api key
     if (!session || !session.user) {
       if (apiKey) {
         const project = await prisma.project.findFirst({
@@ -37,6 +39,40 @@ export async function POST(req: NextRequest) {
       }
     }
 
+    // check if user has access to the project
+    const email = session?.user?.email as string;
+    const user = await prisma.user.findUnique({
+      where: {
+        email,
+      },
+    });
+
+    if (!user) {
+      return NextResponse.json(
+        {
+          message: "user not found",
+        },
+        { status: 404 }
+      );
+    }
+
+    const project = await prisma.project.findFirst({
+      where: {
+        id: projectId,
+        teamId: user.teamId,
+      },
+    });
+
+    if (!project) {
+      return NextResponse.json(
+        {
+          message: "User does not have access to this project",
+        },
+        { status: 403 }
+      );
+    }
+
+    // get traces
     const traceService = new TraceService();
     const traces = await traceService.GetTracesInProjectPaginated(
       projectId,