feat(datatets-supervisor): improvement of the supervisor graph + langsmith datasets

config/guards/default.guards.json

@@ -69,7 +69,6 @@
     "max_profile_length": 255
   },
   "agents": {
-    "prompts_id_max_length": 255,
     "profile": {
       "name_max_length": 100,
       "name_min_length": 3,

datasets/.gitignore

@@ -0,0 +1,8 @@
+# Ignore all CSV files except the example
+*.dataset.csv
+!example-dataset.dataset.csv
+!supervisor-supervisor.dataset.csv
+!supervisor-agentConfigurationHelper.dataset.csv
+!supervisor-mcpConfigurationHelper.dataset.csv
+!supervisor-snakRagAgentHelper.dataset.csv
+!supervisor-agentSelectorHelper.dataset.csv

datasets/supervisor-agentConfigurationHelper.dataset.csv

@@ -0,0 +1,11 @@
+messages,output_direction,dataset_split,operation_type,difficulty
+"[{""role"":""user"",""content"":""can you create an agent""}]","Ask for agent purpose using message_ask_user","[""training""]","[""create_agent""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""can you create a websearch agent""}]","Ask to specify websearch agent purpose using message_ask_user","[""training""]","[""create_agent""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""can you create a websearch agent specialized in retrieving guitar information""}]","Create agent using create_agent tool","[""training""]","[""create_agent""]","[""easy""]"
+"[{""role"":""user"",""content"":""can you create an agent""},{""role"":""assistant"",""content"":""What is the purpose of the agent you want to create?""},{""role"":""user"",""content"":""an agent specialized in websearch""}]","Ask for specific websearch purpose using message_ask_user","[""training""]","[""create_agent""]","[""easy""]"
+"[{""role"":""user"",""content"":""can you create a websearch agent""},{""role"":""assistant"",""content"":""What is the purpose of your websearch agent?""},{""role"":""user"",""content"":""an agent specialized in finding weather for a given city""}]","Create agent using create_agent tool","[""validation""]","[""create_agent""]","[""medium""]"
+"[{""role"":""user"",""content"":""i want to update my agent GuitarFindingAgent""}]","Ask what they want to modify using message_ask_user","[""training""]","[""update_agent""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""i want to increase the memory of my agent TradingShortExtended""}]","Use read_agent tool on TradingShortExtended","[""training""]","[""update_agent""]","[""easy""]"
+"[{""role"":""user"",""content"":""i want to increase the memory of my agent TradingShortExtended""},{""role"":""tool"",""name"":""read_agent"",""content"":""Error: No agent found with name TradingShortExtended"",""tool_call_id"":""call_1""}]","Use list_agents tool to show available agents","[""validation""]","[""update_agent""]","[""medium""]"
+"[{""role"":""user"",""content"":""can you delete an agent""}]","Use list_agents tool","[""training""]","[""delete_agent""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""can you delete agent PokemonTrainer""}]","Ask for deletion confirmation using message_ask_user","[""training""]","[""delete_agent""]","[""easy""]"

datasets/supervisor-agentSelectorHelper.dataset.csv

@@ -0,0 +1 @@
+messages,output_direction,dataset_split,operation_type,difficulty

datasets/supervisor-mcpConfigurationHelper.dataset.csv

@@ -0,0 +1 @@
+messages,output_direction,dataset_split,operation_type,difficulty

datasets/supervisor-snakRagAgentHelper.dataset.csv

@@ -0,0 +1 @@
+messages,output_direction,dataset_split,operation_type,difficulty

datasets/supervisor-supervisor.dataset.csv

@@ -0,0 +1,18 @@
+messages,output_direction,dataset_split,operation_type,difficulty
+"[{""role"":""user"",""content"":""can you create an agent config""}]","The agent should transfer_to_agentconfigurationhelper","[""training""]","[""transfer_to_agentconfigurationhelper""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""can you create an Trading agent""}]","The agent should transfer_to_agentconfigurationhelper","[""training""]","[""transfer_to_agentconfigurationhelper""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""can you update my Trading Agent Config""}]","The agent should transfer_to_agentconfigurationhelper","[""training""]","[""transfer_to_agentconfigurationhelper""]","[""easy""]"
+"[{""role"":""user"",""content"":""update my agents""}]","The agent should transfer_to_agentconfigurationhelper","[""training""]","[""transfer_to_agentconfigurationhelper""]","[""easy""]"
+"[{""role"":""user"",""content"":""delete agent PokemonTrainer""}]","The agent should transfer_to_agentconfigurationhelper","[""validation""]","[""transfer_to_agentconfigurationhelper""]","[""easy""]"
+"[{""role"":""user"",""content"":""what are my agents""}]","The agent should transfer_to_agentconfigurationhelper","[""validation""]","[""transfer_to_agentconfigurationhelper""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""what are the best websearch mcps""}]","The agent should transfer_to_mcpconfigurationhelper","[""training""]","[""transfer_to_mcpconfigurationhelper""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""what are the best mcps""}]","The agent should transfer_to_mcpconfigurationhelper","[""training""]","[""transfer_to_mcpconfigurationhelper""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""can you add the exa-search mcps to my agent""}]","The agent should transfer_to_mcpconfigurationhelper","[""training""]","[""transfer_to_mcpconfigurationhelper""]","[""easy""]"
+"[{""role"":""user"",""content"":""can you update my mcps config of agents PokemonTrainer""}]","The agent should transfer_to_mcpconfigurationhelper","[""training""]","[""transfer_to_mcpconfigurationhelper""]","[""medium""]"
+"[{""role"":""user"",""content"":""Can you delete all the mcps exa-search of my agents""}]","The agent should transfer_to_mcpconfigurationhelper","[""validation""]","[""transfer_to_mcpconfigurationhelper""]","[""medium""]"
+"[{""role"":""user"",""content"":""can you get the best car trends""}]","The agent should transfer_to_agentselectorhelper","[""training""]","[""transfer_to_agentselectorhelper""]","[""easy""]"
+"[{""role"":""user"",""content"":""can you start the agent PokemonTrainer with request : What are the best teams in HearthGold-Soulsilver""}]","The agent should transfer_to_agentselectorhelper","[""training""]","[""transfer_to_agentselectorhelper""]","[""medium""]"
+"[{""role"":""user"",""content"":""can you run an random agent""}]","The agent should transfer_to_agentselectorhelper","[""validation""]","[""transfer_to_agentselectorhelper""]","[""easy""]"
+"[{""role"":""user"",""content"":""What is the posssibilities with SNAK""}]","The agent should transfer_to_snakragagenthelper","[""training""]","[""transfer_to_snakragagenthelper""]","[""easy""]"
+"[{""role"":""user"",""content"":""What SNAK means ?""}]","The agent should transfer_to_snakragagenthelper","[""training""]","[""transfer_to_snakragagenthelper""]","[""very-easy""]"
+"[{""role"":""user"",""content"":""Who created Snak ?""}]","The agent should transfer_to_snakragagenthelper","[""validation""]","[""transfer_to_snakragagenthelper""]","[""very-easy""]"

docker-compose.dev.yml

@@ -30,22 +30,16 @@ services:
       - network
     ports:
       - '127.0.0.1:6379:6379'
-    volumes:
-      - redis_data:/data
     env_file:
       - .env
-    command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD:?REDIS_PASSWORD is required}"]
+    command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?REDIS_PASSWORD is required}"]
     healthcheck:
       test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD:?REDIS_PASSWORD is required}", "ping"]
       interval: 10s
       timeout: 5s
       retries: 5
     restart: unless-stopped
 
-volumes:
-  redis_data:
-    driver: local
-
 networks:
   network:
     driver: bridge

package.json

@@ -49,6 +49,7 @@
     "node:server": "node ./mcps/snak/dist/index.js",
     "start:prod": "lerna run start --scope \"@snakagent/server\"; ECODE=$? ; exit $ECODE",
     "test_metrics": "jest packages/metrics/src/__tests__/metrics.test.ts",
+    "datasets": "tsx --tsconfig=packages/agent/tsconfig.json --env-file=.env packages/agent/src/agents/langsmith/run-datasets.ts",
     "format": "prettier --write \"packages/**/*.{ts,tsx,js,jsx,json,md}\""
   },
   "dependencies": {
@@ -60,11 +61,10 @@
     "@langchain/langgraph": "^0.4.9",
     "@langchain/langgraph-checkpoint": "~0.1.1",
     "@langchain/langgraph-checkpoint-postgres": "^0.1.2",
-    "@langchain/mcp-adapters": "^0.6.0",
     "@langchain/langgraph-supervisor": "^0.0.20",
+    "@langchain/mcp-adapters": "^0.6.0",
     "@langchain/ollama": "^0.1.6",
     "@langchain/openai": "^0.3.17",
-    "langchain": "^0.3.34",
     "@nestjs/platform-socket.io": "^11.1.1",
     "@nestjs/websockets": "^11.1.1",
     "add": "^2.0.6",
@@ -73,6 +73,9 @@
     "dotenv": "^16.6.1",
     "ethers": "^6.15.0",
     "express": "^4.21.2",
+    "langchain": "^0.3.34",
+    "langsmith": "^0.3.73",
+    "openevals": "^0.1.1",
     "pg": "^8.15.6",
     "prom-client": "^15.1.3",
     "socket.io": "^4.8.1",
@@ -96,6 +99,7 @@
     "@types/yargs": "^17.0.33",
     "@typescript-eslint/eslint-plugin": "^8.46.0",
     "@typescript-eslint/parser": "^8.46.0",
+    "concurrently": "^9.2.1",
     "eslint": "^9.37.0",
     "eslint-config-prettier": "^9.1.2",
     "eslint-plugin-prettier": "^5.5.4",
@@ -114,6 +118,7 @@
     "turbo": "^2.5.8",
     "typescript": "^5.9.3",
     "typescript-eslint": "^8.46.0",
+    "wait-on": "^9.0.1",
     "zod-to-json-schema": "^3.24.6"
   },
   "engines": {
@@ -125,7 +130,8 @@
       "is-core-module": "2.13.1"
     },
     "patchedDependencies": {
-      "@google/[email protected]": "patches/@[email protected]"
+      "@google/[email protected]": "patches/@[email protected]",
+      "@langchain/[email protected]": "patches/@[email protected]"
     }
   }
 }

packages/agent/src/agents/core/snakAgent.ts

@@ -25,7 +25,7 @@ import {
   getInterruptCommand,
   isInterrupt,
 } from '@agents/graphs/utils/graph.utils.js';
-
+import { v4 as uuidv4 } from 'uuid';
 /**
  * Main agent for interacting with the Starknet blockchain
  * Supports multiple execution modes: interactive, autonomous, and hybrid
@@ -189,13 +189,14 @@ export class SnakAgent extends BaseAgent {
       ls_model_type: chunk.metadata.ls_model_type,
       ls_temperature: chunk.metadata.ls_temperature,
       tokens: chunk.data.output?.usage_metadata?.total_tokens ?? null,
-      user_request: user_request,
+      content: user_request,
       error: graphError,
       retry: retryCount,
     };
 
     const chunkOutput: ChunkOutput = {
       event: chunk.event,
+      agent_id: this.agentConfig.id,
       run_id: chunk.run_id,
       checkpoint_id: state.config.configurable?.checkpoint_id,
       thread_id: state.config.configurable?.thread_id,
@@ -304,7 +305,7 @@ export class SnakAgent extends BaseAgent {
       const initialMessages: BaseMessage[] = [
         new HumanMessage(request.request),
       ];
-      const threadId = this.agentConfig.id;
+      const threadId = request.thread_id ? request.thread_id : uuidv4();
       const configurable = {
         thread_id: threadId,
         user_request: {
@@ -392,6 +393,7 @@ export class SnakAgent extends BaseAgent {
         }
         yield {
           event: lastChunk.event,
+          agent_id: this.agentConfig.id,
           run_id: lastChunk.run_id,
           from: GraphNode.END_GRAPH,
           thread_id: threadId,
@@ -407,7 +409,7 @@ export class SnakAgent extends BaseAgent {
             error: graphError,
             final: true,
             is_human: isInterruptHandle,
-            user_request: request.request,
+            content: request.request,
           },
           timestamp: new Date().toISOString(),
         };

packages/agent/src/agents/core/supervisorAgent.ts

@@ -5,13 +5,12 @@ import {
   ChunkOutput,
   ChunkOutputMetadata,
 } from '../../shared/types/streaming.type.js';
-import { createSupervisorGraph } from '@agents/graphs/core-graph/supervisor.graph.js';
-import { CheckpointerService } from '@agents/graphs/manager/checkpointer/checkpointer.js';
 import {
-  AIMessage,
-  AIMessageChunk,
-  HumanMessage,
-} from '@langchain/core/messages';
+  createSupervisorGraph,
+  SupervisorGraph,
+} from '@agents/graphs/core-graph/supervisor.graph.js';
+import { CheckpointerService } from '@agents/graphs/manager/checkpointer/checkpointer.js';
+import { AIMessage, HumanMessage } from '@langchain/core/messages';
 import { GraphErrorType, UserRequest } from '@stypes/graph.type.js';
 import { EventType } from '@enums/event.enums.js';
 import { StreamEvent } from '@langchain/core/tracers/log_stream';
@@ -22,11 +21,12 @@ import {
   isInterrupt,
 } from '@agents/graphs/utils/graph.utils.js';
 import { notify } from '@snakagent/database/queries';
-
+import { v4 as uuidv4 } from 'uuid';
 /**
  * Supervisor agent for managing and coordinating multiple agents
  */
 export class SupervisorAgent extends BaseAgent {
+  supervisorGraphInstance: SupervisorGraph | null = null;
   constructor(agent_config: AgentConfig.Runtime) {
     super('supervisor', AgentType.SUPERVISOR, agent_config);
   }
@@ -40,23 +40,27 @@ export class SupervisorAgent extends BaseAgent {
       if (!this.agentConfig) {
         throw new Error('Agent configuration is required for initialization');
       }
-
       this.pgCheckpointer = await CheckpointerService.getInstance();
       if (!this.pgCheckpointer) {
         throw new Error('Failed to initialize Postgres checkpointer');
       }
-      const graph = await createSupervisorGraph(this);
-      if (!graph) {
+      this.supervisorGraphInstance = await createSupervisorGraph(this);
+      if (!this.supervisorGraphInstance) {
         throw new Error('Failed to create supervisor graph');
       }
-      this.compiledStateGraph = graph;
+
+      this.compiledStateGraph = this.supervisorGraphInstance.getCompiledGraph();
       logger.info('[SupervisorAgent] Initialized successfully');
     } catch (error) {
       logger.error(`[SupervisorAgent] Initialization failed: ${error}`);
       throw error;
     }
   }
 
+  public getSupervisorGraphInstance(): SupervisorGraph | null {
+    return this.supervisorGraphInstance;
+  }
+
   /**
    * Creates a standardized chunk output
    */
@@ -78,12 +82,13 @@ export class SupervisorAgent extends BaseAgent {
       ls_model_type: chunk.metadata.ls_model_type,
       ls_temperature: chunk.metadata.ls_temperature,
       tokens: chunk.data.output?.usage_metadata?.total_tokens ?? null,
-      user_request: user_request,
+      content: user_request,
       error: graphError,
       retry: retryCount,
     };
     const chunkOutput: ChunkOutput = {
       event: chunk.event,
+      agent_id: this.agentConfig.id,
       run_id: chunk.run_id,
       checkpoint_id: state.config.configurable?.checkpoint_id,
       thread_id: state.config.configurable?.thread_id,
@@ -154,12 +159,13 @@ export class SupervisorAgent extends BaseAgent {
    * @param input - The input for execution
    * @returns AsyncGenerator yielding ChunkOutput
    */
-  public async *execute(userRequest: UserRequest): AsyncGenerator<ChunkOutput> {
+  public async *execute(request: UserRequest): AsyncGenerator<ChunkOutput> {
     try {
       let currentCheckpointId: string | undefined = undefined;
       let lastChunk: StreamEvent | undefined = undefined;
       let stateSnapshot: StateSnapshot;
       let isInterruptHandle = false;
+      let isTransferHandle = false;
       if (!this.compiledStateGraph) {
         throw new Error('SupervisorAgent is not initialized');
       }
@@ -169,7 +175,7 @@ export class SupervisorAgent extends BaseAgent {
       if (this.pgCheckpointer === null) {
         throw new Error('Checkpointer is not initialized');
       }
-      const threadId = this.agentConfig.id;
+      const threadId = request.thread_id ? request.thread_id : uuidv4();
       const configurable = {
         thread_id: threadId,
         agent_config: this.agentConfig,
@@ -183,22 +189,38 @@ export class SupervisorAgent extends BaseAgent {
         recursionLimit: 500,
         version: 'v2' as const,
       };
-      stateSnapshot = await this.compiledStateGraph.getState(executionConfig);
+      stateSnapshot = await this.compiledStateGraph.getState(executionConfig, {
+        subgraphs: true,
+      });
       if (!stateSnapshot) {
         throw new Error('Failed to retrieve initial graph state');
       }
       const executionInput = isInterrupt(stateSnapshot)
-        ? getInterruptCommand(userRequest.request)
-        : { messages: [new HumanMessage(userRequest.request || '')] };
+        ? getInterruptCommand(request.request)
+        : { messages: [new HumanMessage(request.request || '')] };
 
+      if (
+        stateSnapshot.values.transfer_to &&
+        stateSnapshot.values.transfer_to.length > 0
+      ) {
+        await this.compiledStateGraph.updateState(executionConfig, {
+          transfer_to: [],
+        });
+      }
       for await (const chunk of this.compiledStateGraph.streamEvents(
         executionInput,
         executionConfig
       )) {
-        stateSnapshot = await this.compiledStateGraph.getState(executionConfig);
+        stateSnapshot = await this.compiledStateGraph.getState(
+          executionConfig,
+          { subgraphs: true }
+        );
         if (!stateSnapshot) {
           throw new Error('Failed to retrieve graph state during execution');
         }
+        isTransferHandle =
+          stateSnapshot.values.transfer_to &&
+          stateSnapshot.values.transfer_to.length > 0;
         currentCheckpointId = stateSnapshot.config.configurable?.checkpoint_id;
         lastChunk = chunk;
         if (
@@ -216,30 +238,22 @@ export class SupervisorAgent extends BaseAgent {
         const chunkProcessed = this.processChunkOutput(
           chunk,
           stateSnapshot,
-          userRequest.request,
+          request.request,
           0
         );
         if (chunkProcessed) {
           yield chunkProcessed;
         }
       }
-
-      if (!this.pgCheckpointer) {
-        throw new Error('Checkpointer is not initialized');
-      }
-
-      if (!isInterruptHandle) {
-        const startTime = Date.now();
-        const endTime = Date.now();
-        const duration = endTime - startTime;
-        await this.pgCheckpointer.deleteThread(threadId);
-        logger.info(`[SupervisorAgent] deleteThread took ${duration}ms`);
-      }
       if (!lastChunk || !currentCheckpointId) {
         throw new Error('No output from autonomous execution');
       }
+      logger.info(
+        `[SupervisorAgent] Execution completed for thread ${threadId}`
+      );
       yield {
         event: lastChunk.event,
+        agent_id: this.agentConfig.id,
         run_id: lastChunk.run_id,
         from: SupervisorNode.END_GRAPH,
         thread_id: threadId,
@@ -251,7 +265,10 @@ export class SupervisorAgent extends BaseAgent {
           error: undefined,
           final: true,
           is_human: isInterruptHandle,
-          user_request: userRequest.request,
+          content: request.request,
+          transfer_to: isTransferHandle
+            ? stateSnapshot.values.transfer_to
+            : null,
         },
         timestamp: new Date().toISOString(),
       };

packages/agent/src/agents/graphs/constants/execution-constants.ts

@@ -74,3 +74,5 @@ export const STRING_LIMITS = {
     'execution.max_description_length'
   ),
 } as const;
+
+export const MAX_SUPERVISOR_MESSAGE = 30;

packages/agent/src/agents/graphs/core-graph/agent.graph.ts

@@ -32,7 +32,6 @@ import {
   Memories,
   skipValidationType,
   TaskType,
-  UserRequest,
   userRequestWithHITL,
 } from '../../../shared/types/index.js';
 import { MemoryStateManager } from '../manager/memory/memory-utils.js';