Jambonz support Model context protocol (MCP) (#1150)

* Jambonz support Model context protocol (MCP)

* merged mcp tools with existing llmOptions.tools

* support list of mcp servers

* wip

* wip

* wip

* fix voice agent

* fix open-ai

* fix review comment

* fix deepgram voice agent

* update verb specification version

lib/tasks/llm/index.js

@@ -4,6 +4,7 @@ const TaskLlmOpenAI_S2S = require('./llms/openai_s2s');
 const TaskLlmVoiceAgent_S2S = require('./llms/voice_agent_s2s');
 const TaskLlmUltravox_S2S = require('./llms/ultravox_s2s');
 const TaskLlmElevenlabs_S2S = require('./llms/elevenlabs_s2s');
+const LlmMcpService = require('../../utils/llm-mcp');
 
 class TaskLlm extends Task {
   constructor(logger, opts) {
@@ -18,6 +19,8 @@ class TaskLlm extends Task {
 
     // delegate to the specific llm model
     this.llm = this.createSpecificLlm();
+    // MCP
+    this.mcpServers = this.data.mcpServers || [];
   }
 
   get name() { return this.llm.name ; }
@@ -28,14 +31,32 @@ class TaskLlm extends Task {
 
   get ep() { return this.cs.ep; }
 
+  get mcpService() {
+    return this.llmMcpService;
+  }
+
+  get isMcpEnabled() {
+    return this.mcpServers.length > 0;
+  }
+
   async exec(cs, {ep}) {
     await super.exec(cs, {ep});
+
+    // create the MCP service if we have MCP servers
+    if (this.isMcpEnabled) {
+      this.llmMcpService = new LlmMcpService(this.logger, this.mcpServers);
+      await this.llmMcpService.init();
+    }
     await this.llm.exec(cs, {ep});
   }
 
   async kill(cs) {
     super.kill(cs);
     await this.llm.kill(cs);
+    // clean up MCP clients
+    if (this.isMcpEnabled) {
+      await this.mcpService.close();
+    }
   }
 
   createSpecificLlm() {

lib/tasks/llm/llms/elevenlabs_s2s.js

@@ -244,13 +244,36 @@ class TaskLlmElevenlabs_S2S extends Task {
     /* tool calls */
     else if (type === 'client_tool_call') {
       this.logger.debug({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent - function_call');
-      if (!this.toolHook) {
+      const {tool_name: name, tool_call_id: call_id, parameters: args} = evt.client_tool_call;
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (mcpTools.some((tool) => tool.name === name)) {
+        this.logger.debug({name, args}, 'TaskLlmElevenlabs_S2S:_onServerEvent - calling mcp tool');
+        try {
+          const res = await this.parent.mcpService.callMcpTool(name, args);
+          this.logger.debug({res}, 'TaskLlmElevenlabs_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(ep, call_id, {
+            data: {
+              type: 'client_tool_result',
+              tool_call_id: call_id,
+              result: res.content?.length ? res.content[0] : res.content,
+              is_error: false
+            }
+          });
+          return;
+        }
+        catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmElevenlabs_S2S - error calling mcp tool');
+          this.results = {
+            completionReason: 'client error calling mcp function',
+            error: err
+          };
+          endConversation = true;
+        }
+      } else if (!this.toolHook) {
         this.logger.warn({evt}, 'TaskLlmElevenlabs_S2S:_onServerEvent - no toolHook defined!');
       }
       else {
-        const {client_tool_call} = evt;
-        const {tool_name: name, tool_call_id: call_id, parameters: args} = client_tool_call;
-
         try {
           await this.parent.sendToolHook(call_id, {name, args});
         } catch (err) {

lib/tasks/llm/llms/openai_s2s.js

@@ -235,6 +235,23 @@ class TaskLlmOpenAI_S2S extends Task {
 
     /* send immediate session.update if present */
     else if (this.session_update) {
+      if (this.parent.isMcpEnabled) {
+        this.logger.debug('TaskLlmOpenAI_S2S:_sendInitialMessage - mcp enabled');
+        const tools = await this.parent.mcpService.getAvailableMcpTools();
+        if (tools && tools.length > 0 && this.session_update) {
+          const convertedTools = tools.map((tool) => ({
+            name: tool.name,
+            type: 'function',
+            description: tool.description,
+            parameters: tool.inputSchema
+          }));
+
+          this.session_update.tools = [
+            ...convertedTools,
+            ...(this.session_update.tools || [])
+          ];
+        }
+      }
       obj = {type: 'session.update', session: this.session_update};
       this.logger.debug({obj}, 'TaskLlmOpenAI_S2S:_sendInitialMessage - sending session.update');
       if (!await this._sendClientEvent(ep, obj)) {
@@ -299,13 +316,37 @@ class TaskLlmOpenAI_S2S extends Task {
     /* tool calls */
     else if (type === 'response.output_item.done' && evt.item?.type === 'function_call') {
       this.logger.debug({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - function_call');
-      if (!this.toolHook) {
+      const {name, call_id} = evt.item;
+      const args = JSON.parse(evt.item.arguments);
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (mcpTools.some((tool) => tool.name === name)) {
+        this.logger.debug({call_id, name, args}, 'TaskLlmOpenAI_S2S:_onServerEvent - calling mcp tool');
+        try {
+          const res = await this.parent.mcpService.callMcpTool(name, args);
+          this.logger.debug({res}, 'TaskLlmOpenAI_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(ep, call_id, {
+            type: 'conversation.item.create',
+            item: {
+              type: 'function_call_output',
+              call_id,
+              output: res.content[0]?.text || 'There is no output from the function call',
+            }
+          });
+          return;
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmOpenAI_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling mcp function',
+            error: err
+          };
+          endConversation = true;
+        }
+      }
+      else if (!this.toolHook) {
         this.logger.warn({evt}, 'TaskLlmOpenAI_S2S:_onServerEvent - no toolHook defined!');
       }
       else {
-        const {name, call_id} = evt.item;
-        const args = JSON.parse(evt.item.arguments);
-
         try {
           await this.parent.sendToolHook(call_id, {name, args});
         } catch (err) {

lib/tasks/llm/llms/ultravox_s2s.js

@@ -67,7 +67,50 @@ class TaskLlmUltravox_S2S extends Task {
     }
   }
 
+  /**
+   * Converts a JSON Schema to the dynamic parameters format used in the Ultravox API
+   * @param {Object} jsonSchema - A JSON Schema object defining parameters
+   * @param {string} locationDefault - Default location value for parameters (default: 'PARAMETER_LOCATION_BODY')
+   * @returns {Array} Array of dynamic parameters objects
+   */
+  transformSchemaToParameters(jsonSchema, locationDefault = 'PARAMETER_LOCATION_BODY') {
+    if (jsonSchema.properties) {
+      const required = jsonSchema.required || [];
+
+      return Object.entries(jsonSchema.properties).map(([name]) => {
+        return {
+          name,
+          location: locationDefault,
+          required: required.includes(name)
+        };
+      });
+    }
+
+    return [];
+  }
+
   async createCall() {
+    const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+    if (mcpTools && mcpTools.length > 0) {
+      const convertedTools = mcpTools.map((tool) => {
+        return {
+          temporaryTool: {
+            modelToolName: tool.name,
+            description: tool.description,
+            dynamicParameters: this.transformSchemaToParameters(tool.inputSchema),
+            // use client tool that ultravox call tool via freeswitch module.
+            client: {}
+          }
+        };
+      }
+      );
+      // merge with any existing tools
+      this.data.llmOptions.selectedTools = [
+        ...convertedTools,
+        ...(this.data.llmOptions.selectedTools || [])
+      ];
+    }
+
     const payload = {
       ...this.data.llmOptions,
       model: this.model,
@@ -182,12 +225,35 @@ class TaskLlmUltravox_S2S extends Task {
     /* tool calls */
     else if (type === 'client_tool_invocation') {
       this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call');
-      if (!this.toolHook) {
+      const {toolName: name, invocationId: call_id, parameters: args} = evt;
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (mcpTools.some((tool) => tool.name === name)) {
+        this.logger.debug({
+          name,
+          input: args
+        }, 'TaskLlmUltravox_S2S:_onServerEvent - function_call - mcp tool');
+        try {
+          const res = await this.parent.mcpService.callMcpTool(name, args);
+          this.logger.debug({res}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(_ep, call_id, {
+            type: 'client_tool_result',
+            invocation_id: call_id,
+            result: res.content
+          });
+          return;
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmUltravox_S2S - error calling mcp tool');
+          this.results = {
+            completionReason: 'client error calling mcp function',
+            error: err
+          };
+          endConversation = true;
+        }
+      } else if (!this.toolHook) {
         this.logger.info({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - no toolHook defined!');
       }
       else {
-        const {toolName: name, invocationId: call_id, parameters: args} = evt;
-
         try {
           await this.parent.sendToolHook(call_id, {name, args});
         } catch (err) {

lib/tasks/llm/llms/voice_agent_s2s.js

@@ -193,6 +193,19 @@ class TaskLlmVoiceAgent_S2S extends Task {
   }
 
   async _sendInitialMessage(ep) {
+    const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+    if (mcpTools && mcpTools.length > 0 && this.settingsConfiguration.agent?.think) {
+      const convertedTools = mcpTools.map((tool) => ({
+        name: tool.name,
+        description: tool.description,
+        parameters: tool.inputSchema
+      }));
+
+      this.settingsConfiguration.agent.think.functions = [
+        ...convertedTools,
+        ...(this.settingsConfiguration.agent.think?.functions || [])
+      ];
+    }
     if (!await this._sendClientEvent(ep, this.settingsConfiguration)) {
       this.notifyTaskDone();
     }
@@ -254,13 +267,34 @@ class TaskLlmVoiceAgent_S2S extends Task {
     /* tool calls */
     else if (type === 'FunctionCallRequest') {
       this.logger.debug({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - function_call');
-      if (!this.toolHook) {
+      const {function_name:name, function_call_id:call_id, input: args} = evt;
+
+      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
+      if (mcpTools.some((tool) => tool.name === name)) {
+        this.logger.debug({call_id, name, args}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - calling mcp tool');
+        try {
+          const res = await this.parent.mcpService.callMcpTool(name, args);
+          this.logger.debug({res}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - function_call - mcp result');
+          this.processToolOutput(_ep, call_id, {
+            data: {
+              type: 'FunctionCallResponse',
+              function_call_id: call_id,
+              output: res.content[0]?.text || 'There is no output from the function call',
+            }
+          });
+          return;
+        } catch (err) {
+          this.logger.info({err, evt}, 'TaskLlmVoiceAgent_S2S - error calling function');
+          this.results = {
+            completionReason: 'client error calling mcp function',
+            error: err
+          };
+          endConversation = true;
+        }
+      } else if (!this.toolHook) {
         this.logger.warn({evt}, 'TaskLlmVoiceAgent_S2S:_onServerEvent - no toolHook defined!');
       }
       else {
-        const {function_name:name, function_call_id:call_id} = evt;
-        const args = evt.input;
-
         try {
           await this.parent.sendToolHook(call_id, {name, args});
         } catch (err) {