jambonz-feature-server/lib/tasks/llm/llms/ultravox_s2s.js

const Task = require('../../task');
const TaskName = 'Llm_Ultravox_s2s';
const {request} = require('undici');
const {LlmEvents_Ultravox} = require('../../../utils/constants');

const ultravox_server_events = [
  'createCall',
  'pong',
  'state',
  'transcript',
  'conversationText',
  'clientToolInvocation',
  'playbackClearBuffer',
];

const ClientEvent = 'client.event';

const expandWildcards = (events) => {
  // no-op for deepgram
  return events;
};

const SessionDelete = 'session.delete';

class TaskLlmUltravox_S2S extends Task {
  constructor(logger, opts, parentTask) {
    super(logger, opts, parentTask);
    this.parent = parentTask;

    this.vendor = this.parent.vendor;
    this.model = this.parent.model || 'fixie-ai/ultravox';
    this.auth = this.parent.auth;
    this.connectionOptions = this.parent.connectOptions;

    const {apiKey} = this.auth || {};
    if (!apiKey) throw new Error('auth.apiKey is required for Vendor: Ultravox');
    this.apiKey = apiKey;
    this.actionHook = this.data.actionHook;
    this.eventHook = this.data.eventHook;
    this.toolHook = this.data.toolHook;

    this.results = {
      completionReason: 'normal conversation end'
    };

    /**
     * only one of these will have items,
     * if includeEvents, then these are the events to include
     * if excludeEvents, then these are the events to exclude
     */
    this.includeEvents = [];
    this.excludeEvents = [];

    /* default to all events if user did not specify */
    this._populateEvents(this.data.events || ultravox_server_events);

    this.addCustomEventListener = parentTask.addCustomEventListener.bind(parentTask);
    this.removeCustomEventListeners = parentTask.removeCustomEventListeners.bind(parentTask);
  }

  get name() { return TaskName; }

  async _api(ep, args) {
    const res = await ep.api('uuid_ultravox_s2s', `^^|${args.join('|')}`);
    if (!res.body?.startsWith('+OK')) {
      throw new Error(`Error calling uuid_ultravox_s2s: ${JSON.stringify(res.body)}`);
    }
  }

  /**
   * Converts a JSON Schema to the dynamic parameters format used in the Ultravox API
   * @param {Object} jsonSchema - A JSON Schema object defining parameters
   * @param {string} locationDefault - Default location value for parameters (default: 'PARAMETER_LOCATION_BODY')
   * @returns {Array} Array of dynamic parameters objects
   */
  transformSchemaToParameters(jsonSchema, locationDefault = 'PARAMETER_LOCATION_BODY') {
    if (jsonSchema.properties) {
      const required = jsonSchema.required || [];

      return Object.entries(jsonSchema.properties).map(([name]) => {
        return {
          name,
          location: locationDefault,
          required: required.includes(name)
        };
      });
    }

    return [];
  }

  async createCall() {
    const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
    if (mcpTools && mcpTools.length > 0) {
      const convertedTools = mcpTools.map((tool) => {
        return {
          temporaryTool: {
            modelToolName: tool.name,
            description: tool.description,
            dynamicParameters: this.transformSchemaToParameters(tool.inputSchema),
            // use client tool that ultravox call tool via freeswitch module.
            client: {}
          }
        };
      }
      );
      // merge with any existing tools
      this.data.llmOptions.selectedTools = [
        ...convertedTools,
        ...(this.data.llmOptions.selectedTools || [])
      ];
    }

    const payload = {
      ...this.data.llmOptions,
      model: this.model,
      medium: {
        ...(this.data.llmOptions.medium || {}),
        serverWebSocket: {
          inputSampleRate: 8000,
          outputSampleRate: 8000,
        }
      }
    };
    const {statusCode, body} = await request('https://api.ultravox.ai/api/calls', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'X-API-Key': this.apiKey
      },
      body: JSON.stringify(payload)
    });
    const data = await body.json();
    if (statusCode !== 201 || !data?.joinUrl) {
      this.logger.info({statusCode, data}, 'Ultravox Error registering call');
      throw new Error(`Ultravox Error registering call:${statusCode} - ${data.detail}`);
    }
    this.logger.debug({joinUrl: data.joinUrl}, 'Ultravox Call registered');
    return data;
  }

  _unregisterHandlers() {
    this.removeCustomEventListeners();
  }

  _registerHandlers(ep) {
    this.addCustomEventListener(ep, LlmEvents_Ultravox.Connect, this._onConnect.bind(this, ep));
    this.addCustomEventListener(ep, LlmEvents_Ultravox.ConnectFailure, this._onConnectFailure.bind(this, ep));
    this.addCustomEventListener(ep, LlmEvents_Ultravox.Disconnect, this._onDisconnect.bind(this, ep));
    this.addCustomEventListener(ep, LlmEvents_Ultravox.ServerEvent, this._onServerEvent.bind(this, ep));
  }

  async _startListening(cs, ep) {
    this._registerHandlers(ep);

    try {
      const data = await this.createCall();
      const {joinUrl} = data;
      // split the joinUrl into host and path
      const {host, pathname, search} = new URL(joinUrl);
      const args = [ep.uuid, 'session.create', host, pathname + search];
      await this._api(ep, args);
      // Notify the application that the session has been created with detail information
      this._sendLlmEvent('createCall', {
        type: 'createCall',
        ...data
      });
    } catch (err) {
      this.logger.info({err}, 'TaskLlmUltraVox_S2S:_startListening - Error sending createCall');
      this.results = {completionReason: `connection failure - ${err}`};
      this.notifyTaskDone();
    }
  }

  async exec(cs, {ep}) {
    await super.exec(cs);

    await this._startListening(cs, ep);

    await this.awaitTaskDone();

    /* note: the parent llm verb started the span, which is why this is necessary */
    await this.parent.performAction(this.results);

    this._unregisterHandlers();
  }

  async kill(cs) {
    super.kill(cs);

    this._api(cs.ep, [cs.ep.uuid, SessionDelete])
      .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:kill - error deleting session'));

    this.notifyTaskDone();
  }

  _onConnect(ep) {
    this.logger.info('TaskLlmUltravox_S2S:_onConnect');
  }
  _onConnectFailure(_ep, evt) {
    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
    this.results = {completionReason: 'connection failure'};
    this.notifyTaskDone();
  }
  _onDisconnect(_ep, evt) {
    this.logger.info(evt, 'TaskLlmUltravox_S2S:_onConnectFailure');
    this.results = {completionReason: 'disconnect from remote end'};
    this.notifyTaskDone();
  }

  async _onServerEvent(_ep, evt) {
    let endConversation = false;
    const type = evt.type;
    this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent');

    /* server errors of some sort */
    if (type === 'error') {
      endConversation = true;
      this.results = {
        completionReason: 'server error',
        error: evt.error
      };
    }

    /* tool calls */
    else if (type === 'client_tool_invocation') {
      this.logger.debug({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call');
      const {toolName: name, invocationId: call_id, parameters: args} = evt;

      const mcpTools = this.parent.isMcpEnabled ? await this.parent.mcpService.getAvailableMcpTools() : [];
      if (mcpTools.some((tool) => tool.name === name)) {
        this.logger.debug({
          name,
          input: args
        }, 'TaskLlmUltravox_S2S:_onServerEvent - function_call - mcp tool');
        try {
          const res = await this.parent.mcpService.callMcpTool(name, args);
          this.logger.debug({res}, 'TaskLlmUltravox_S2S:_onServerEvent - function_call - mcp result');
          this.processToolOutput(_ep, call_id, {
            type: 'client_tool_result',
            invocation_id: call_id,
            result: res.content
          });
          return;
        } catch (err) {
          this.logger.info({err, evt}, 'TaskLlmUltravox_S2S - error calling mcp tool');
          this.results = {
            completionReason: 'client error calling mcp function',
            error: err
          };
          endConversation = true;
        }
      } else if (!this.toolHook) {
        this.logger.info({evt}, 'TaskLlmUltravox_S2S:_onServerEvent - no toolHook defined!');
      }
      else {
        try {
          await this.parent.sendToolHook(call_id, {name, args});
        } catch (err) {
          this.logger.info({err, evt}, 'TaskLlmUltravox_S2S - error calling function');
          this.results = {
            completionReason: 'client error calling function',
            error: err
          };
          endConversation = true;
        }
      }
    }

    this._sendLlmEvent(type, evt);

    if (endConversation) {
      this.logger.info({results: this.results},
        'TaskLlmUltravox_S2S:_onServerEvent - ending conversation due to error');
      this.notifyTaskDone();
    }
  }

  _sendLlmEvent(type, evt) {
    /* check whether we should notify on this event */
    if (this.includeEvents.length > 0 ? this.includeEvents.includes(type) : !this.excludeEvents.includes(type)) {
      this.parent.sendEventHook(evt)
        .catch((err) => this.logger.info({err}, 'TaskLlmUltravox_S2S:_onServerEvent - error sending event hook'));
    }
  }

  async processLlmUpdate(ep, data, _callSid) {
    try {
      this.logger.debug({data, _callSid}, 'TaskLlmUltravox_S2S:processLlmUpdate');

      if (!data.type || ![
        'input_text_message'
      ].includes(data.type)) {
        this.logger.info({data},
          'TaskLlmUltravox_S2S:processLlmUpdate - invalid mid-call request, only input_text_message supported');
      }
      else {
        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
      }
    } catch (err) {
      this.logger.info({err, data}, 'TaskLlmUltravox_S2S:processLlmUpdate - Error processing LLM update');
    }
  }

  async processToolOutput(ep, tool_call_id, data) {
    try {
      this.logger.debug({tool_call_id, data}, 'TaskLlmUltravox_S2S:processToolOutput');

      if (!data.type || data.type !== 'client_tool_result') {
        this.logger.info({data},
          'TaskLlmUltravox_S2S:processToolOutput - invalid tool output, must be client_tool_result');
      }
      else {
        await this._api(ep, [ep.uuid, ClientEvent, JSON.stringify(data)]);
      }
    } catch (err) {
      this.logger.info({err, data}, 'TaskLlmUltravox_S2S:processToolOutput - Error processing tool output');
    }
  }

  _populateEvents(events) {
    if (events.includes('all')) {
      /* work by excluding specific events */
      const exclude = events
        .filter((evt) => evt.startsWith('-'))
        .map((evt) => evt.slice(1));
      if (exclude.length === 0) this.includeEvents = ultravox_server_events;
      else this.excludeEvents = expandWildcards(exclude);
    }
    else {
      /* work by including specific events */
      const include = events
        .filter((evt) => !evt.startsWith('-'));
      this.includeEvents = expandWildcards(include);
    }

    this.logger.debug({
      includeEvents: this.includeEvents,
      excludeEvents: this.excludeEvents
    }, 'TaskLlmUltravox_S2S:_populateEvents');
  }
}

module.exports = TaskLlmUltravox_S2S;