bugfix: handle error if we cannot get our own ipv4

simplify error message
send error notification over websocket if tts fails
2026-02-10 08:21:33 +00:00 · 2022-04-21 19:09:23 -04:00 · 2022-04-21 14:43:09 -04:00 · 2022-04-21 14:33:49 -04:00 · 2022-04-21 10:27:33 -04:00 · 2022-04-17 17:53:16 -04:00
36 changed files with 2283 additions and 227 deletions
--- a/8
+++ b/8
@@ -1,10 +1,10 @@
-FROM node:17.4-slim
+FROM node:slim
 WORKDIR /opt/app/
-COPY package.json ./
-RUN npm install
+COPY package.json package-lock.json ./
+RUN npm ci
 RUN npm prune
 COPY . /opt/app
 ARG NODE_ENV
 ENV NODE_ENV $NODE_ENV

-CMD [ "npm", "start" ]
+CMD [ "npm", "start" ]
--- a/app.js
+++ b/app.js
@@ -11,6 +11,10 @@ assert.ok(process.env.JAMBONES_NETWORK_CIDR || process.env.K8S, 'missing JAMBONE

 const Srf = require('drachtio-srf');
 const srf = new Srf();
+const tracer = require('./tracer')(process.env.JAMBONES_OTEL_SERVICE_NAME || 'jambonz-feature-server');
+const api = require('@opentelemetry/api');
+srf.locals = {...srf.locals, otel: {tracer, api}};
+
 const PORT = process.env.HTTP_PORT || 3000;
 const opts = {
  timestamp: () => {return `, "time": "${new Date().toISOString()}"`;},
@@ -23,6 +27,7 @@ installSrfLocals(srf, logger);

 const {
  initLocals,
+  createRootSpan,
  getAccountDetails,
  normalizeNumbers,
  retrieveApplication,
@@ -62,6 +67,7 @@ if (process.env.NODE_ENV === 'test') {

 srf.use('invite', [
  initLocals,
+  createRootSpan,
  getAccountDetails,
  normalizeNumbers,
  retrieveApplication,
--- a/lib/http-routes/api/create-call.js
+++ b/lib/http-routes/api/create-call.js
@@ -8,10 +8,12 @@ const SipError = require('drachtio-srf').SipError;
 const sysError = require('./error');
 const HttpRequestor = require('../../utils/http-requestor');
 const WsRequestor = require('../../utils/ws-requestor');
+const RootSpan = require('../../utils/call-tracer');
 const dbUtils = require('../../utils/db-utils');

 router.post('/', async(req, res) => {
  const {logger} = req.app.locals;
+  const accountSid = req.body.account_sid;

  logger.debug({body: req.body}, 'got createCall request');
  try {
@@ -39,7 +41,7 @@ router.post('/', async(req, res) => {
      'X-Jambonz-Routing': target.type,
      'X-Jambonz-FS-UUID': srf.locals.fsUUID,
      'X-Call-Sid': callSid,
-      'X-Account-Sid': req.body.account_sid
+      'X-Account-Sid': accountSid
    };

    switch (target.type) {
@@ -48,7 +50,7 @@ router.post('/', async(req, res) => {
        uri = `sip:${target.number}@${sbcAddress}`;
        to = target.number;
        if ('teams' === target.type) {
-          const obj = await lookupTeamsByAccount(req.body.account_sid);
+          const obj = await lookupTeamsByAccount(accountSid);
          if (!obj) throw new Error('dial to ms teams not allowed; account must first be configured with teams info');
          Object.assign(opts.headers, {
            'X-MS-Teams-FQDN': obj.ms_teams_fqdn,
@@ -131,8 +133,14 @@ router.post('/', async(req, res) => {
    try {
      const dlg = await srf.createUAC(uri, {...opts,  followRedirects: true, keepUriOnRedirect: true}, {
        cbRequest: (err, inviteReq) => {
-          /* in case of 302 redirect, this gets called twice, ignore the second */
-          if (res.headersSent) return;
+          /* in case of 302 redirect, this gets called twice, ignore the second
+              except to update the req so that it can later be canceled if need be
+          */
+          if (res.headersSent) {
+            logger.info(`create-call: got redirect, updating request to new call-id ${req.get('Call-ID')}`);
+            if (cs) cs.req = inviteReq;
+            return;
+          }

          if (err) {
            logger.error(err, 'createCall Error creating call');
@@ -140,9 +148,22 @@ router.post('/', async(req, res) => {
            return;
          }
          inviteReq.srf = srf;
+          inviteReq.locals = {
+            ...(inviteReq || {}),
+            callSid,
+            application_sid: app.application_sid
+          };
          /* ok our outbound INVITE is in flight */

          const tasks = [restDial];
+          const rootSpan = new RootSpan('rest-call', inviteReq);
+          sipLogger = logger.child({
+            callSid,
+            callId: inviteReq.get('Call-ID'),
+            accountSid,
+            traceId: rootSpan.traceId
+          });
+          app.requestor.logger = app.notifier.logger = sipLogger;
          const callInfo = new CallInfo({
            direction: CallDirection.Outbound,
            req: inviteReq,
@@ -150,17 +171,24 @@ router.post('/', async(req, res) => {
            tag: app.tag,
            callSid,
            accountSid: req.body.account_sid,
-            applicationSid: app.application_sid
+            applicationSid: app.application_sid,
+            traceId: rootSpan.traceId
+          });
+          cs = new RestCallSession({
+            logger: sipLogger,
+            application: app,
+            srf,
+            req: inviteReq,
+            ep,
+            tasks,
+            callInfo,
+            accountInfo,
+            rootSpan
          });
-          cs = new RestCallSession({logger, application: app, srf, req: inviteReq, ep, tasks, callInfo, accountInfo});
          cs.exec(req);

          res.status(201).json({sid: cs.callSid});

-          sipLogger = logger.child({
-            callSid: cs.callSid,
-            callId: callInfo.callId
-          });
          sipLogger.info(`outbound REST call attempt to ${JSON.stringify(target)} has been sent`);
        },
        cbProvisional: (prov) => {
--- a/lib/middleware.js
+++ b/lib/middleware.js
@@ -7,6 +7,8 @@ const makeTask = require('./tasks/make_task');
 const parseUri = require('drachtio-srf').parseUri;
 const normalizeJambones = require('./utils/normalize-jambones');
 const dbUtils = require('./utils/db-utils');
+const RootSpan = require('./utils/call-tracer');
+const listTaskNames = require('./utils/summarize-tasks');

 module.exports = function(srf, logger) {
  const {
@@ -17,15 +19,18 @@ module.exports = function(srf, logger) {
    lookupAppByTeamsTenant
  }  = srf.locals.dbHelpers;
  const {lookupAccountDetails} = dbUtils(logger, srf);
+
  function initLocals(req, res, next) {
+    if (!req.has('X-Account-Sid')) {
+      logger.info('getAccountDetails - rejecting call due to missing X-Account-Sid header');
+      return res.send(500);
+    }
    const callSid = req.has('X-Retain-Call-Sid') ? req.get('X-Retain-Call-Sid') : uuidv4();
-    req.locals = {
-      callSid,
-      logger: logger.child({callId: req.get('Call-ID'), callSid})
-    };
+    const account_sid = req.get('X-Account-Sid');
+    req.locals = {callSid, account_sid};
    if (req.has('X-Application-Sid')) {
      const application_sid = req.get('X-Application-Sid');
-      req.locals.logger.debug(`got application from X-Application-Sid header: ${application_sid}`);
+      logger.debug(`got application from X-Application-Sid header: ${application_sid}`);
      req.locals.application_sid = application_sid;
    }
    if (req.has('X-Authenticated-User')) req.locals.originatingUser = req.get('X-Authenticated-User');
@@ -34,19 +39,50 @@ module.exports = function(srf, logger) {
    next();
  }

+  function createRootSpan(req, res, next) {
+    const {callSid, account_sid} = req.locals;
+    const rootSpan = new RootSpan('incoming-call', req);
+    const traceId = rootSpan.traceId;
+
+    req.locals = {
+      ...req.locals,
+      traceId,
+      logger: logger.child({
+        callId: req.get('Call-ID'),
+        callSid,
+        accountSid: account_sid,
+        callingNumber: req.callingNumber,
+        calledNumber: req.calledNumber,
+        traceId}),
+      rootSpan
+    };
+
+    /**
+     * end the span on final failure or cancel from caller;
+     * otherwise it will be closed when sip dialog is destroyed
+    */
+    req.once('cancel', () => {
+      rootSpan.setAttributes({finalStatus: 487});
+      rootSpan.end();
+    });
+    res.once('finish', () => {
+      rootSpan.setAttributes({finalStatus: res.statusCode});
+      res.statusCode >= 300 && rootSpan.end();
+    });
+
+    next();
+  }
+
  /**
   * retrieve account information for the incoming call
   */
  async function getAccountDetails(req, res, next) {
+    const {rootSpan, account_sid} = req.locals;

-    if (!req.has('X-Account-Sid')) {
-      logger.info('getAccountDetails - rejecting call due to missing X-Account-Sid header');
-      return res.send(500);
-    }
-    const account_sid = req.locals.account_sid = req.get('X-Account-Sid');
-
+    const {span} = rootSpan.startChildSpan('lookupAccountDetails');
    try {
      req.locals.accountInfo = await lookupAccountDetails(account_sid);
+      span.end();
      if (!req.locals.accountInfo.account.is_active) {
        logger.info(`Account is inactive or suspended ${account_sid}`);
        // TODO: alert
@@ -55,6 +91,7 @@ module.exports = function(srf, logger) {
      logger.debug({accountInfo: req.locals?.accountInfo?.account}, `retrieved account info for ${account_sid}`);
      next();
    } catch (err) {
+      span.end();
      logger.info({err}, `Error retrieving account details for account ${account_sid}`);
      res.send(503, {headers: {'X-Reason': `No Account exists for sid ${account_sid}`}});
    }
@@ -86,7 +123,8 @@ module.exports = function(srf, logger) {
   */
  async function retrieveApplication(req, res, next) {
    const logger = req.locals.logger;
-    const {accountInfo, account_sid} = req.locals;
+    const {accountInfo, account_sid, rootSpan} = req.locals;
+    const {span} = rootSpan.startChildSpan('lookupApplication');
    try {
      let app;
      if (req.locals.application_sid) app = await lookupAppBySid(req.locals.application_sid);
@@ -130,6 +168,11 @@ module.exports = function(srf, logger) {
        }
      }

+      span.setAttributes({
+        'app.hook': app?.call_hook?.url,
+        'application_sid': req.locals.application_sid
+      });
+      span.end();
      if (!app || !app.call_hook || !app.call_hook.url) {
        logger.info(`rejecting call to ${req.locals.calledNumber}: no application or webhook url`);
        return res.send(480, {
@@ -163,9 +206,15 @@ module.exports = function(srf, logger) {
      // eslint-disable-next-line no-unused-vars
      const {call_hook, call_status_hook, ...appInfo} = obj;  // mask sensitive data like user/pass on webhook
      logger.info({app: appInfo}, `retrieved application for incoming call to ${req.locals.calledNumber}`);
-      req.locals.callInfo = new CallInfo({req, app, direction: CallDirection.Inbound});
+      req.locals.callInfo = new CallInfo({
+        req,
+        app,
+        direction: CallDirection.Inbound,
+        traceId: rootSpan.traceId
+      });
      next();
    } catch (err) {
+      span.end();
      logger.error(err, `${req.get('Call-ID')} Error looking up application for ${req.calledNumber}`);
      res.send(500);
    }
@@ -176,9 +225,9 @@ module.exports = function(srf, logger) {
   */
  async function invokeWebCallback(req, res, next) {
    const logger = req.locals.logger;
-    const app = req.locals.application;
+    const {rootSpan, application:app} = req.locals;
+    let span;
    try {
-
      if (app.tasks) {
        app.tasks = normalizeJambones(logger, app.tasks).map((tdata) => makeTask(logger, tdata));
        if (0 === app.tasks.length) throw new Error('no application provided');
@@ -186,12 +235,36 @@ module.exports = function(srf, logger) {
      }
      /* retrieve the application to execute for this inbound call */
      const params = Object.assign(['POST', 'WS'].includes(app.call_hook.method) ? {sip: req.msg} : {},
-        req.locals.callInfo);
-      const json = await app.requestor.request('session:new', app.call_hook, params);
+        req.locals.callInfo, {
+          defaults: {
+            synthesizer: {
+              vendor: app.speech_synthesis_vendor,
+              language: app.speech_synthesis_language,
+              voice: app.speech_synthesis_voice
+            },
+            recognizer: {
+              vendor: app.speech_recognizer_vendor,
+              language: app.speech_recognizer_language
+            }
+          }
+        });
+      logger.debug({params}, 'sending initial webhook');
+      const obj = rootSpan.startChildSpan('performAppWebhook');
+      span = obj.span;
+      const b3 = rootSpan.getTracingPropagation();
+      const httpHeaders = b3 && {b3};
+      const json = await app.requestor.request('session:new', app.call_hook, params, httpHeaders);
      app.tasks = normalizeJambones(logger, json).map((tdata) => makeTask(logger, tdata));
+      span.setAttributes({
+        'http.statusCode': 200,
+        'app.tasks': listTaskNames(app.tasks)
+      });
+      span.end();
      if (0 === app.tasks.length) throw new Error('no application provided');
      next();
    } catch (err) {
+      span?.setAttributes({webhookStatus: err.statusCode});
+      span?.end();
      logger.info({err}, `Error retrieving or parsing application: ${err?.message}`);
      res.send(480, {headers: {'X-Reason': err?.message || 'unknown'}});
      app.requestor.close();
@@ -200,6 +273,7 @@ module.exports = function(srf, logger) {

  return {
    initLocals,
+    createRootSpan,
    getAccountDetails,
    normalizeNumbers,
    retrieveApplication,
--- a/lib/session/adulting-call-session.js
+++ b/lib/session/adulting-call-session.js
@@ -8,14 +8,15 @@ const CallSession = require('./call-session');

 */
 class AdultingCallSession extends CallSession {
-  constructor({logger, application, singleDialer, tasks, callInfo, accountInfo}) {
+  constructor({logger, application, singleDialer, tasks, callInfo, accountInfo, rootSpan}) {
    super({
      logger,
      application,
      srf: singleDialer.dlg.srf,
      tasks,
      callInfo,
-      accountInfo
+      accountInfo,
+      rootSpan
    });
    this.sd = singleDialer;

--- a/lib/session/call-info.js
+++ b/lib/session/call-info.js
@@ -10,6 +10,7 @@ class CallInfo {
    let from ;
    let srf;
    this.direction = opts.direction;
+    this.traceId = opts.traceId;
    if (opts.req) {
      const u = opts.req.getParsedHeader('from');
      const uri = parseUri(u.uri);
@@ -114,6 +115,7 @@ class CallInfo {
      callStatus: this.callStatus,
      callerId: this.callerId,
      accountSid: this.accountSid,
+      traceId: this.traceId,
      applicationSid: this.applicationSid,
      fsSipAddress: this.localSipAddress
    };
--- a/lib/session/call-session.js
+++ b/lib/session/call-session.js
@@ -35,7 +35,7 @@ class CallSession extends Emitter {
   * @param {array} opts.tasks - tasks we are to execute
   * @param {callInfo} opts.callInfo - information about the call
   */
-  constructor({logger, application, srf, tasks, callInfo, accountInfo, memberId, confName, confUuid}) {
+  constructor({logger, application, srf, tasks, callInfo, accountInfo, rootSpan, memberId, confName, confUuid}) {
    super();
    this.logger = logger;
    this.application = application;
@@ -50,6 +50,9 @@ class CallSession extends Emitter {
    this.stackIdx = 0;
    this.callGone = false;
    this.notifiedComplete = false;
+    this.rootSpan = rootSpan;
+
+    assert(rootSpan);

    this.tmpFiles = new Set();

@@ -65,6 +68,7 @@ class CallSession extends Emitter {
    this._pool = srf.locals.dbHelpers.pool;

    this.requestor.on('command', this._onCommand.bind(this));
+    this.requestor.on('connection-dropped', this._onWsConnectionDropped.bind(this));
  }

  /**
@@ -226,26 +230,40 @@ class CallSession extends Emitter {
    return this.backgroundGatherTask;
  }

-  async enableBotMode(gather) {
+  get b3() {
+    return this.rootSpan?.getTracingPropagation();
+  }
+
+  async enableBotMode(gather, autoEnable) {
    try {
      const t = normalizeJambones(this.logger, [gather]);
      this.backgroundGatherTask = makeTask(this.logger, t[0]);
      this.backgroundGatherTask
        .on('dtmf', this._clearTasks.bind(this))
+        .on('vad', this._clearTasks.bind(this))
        .on('transcription', this._clearTasks.bind(this))
        .on('timeout', this._clearTasks.bind(this));
      this.logger.info({gather}, 'CallSession:enableBotMode - starting background gather');
      const resources = await this._evaluatePreconditions(this.backgroundGatherTask);
+      const {span, ctx} = this.rootSpan.startChildSpan(`background-gather:${this.backgroundGatherTask.summary}`);
+      this.backgroundGatherTask.span = span;
+      this.backgroundGatherTask.ctx = ctx;
      this.backgroundGatherTask.exec(this, resources)
        .then(() => {
          this.logger.info('CallSession:enableBotMode: gather completed');
          this.backgroundGatherTask && this.backgroundGatherTask.removeAllListeners();
+          this.backgroundGatherTask && this.backgroundGatherTask.span.end();
          this.backgroundGatherTask = null;
+          if (autoEnable && !this.callGone && !this._stopping) {
+            this.logger.info('CallSession:enableBotMode: restarting background gather');
+            setImmediate(() => this.enableBotMode(gather, true));
+          }
          return;
        })
        .catch((err) => {
          this.logger.info({err}, 'CallSession:enableBotMode: gather threw error');
          this.backgroundGatherTask && this.backgroundGatherTask.removeAllListeners();
+          this.backgroundGatherTask && this.backgroundGatherTask.span.end();
          this.backgroundGatherTask = null;
        });
    } catch (err) {
@@ -256,7 +274,7 @@ class CallSession extends Emitter {
    if (this.backgroundGatherTask) {
      try {
        this.backgroundGatherTask.removeAllListeners();
-        this.backgroundGatherTask.kill();
+        this.backgroundGatherTask.kill().catch((err) => {});
      } catch (err) {}
      this.backgroundGatherTask = null;
    }
@@ -313,7 +331,7 @@ class CallSession extends Emitter {
            speech_credential_sid: credential.speech_credential_sid,
            accessKeyId: credential.access_key_id,
            secretAccessKey: credential.secret_access_key,
-            region: process.env.AWS_REGION || credential.aws_region
+            region: credential.aws_region || process.env.AWS_REGION
          };
        }
        else if ('microsoft' === vendor) {
@@ -362,11 +380,16 @@ class CallSession extends Emitter {
          this.backgroundGatherTask.updateTimeout(timeout);
        }
        else {
+          const {span, ctx} = this.rootSpan.startChildSpan(`verb:${task.summary}`);
+          task.span = span;
+          task.ctx = ctx;
          await task.exec(this, resources);
+          task.span.end();
        }
        this.currentTask = null;
        this.logger.info(`CallSession:exec completed task #${stackNum}:${taskNum}: ${task.name}`);
      } catch (err) {
+        task.span?.end();
        this.currentTask = null;
        if (err.message?.includes(BADPRECONDITIONS)) {
          this.logger.info(`CallSession:exec task #${stackNum}:${taskNum}: ${task.name}: ${err.message}`);
@@ -378,10 +401,19 @@ class CallSession extends Emitter {
      }

      if (0 === this.tasks.length && this.hasStableDialog && this.requestor instanceof WsRequestor) {
+        let span;
        try {
-          await this._awaitCommandsOrHangup();
+          const {span} = this.rootSpan.startChildSpan('waiting for commands');
+          const {reason, queue, command} = await this._awaitCommandsOrHangup();
+          span.setAttributes({
+            'completion.reason': reason,
+            'async.request.queue': queue,
+            'async.request.command': command
+          });
+          span.end();
          if (!this.hasStableDialog || this.callGone) break;
        } catch (err) {
+          span.end();
          this.logger.info(err, 'CallSession:exec - error waiting for new commands');
          break;
        }
@@ -390,6 +422,8 @@ class CallSession extends Emitter {

    // all done - cleanup
    this.logger.info('CallSession:exec all tasks complete');
+    this._stopping = true;
+    this.disableBotMode();
    this._onTasksDone();
    this._clearResources();

@@ -438,17 +472,15 @@ class CallSession extends Emitter {
   * this is called to clean up when the call is released from one side or another
   */
  _callReleased() {
-    this.logger.debug('CallSession:_callReleased - caller hung up');
    this.callGone = true;
    if (this.currentTask) {
      this.currentTask.kill(this);
      this.currentTask = null;
    }
    if (this.wakeupResolver) {
-      this.wakeupResolver();
+      this.wakeupResolver({reason: 'session ended'});
      this.wakeupResolver = null;
    }
-    this.requestor && this.requestor.close();
  }

  /**
@@ -486,17 +518,20 @@ class CallSession extends Emitter {
  async _lccCallHook(opts) {
    const webhooks = [];
    let sd, tasks, childTasks;
+    const b3 = this.b3;
+    const httpHeaders = b3 && {b3};

    if (opts.call_hook || opts.child_call_hook) {
      if (opts.call_hook) {
-        webhooks.push(this.requestor.request('session:redirect', opts.call_hook, this.callInfo.toJSON()));
+        webhooks.push(this.requestor.request('session:redirect', opts.call_hook, this.callInfo.toJSON(), httpHeaders));
      }
      if (opts.child_call_hook) {
        /* child call hook only allowed from a connected Dial state */
        const task = this.currentTask;
        sd = task.sd;
        if (task && TaskName.Dial === task.name && sd) {
-          webhooks.push(this.requestor.request('session:redirect', opts.child_call_hook, sd.callInfo.toJSON()));
+          webhooks.push(this.requestor.request(
+            'session:redirect', opts.child_call_hook, sd.callInfo.toJSON(), httpHeaders));
        }
      }
      const [tasks1, tasks2] = await Promise.all(webhooks);
@@ -584,7 +619,7 @@ class CallSession extends Emitter {
    task.doConferenceMuteNonModerators(this, opts);
  }

-  async _lccSipRequest(opts) {
+  async _lccSipRequest(opts, callSid) {
    const {sip_request} = opts;
    const {method, content_type, content, headers = {}} = sip_request;
    if (!this.hasStableDialog) {
@@ -592,13 +627,14 @@ class CallSession extends Emitter {
      return;
    }
    try {
-      const res = await this.dlg.request({
+      const dlg = callSid === this.callSid ? this.dlg : this.currentTask.dlg;
+      const res = await dlg.request({
        method,
        headers: {
          ...headers,
-          'Content-Type': content_type,
-          'Content': content
-        }
+          'Content-Type': content_type
+        },
+        body: content
      });
      this.logger.debug({res}, `CallSession:_lccSipRequest got response to ${method}`);
      return res;
@@ -614,6 +650,8 @@ class CallSession extends Emitter {
  async _lccWhisper(opts, callSid) {
    const {whisper} = opts;
    let tasks;
+    const b3 = this.b3;
+    const httpHeaders = b3 && {b3};

    // this whole thing requires us to be in a Dial verb
    const task = this.currentTask;
@@ -624,7 +662,7 @@ class CallSession extends Emitter {
    // allow user to provide a url object, a url string, an array of tasks, or a single task
    if (typeof whisper === 'string' || (typeof whisper === 'object' && whisper.url)) {
      // retrieve a url
-      const json = await this.requestor(opts.call_hook, this.callInfo.toJSON());
+      const json = await this.requestor(opts.call_hook, this.callInfo.toJSON(), httpHeaders);
      tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
    }
    else if (Array.isArray(whisper)) {
@@ -683,7 +721,7 @@ class CallSession extends Emitter {
      await this._lccConfMuteStatus(opts);
    }
    else if (opts.sip_request) {
-      const res = await this._lccSipRequest(opts);
+      const res = await this._lccSipRequest(opts, callSid);
      return {status: res.status, reason: res.reason};
    }

@@ -725,8 +763,52 @@ class CallSession extends Emitter {
    this.taskIdx = 0;
  }

-  _onCommand({msgid, command, queueCommand, data}) {
+  /**
+   * Append tasks to the current execution stack UNLESS there is a gather in the stack.
+   * in that case, insert the tasks before the gather AND if the tasks include
+   * a gather then delete/remove the gather from the existing stack
+   * @param {*} t array of tasks
+   */
+  _injectTasks(newTasks) {
+    const gatherPos = this.tasks.map((t) => t.name).indexOf(TaskName.Gather);
+    const currentlyExecutingGather = this.currentTask?.name === TaskName.Gather;
+
+    this.logger.debug({
+      currentTaskList: listTaskNames(this.tasks),
+      newContent: listTaskNames(newTasks),
+      currentlyExecutingGather,
+      gatherPos
+    }, 'CallSession:_injectTasks - starting');
+
+    const killGather = () => {
+      this.logger.debug('CallSession:_injectTasks - killing current gather because we have new content');
+      this.currentTask.kill(this);
+    };
+
+    if (-1 === gatherPos) {
+      /* no gather in the stack  simply append tasks */
+      this.tasks.push(...newTasks);
+      this.logger.debug({
+        updatedTaskList: listTaskNames(this.tasks)
+      }, 'CallSession:_injectTasks - completed (simple append)');
+
+      /* we do need to kill the current gather if we are executing one */
+      if (currentlyExecutingGather) killGather();
+      return;
+    }
+
+    if (currentlyExecutingGather) killGather();
+    const newTasksHasGather = newTasks.find((t) => t.name === TaskName.Gather);
+    this.tasks.splice(gatherPos, newTasksHasGather ? 1 : 0, ...newTasks);
+
+    this.logger.debug({
+      updatedTaskList: listTaskNames(this.tasks)
+    }, 'CallSession:_injectTasks - completed');
+  }
+
+  _onCommand({msgid, command, call_sid, queueCommand, data}) {
    this.logger.info({msgid, command, queueCommand}, 'CallSession:_onCommand - received command');
+    const resolution = {reason: 'received command', queue: queueCommand, command};
    switch (command) {
      case 'redirect':
        if (Array.isArray(data)) {
@@ -736,11 +818,17 @@ class CallSession extends Emitter {
            this.logger.info({tasks: listTaskNames(t)}, 'CallSession:_onCommand new task list');
            this.replaceApplication(t);
          }
-          else {
-            this.logger.info({tasks: listTaskNames(t)}, 'CallSession:_onCommand - queueing tasks');
-            this.tasks.push(...t);
-            this.logger.debug({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
+          else if (process.env.JAMBONES_INJECT_CONTENT) {
+            this.logger.debug({tasks: listTaskNames(t)}, 'CallSession:_onCommand - queueing tasks (injecting content)');
+            this._injectTasks(t);
+            this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
          }
+          else {
+            this.logger.debug({tasks: listTaskNames(t)}, 'CallSession:_onCommand - queueing tasks');
+            this.tasks.push(...t);
+            this.logger.info({tasks: listTaskNames(this.tasks)}, 'CallSession:_onCommand - updated task list');
+          }
+          resolution.command = listTaskNames(t);
        }
        else this._lccCallHook(data);
        break;
@@ -750,7 +838,7 @@ class CallSession extends Emitter {
        break;

      case 'mute:status':
-        this._lccMuteStatus(this.callSid, data);
+        this._lccMuteStatus(call_sid, data);
        break;

      case 'conf:mute-status':
@@ -766,11 +854,11 @@ class CallSession extends Emitter {
        break;

      case 'whisper':
-        this._lccWhisper(data, this.callSid);
+        this._lccWhisper(data, call_sid);
        break;

      case 'sip:request':
-        this._lccSipRequest(data)
+        this._lccSipRequest(data, call_sid)
          .catch((err) => {
            this.logger.info({err, data}, `CallSession:_onCommand - error sending ${data.method}`);
          });
@@ -780,10 +868,24 @@ class CallSession extends Emitter {
        this.logger.info(`CallSession:_onCommand - invalid command ${command}`);
    }
    if (this.wakeupResolver) {
-      this.logger.info('CallSession:_onCommand - got commands, waking up..');
-      this.wakeupResolver();
+      this.logger.debug({resolution}, 'CallSession:_onCommand - got commands, waking up..');
+      this.wakeupResolver(resolution);
      this.wakeupResolver = null;
    }
+    else {
+      const {span} = this.rootSpan.startChildSpan('async command');
+      const {queue, command} = resolution;
+      span.setAttributes({
+        'async.request.queue': queue,
+        'async.request.command': command
+      });
+      span.end();
+    }
+  }
+
+  _onWsConnectionDropped() {
+    const {stats} = this.srf.locals;
+    stats.increment('app.hook.remote_close');
  }

  _evaluatePreconditions(task) {
@@ -927,6 +1029,8 @@ class CallSession extends Emitter {
    }
    this.tmpFiles.clear();
    this.requestor && this.requestor.close();
+
+    this.rootSpan && this.rootSpan.end();
  }

  /**
@@ -959,7 +1063,13 @@ class CallSession extends Emitter {
  async propagateAnswer() {
    if (!this.dlg) {
      assert(this.ep);
-      this.dlg = await this.srf.createUAS(this.req, this.res, {localSdp: this.ep.local.sdp});
+      this.dlg = await this.srf.createUAS(this.req, this.res, {
+        headers: {
+          'X-Trace-ID': this.req.locals.traceId,
+          'X-Call-Sid': this.req.locals.callSid
+        },
+        localSdp: this.ep.local.sdp
+      });
      this.logger.debug('answered call');
      this.dlg.on('destroy', this._callerHungup.bind(this));
      this.wrapDialog(this.dlg);
@@ -1147,9 +1257,10 @@ class CallSession extends Emitter {
        const duration = moment().diff(this.dlg.connectTime, 'seconds');
        this.emit('callStatusChange', {callStatus: CallStatus.Completed, duration});
        this.logger.debug('CallSession: call terminated by jambones');
+        this.rootSpan.setAttributes({'call.termination': 'hangup by jambonz'});
        origDestroy();
        if (this.wakeupResolver) {
-          this.wakeupResolver();
+          this.wakeupResolver({reason: 'session ended'});
          this.wakeupResolver = null;
        }
      }
@@ -1209,9 +1320,15 @@ class CallSession extends Emitter {

    this.callInfo.updateCallStatus(callStatus, sipStatus, sipReason);
    if (typeof duration === 'number') this.callInfo.duration = duration;
+    const {span} = this.rootSpan.startChildSpan(`call-status:${this.callInfo.callStatus}`);
+    span.setAttributes(this.callInfo.toJSON());
    try {
-      this.notifier.request('call:status', this.call_status_hook, this.callInfo.toJSON());
+      const b3 = this.b3;
+      const httpHeaders = b3 && {b3};
+      this.notifier.request('call:status', this.call_status_hook, this.callInfo.toJSON(), httpHeaders);
+      span.end();
    } catch (err) {
+      span.end();
      this.logger.info(err, `CallSession:_notifyCallStatusChange error sending ${callStatus} ${sipStatus}`);
    }

--- a/lib/session/confirm-call-session.js
+++ b/lib/session/confirm-call-session.js
@@ -8,7 +8,7 @@ const CallSession = require('./call-session');

 */
 class ConfirmCallSession extends CallSession {
-  constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName}) {
+  constructor({logger, application, dlg, ep, tasks, callInfo, accountInfo, memberId, confName, rootSpan}) {
    super({
      logger,
      application,
@@ -18,7 +18,8 @@ class ConfirmCallSession extends CallSession {
      callInfo,
      accountInfo,
      memberId,
-      confName
+      confName,
+      rootSpan
    });
    this.dlg = dlg;
    this.ep = ep;
--- a/lib/session/inbound-call-session.js
+++ b/lib/session/inbound-call-session.js
@@ -16,7 +16,8 @@ class InboundCallSession extends CallSession {
      application: req.locals.application,
      callInfo: req.locals.callInfo,
      accountInfo: req.locals.accountInfo,
-      tasks: req.locals.application.tasks
+      tasks: req.locals.application.tasks,
+      rootSpan: req.locals.rootSpan
    });
    this.req = req;
    this.res = res;
@@ -32,6 +33,7 @@ class InboundCallSession extends CallSession {
  }

  _onCancel() {
+    this.rootSpan.setAttributes({'call.termination': 'caller abandoned'});
    this._notifyCallStatusChange({
      callStatus: CallStatus.NoAnswer,
      sipStatus: 487,
@@ -43,6 +45,7 @@ class InboundCallSession extends CallSession {
  _onTasksDone() {
    if (!this.res.finalResponseSent) {
      if (this._mediaServerFailure) {
+        this.rootSpan.setAttributes({'call.termination': 'media server failure'});
        this.logger.info('InboundCallSession:_onTasksDone generating 480 due to media server failure');
        this.res.send(480, {
          headers: {
@@ -51,6 +54,7 @@ class InboundCallSession extends CallSession {
        });
      }
      else {
+        this.rootSpan.setAttributes({'call.termination': 'tasks completed without answering call'});
        this.logger.info('InboundCallSession:_onTasksDone auto-generating non-success response to invite');
        this.res.send(603);
      }
@@ -64,11 +68,12 @@ class InboundCallSession extends CallSession {
  _callerHungup() {
    assert(this.dlg.connectTime);
    const duration = moment().diff(this.dlg.connectTime, 'seconds');
+    this.rootSpan.setAttributes({'call.termination': 'hangup by caller'});
    this.emit('callStatusChange', {
      callStatus: CallStatus.Completed,
      duration
    });
-    this.logger.debug('InboundCallSession: caller hung up');
+    this.logger.info('InboundCallSession: caller hung up');
    this._callReleased();
    this.req.removeAllListeners('cancel');
  }
--- a/lib/session/rest-call-session.js
+++ b/lib/session/rest-call-session.js
@@ -8,7 +8,7 @@ const moment = require('moment');
 * @extends CallSession
 */
 class RestCallSession extends CallSession {
-  constructor({logger, application, srf, req, ep, tasks, callInfo, accountInfo}) {
+  constructor({logger, application, srf, req, ep, tasks, callInfo, accountInfo, rootSpan}) {
    super({
      logger,
      application,
@@ -16,7 +16,8 @@ class RestCallSession extends CallSession {
      callSid: callInfo.callSid,
      tasks,
      callInfo,
-      accountInfo
+      accountInfo,
+      rootSpan
    });
    this.req = req;
    this.ep = ep;
--- a/lib/tasks/conference.js
+++ b/lib/tasks/conference.js
@@ -529,7 +529,9 @@ class Conference extends Task {

  async _playHook(cs, dlg, hook, allowed = [TaskName.Play, TaskName.Say, TaskName.Pause]) {
    assert(!this._playSession);
-    const json = await cs.application.requestor.request('verb:hook', hook, cs.callInfo);
+    const b3 = this.getTracingPropagation();
+    const httpHeaders = b3 && {b3};
+    const json = await cs.application.requestor.request('verb:hook', hook, cs.callInfo, httpHeaders);
    const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));

    const allowedTasks = tasks.filter((t) => allowed.includes(t.name));
@@ -582,11 +584,14 @@ class Conference extends Task {

  _notifyConferenceEvent(cs, eventName, params = {}) {
    if (this.statusEvents.includes(eventName)) {
+      const b3 = this.getTracingPropagation();
+      const httpHeaders = b3 && {b3};
      params.event = eventName;
      params.duration = (Date.now() - this.conferenceStartTime.getTime()) / 1000;
      if (!params.time) params.time = (new Date()).toISOString();
      if (!params.members && typeof this.participantCount === 'number') params.members = this.participantCount;
-      cs.application.requestor.request('verb:hook', this.statusHook, Object.assign(params, this.statusParams))
+      cs.application.requestor
+        .request('verb:hook', this.statusHook, Object.assign(params, this.statusParams, httpHeaders))
        .catch((err) => this.logger.info(err, 'Conference:notifyConferenceEvent - error'));
    }
  }
--- a/lib/tasks/config.js
+++ b/lib/tasks/config.js
@@ -12,11 +12,12 @@ class TaskConfig extends Task {
      'bargeIn'
    ].forEach((k) => this[k] = this.data[k] || {});

-    if (this.hasBargeIn && this.bargeIn.enable === true) {
+    if (this.bargeIn.enable) {
      this.gatherOpts = {
        verb: 'gather',
        timeout: 0,
-        bargein: true
+        bargein: true,
+        input: ['speech']
      };
      [
        'finishOnKey', 'input', 'numDigits', 'minDigits', 'maxDigits',
@@ -25,7 +26,8 @@ class TaskConfig extends Task {
        if (this.bargeIn[k]) this.gatherOpts[k] = this.bargeIn[k];
      });
    }
-    this.preconditions = this.hasBargeIn ? TaskPreconditions.Endpoint : TaskPreconditions.None;
+    if (this.bargeIn.sticky) this.autoEnable = true;
+    this.preconditions = this.bargeIn.enable ? TaskPreconditions.Endpoint : TaskPreconditions.None;
  }

  get name() { return TaskName.Config; }
@@ -34,7 +36,21 @@ class TaskConfig extends Task {

  get hasRecognizer() { return Object.keys(this.recognizer).length; }

-  get hasBargeIn() { return Object.keys(this.bargeIn).length; }
+  get summary() {
+    const phrase = [];
+    if (this.bargeIn.enable) phrase.push('enable barge-in');
+    if (this.hasSynthesizer) {
+      const {vendor:v, language:l, voice} = this.synthesizer;
+      const s = `{${v},${l},${voice}}`;
+      phrase.push(`set synthesizer${s}`);
+    }
+    if (this.hasRecognizer) {
+      const {vendor:v, language:l} = this.recognizer;
+      const s = `{${v},${l}}`;
+      phrase.push(`set recognizer${s}`);
+    }
+    return  `${this.name}{${phrase.join(',')}`;
+  }

  async exec(cs) {
    await super.exec(cs);
@@ -60,13 +76,19 @@ class TaskConfig extends Task {
        : cs.speechRecognizerLanguage;
      this.logger.info({recognizer: this.recognizer}, 'Config: updated recognizer');
    }
-    if (this.hasBargeIn) {
+    if ('enable' in this.bargeIn) {
      if (this.gatherOpts) {
-        this.logger.debug({opts: this.gatherOpts}, 'Config: enabling bargeIn');
-        cs.enableBotMode(this.gatherOpts);
+        this.gatherOpts.recognizer = this.hasRecognizer ?
+          this.recognizer :
+          {
+            vendor: cs.speechRecognizerVendor,
+            language: cs.speechRecognizerLanguage
+          };
+        this.logger.info({opts: this.gatherOpts}, 'Config: enabling bargeIn');
+        cs.enableBotMode(this.gatherOpts, this.autoEnable);
      }
      else {
-        this.logger.debug('Config: disabling bargeIn');
+        this.logger.info('Config: disabling bargeIn');
        cs.disableBotMode();
      }
    }
--- a/lib/tasks/dial.js
+++ b/lib/tasks/dial.js
@@ -138,7 +138,20 @@ class TaskDial extends Task {
  }

  get summary() {
-    if (this.target.length === 1) return `${this.name}{type=${this.target[0].type}}`;
+    if (this.target.length === 1) {
+      const target = this.target[0];
+      switch (target.type) {
+        case 'phone':
+        case 'teams':
+          return `${this.name}{type=${target.type},number=${target.number}}`;
+        case 'user':
+          return `${this.name}{type=${target.type},name=${target.name}}`;
+        case 'sip':
+          return `${this.name}{type=${target.type},sipUri=${target.sipUri}}`;
+        default:
+          return `${this.name}`;
+      }
+    }
    else return `${this.name}{${this.target.length} targets}`;
  }

@@ -213,7 +226,11 @@ class TaskDial extends Task {
      this.logger.debug('Dial:whisper executing tasks');
      while (tasks.length && !cs.callGone) {
        const task = tasks.shift();
+        const {span, ctx} = this.startChildSpan(`whisper:${this.sayTask.summary}`);
+        task.span = span;
+        task.ctx = ctx;
        await task.exec(cs, callSid === this.callSid ? this.ep : this.epOther);
+        span.end();
      }
      this.logger.debug('Dial:whisper tasks complete');
      if (!cs.callGone && this.epOther) {
@@ -254,6 +271,9 @@ class TaskDial extends Task {
        const referring_call_sid = isChild ? callInfo.callSid : cs.callSid;
        const referred_call_sid = isChild ? callInfo.parentCallSid : this.sd.callSid;

+        const b3 = this.getTracingPropagation();
+        const httpHeaders = b3 && {b3};
+
        const to = parseUri(req.getParsedHeader('Refer-To').uri);
        const by = parseUri(req.getParsedHeader('Referred-By').uri);
        this.logger.info({to}, 'refer to parsed');
@@ -268,7 +288,7 @@ class TaskDial extends Task {
            referring_call_sid,
            referred_call_sid
          }
-        });
+        }, httpHeaders);
        res.send(202);
        this.logger.info('DialTask:handleRefer - sent 202 Accepted');
      } catch (err) {
@@ -328,8 +348,10 @@ class TaskDial extends Task {
    const key = arr[1];
    const match = dtmfDetector.keyPress(key);
    if (match) {
+      const b3 = this.getTracingPropagation();
+      const httpHeaders = b3 && {b3};
      this.logger.info({callSid}, `Dial:_onInfo triggered dtmf match: ${match}`);
-      requestor.request('verb:hook', this.dtmfHook, {dtmf: match, ...callInfo.toJSON()})
+      requestor.request('verb:hook', this.dtmfHook, {dtmf: match, ...callInfo.toJSON()}, httpHeaders)
        .catch((err) => this.logger.info(err, 'Dial:_onDtmf - error'));
    }
  }
@@ -384,10 +406,11 @@ class TaskDial extends Task {
      this._killOutdials();
    }, this.timeout * 1000);

+    this.span.setAttributes({'dial.target': JSON.stringify(this.target)});
    this.target.forEach(async(t) => {
      try {
-        t.url = t.url || this.confirmUrl;
-        t.method = t.method || this.confirmMethod || 'POST';
+        t.confirmHook = t.confirmHook || this.confirmHook;
+        //t.method = t.method || this.confirmMethod || 'POST';
        if (t.type === 'teams') t.teamsInfo = teamsInfo;
        if (t.type === 'user' && !t.name.includes('@') && !fqdn) {
          const user = t.name;
@@ -420,7 +443,9 @@ class TaskDial extends Task {
          target: t,
          opts,
          callInfo: cs.callInfo,
-          accountInfo: cs.accountInfo
+          accountInfo: cs.accountInfo,
+          rootSpan: cs.rootSpan,
+          startSpan: this.startSpan.bind(this)
        });
        this.dials.set(sd.callSid, sd);

--- a/lib/tasks/dialogflow/index.js
+++ b/lib/tasks/dialogflow/index.js
@@ -295,9 +295,9 @@ class Dialogflow extends Task {
    }

    // if a final transcription, start a typing sound
-    if (this.thinkingSound > 0 && !transcription.isEmpty && transcription.isFinal &&
+    if (this.thinkingMusic && !transcription.isEmpty && transcription.isFinal &&
      transcription.confidence > 0.8) {
-      ep.play(this.data.thinkingSound).catch((err) => this.logger.info(err, 'Error playing typing sound'));
+      ep.play(this.data.thinkingMusic).catch((err) => this.logger.info(err, 'Error playing typing sound'));
    }

    // interrupt playback on speaking if bargein = true
@@ -405,8 +405,8 @@ class Dialogflow extends Task {
    this.dtmfEntry = dtmfEntry;
    this.digitBuffer = null;
    // if a final transcription, start a typing sound
-    if (this.thinkingSound > 0) {
-      ep.play(this.thinkingSound).catch((err) => this.logger.info(err, 'Error playing typing sound'));
+    if (this.thinkingMusic) {
+      ep.play(this.thinkingMusic).catch((err) => this.logger.info(err, 'Error playing typing sound'));
    }

    // kill the current dialogflow, which will result in us getting an immediate intent
@@ -453,7 +453,10 @@ class Dialogflow extends Task {
  }

  async _performHook(cs, hook, results = {}) {
-    const json = await this.cs.requestor.request('verb:hook', hook, {...results, ...cs.callInfo.toJSON()});
+    const b3 = this.getTracingPropagation();
+    const httpHeaders = b3 && {b3};
+    const json = await this.cs.requestor.request('verb:hook', hook,
+      {...results, ...cs.callInfo.toJSON()}, httpHeaders);
    if (json && Array.isArray(json)) {
      const makeTask = require('../make_task');
      const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
--- a/lib/tasks/enqueue.js
+++ b/lib/tasks/enqueue.js
@@ -302,6 +302,8 @@ class TaskEnqueue extends Task {

  async _playHook(cs, dlg, hook, allowed = [TaskName.Play, TaskName.Say, TaskName.Pause, TaskName.Leave]) {
    const {lengthOfList, getListPosition} = cs.srf.locals.dbHelpers;
+    const b3 = this.getTracingPropagation();
+    const httpHeaders = b3 && {b3};

    assert(!this._playSession);
    if (this.killed) return [];
@@ -317,7 +319,7 @@ class TaskEnqueue extends Task {
    } catch (err) {
      this.logger.error({err}, `TaskEnqueue:_playHook error retrieving list info for queue ${this.queueName}`);
    }
-    const json = await cs.application.requestor.request('verb:hook', hook, params);
+    const json = await cs.application.requestor.request('verb:hook', hook, params, httpHeaders);
    const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));

    const allowedTasks = tasks.filter((t) => allowed.includes(t.name));
--- a/lib/tasks/gather.js
+++ b/lib/tasks/gather.js
@@ -37,6 +37,7 @@ class TaskGather extends Task {
      this.hints = recognizer.hints || [];
      this.hintsBoost = recognizer.hintsBoost;
      this.altLanguages = recognizer.altLanguages || [];
+      this.punctuation = !!recognizer.punctuation;

      /* vad: if provided, we dont connect to recognizer until voice activity is detected */
      const {enable, voiceMs = 0, mode = -1} = recognizer.vad || {};
@@ -58,8 +59,12 @@ class TaskGather extends Task {
    this.digitBuffer = '';
    this._earlyMedia = this.data.earlyMedia === true;

-    if (this.say) this.sayTask = makeTask(this.logger, {say: this.say}, this);
-    if (this.play) this.playTask = makeTask(this.logger, {play: this.play}, this);
+    if (this.say) {
+      this.sayTask = makeTask(this.logger, {say: this.say}, this);
+    }
+    if (this.play) {
+      this.playTask = makeTask(this.logger, {play: this.play}, this);
+    }
    if (!this.sayTask && !this.playTask) this.listenDuringPrompt = false;

    this.parentTask = parentTask;
@@ -124,16 +129,26 @@ class TaskGather extends Task {

    try {
      if (this.sayTask) {
+        const {span, ctx} = this.startChildSpan(`nested:${this.sayTask.summary}`);
+        this.sayTask.span = span;
+        this.sayTask.ctx = ctx;
        this.sayTask.exec(cs, ep);  // kicked off, _not_ waiting for it to complete
        this.sayTask.on('playDone', (err) => {
-          if (err) return this.logger.error({err}, 'Gather:exec Error playing tts');
-          this.logger.debug('Gather: say task completed');
+          span.end();
+          if (err) this.logger.error({err}, 'Gather:exec Error playing tts');
+          this.logger.debug('Gather: nested say task completed');
+          if (!this.killed) startListening(cs, ep);
        });
      }
      else if (this.playTask) {
+        const {span, ctx} = this.startChildSpan(`nested:${this.playTask.summary}`);
+        this.playTask.span = span;
+        this.playTask.ctx = ctx;
        this.playTask.exec(cs, ep);  // kicked off, _not_ waiting for it to complete
        this.playTask.on('playDone', (err) => {
-          if (err) return this.logger.error({err}, 'Gather:exec Error playing url');
+          span.end();
+          if (err) this.logger.error({err}, 'Gather:exec Error playing url');
+          this.logger.debug('Gather: nested play task completed');
          if (!this.killed) startListening(cs, ep);
        });
      }
@@ -156,9 +171,12 @@ class TaskGather extends Task {
    }
    ep.removeCustomEventListener(GoogleTranscriptionEvents.Transcription);
    ep.removeCustomEventListener(GoogleTranscriptionEvents.EndOfUtterance);
+    ep.removeCustomEventListener(GoogleTranscriptionEvents.VadDetected);
    ep.removeCustomEventListener(AwsTranscriptionEvents.Transcription);
+    ep.removeCustomEventListener(AwsTranscriptionEvents.VadDetected);
    ep.removeCustomEventListener(AzureTranscriptionEvents.Transcription);
    ep.removeCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected);
+    ep.removeCustomEventListener(AzureTranscriptionEvents.VadDetected);
  }

  kill(cs) {
@@ -166,11 +184,13 @@ class TaskGather extends Task {
    this._killAudio(cs);
    this.ep.removeAllListeners('dtmf');
    clearTimeout(this.interDigitTimer);
+    this.playTask?.span.end();
+    this.sayTask?.span.end();
    this._resolve('killed');
  }

  updateTimeout(timeout) {
-    this.logger.info(`TaskGather:updateTimout - updating timeout to ${timeout}`);
+    this.logger.info(`TaskGather:updateTimeout - updating timeout to ${timeout}`);
    this.timeout = timeout;
    this._startTimer();
  }
@@ -206,6 +226,7 @@ class TaskGather extends Task {
    if (this.vad?.enable) {
      opts.START_RECOGNIZING_ON_VAD = 1;
      if (this.vad.voiceMs) opts.RECOGNIZER_VAD_VOICE_MS = this.vad.voiceMs;
+      else opts.RECOGNIZER_VAD_VOICE_MS = 125;
      if (this.vad.mode >= 0 && this.vad.mode <= 3) opts.RECOGNIZER_VAD_MODE = this.vad.mode;
    }

@@ -214,7 +235,8 @@ class TaskGather extends Task {
      Object.assign(opts, {
        GOOGLE_SPEECH_USE_ENHANCED: true,
        GOOGLE_SPEECH_SINGLE_UTTERANCE: true,
-        GOOGLE_SPEECH_MODEL: 'command_and_search'
+        GOOGLE_SPEECH_MODEL: 'command_and_search',
+        GOOGLE_SPEECH_ENABLE_AUTOMATIC_PUNCTUATION: !!this.punctuation
      });
      if (this.hints && this.hints.length > 1) {
        opts.GOOGLE_SPEECH_HINTS = this.hints.map((h) => h.trim()).join(',');
@@ -230,6 +252,7 @@ class TaskGather extends Task {
      }
      ep.addCustomEventListener(GoogleTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
      ep.addCustomEventListener(GoogleTranscriptionEvents.EndOfUtterance, this._onEndOfUtterance.bind(this, cs, ep));
+      ep.addCustomEventListener(GoogleTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
    }
    else if (['aws', 'polly'].includes(this.vendor)) {
      if (this.vocabularyName) opts.AWS_VOCABULARY_NAME = this.vocabularyName;
@@ -245,6 +268,7 @@ class TaskGather extends Task {
        });
      }
      ep.addCustomEventListener(AwsTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
+      ep.addCustomEventListener(AwsTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
    }
    else if ('microsoft' === this.vendor) {
      if (this.sttCredentials) {
@@ -257,13 +281,15 @@ class TaskGather extends Task {
        opts.AZURE_SPEECH_HINTS = this.hints.map((h) => h.trim()).join(',');
      }
      if (this.requestSnr) opts.AZURE_REQUEST_SNR = 1;
-      if (this.profanityOption !== 'raw') opts.AZURE_PROFANITY_OPTION = this.profanityOption;
+      if (this.profanityOption && this.profanityOption !== 'raw') opts.AZURE_PROFANITY_OPTION = this.profanityOption;
      if (this.azureServiceEndpoint) opts.AZURE_SERVICE_ENDPOINT = this.azureServiceEndpoint;
      if (this.initialSpeechTimeoutMs > 0) opts.AZURE_INITIAL_SPEECH_TIMEOUT_MS = this.initialSpeechTimeoutMs;
+      else if (this.timeout === 0) opts.AZURE_INITIAL_SPEECH_TIMEOUT_MS = 120000;  // lengthy
      opts.AZURE_USE_OUTPUT_FORMAT_DETAILED = 1;

      ep.addCustomEventListener(AzureTranscriptionEvents.Transcription, this._onTranscription.bind(this, cs, ep));
      ep.addCustomEventListener(AzureTranscriptionEvents.NoSpeechDetected, this._onNoSpeechDetected.bind(this, cs, ep));
+      ep.addCustomEventListener(AzureTranscriptionEvents.VadDetected, this._onVadDetected.bind(this, cs, ep));
    }
    await ep.set(opts)
      .catch((err) => this.logger.info(err, 'Error setting channel variables'));
@@ -293,6 +319,10 @@ class TaskGather extends Task {

  _startTimer() {
    if (0 === this.timeout) return;
+    if (this._timeoutTimer) {
+      clearTimeout(this._timeoutTimer);
+      this._timeoutTimer = null;
+    }
    assert(!this._timeoutTimer);
    this.logger.debug(`Gather:_startTimer: timeout ${this.timeout}`);
    this._timeoutTimer = setTimeout(() => this._resolve('timeout'), this.timeout);
@@ -332,6 +362,8 @@ class TaskGather extends Task {
    if ('microsoft' === this.vendor) {
      const final = evt.RecognitionStatus === 'Success';
      if (final) {
+        // don't sort based on confidence: https://github.com/Azure-Samples/cognitive-services-speech-sdk/issues/1463
+        //const nbest = evt.NBest.sort((a, b) => b.Confidence - a.Confidence);
        const nbest = evt.NBest;
        evt = {
          is_final: true,
@@ -354,7 +386,13 @@ class TaskGather extends Task {
        };
      }
    }
-    if (evt.is_final) this._resolve('speech', evt);
+    if (evt.is_final) {
+      if (evt.alternatives[0].transcript === '' && !this.callSession.callGone && !this.killed) {
+        this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, listen again');
+        return this._startTranscribing(ep);
+      }
+      this._resolve('speech', evt);
+    }
    else {
      /* google has a measure of stability:
        https://cloud.google.com/speech-to-text/docs/basics#streaming_responses
@@ -370,12 +408,15 @@ class TaskGather extends Task {
        this._killAudio(cs);
      }
      if (this.partialResultHook) {
-        this.cs.requestor.request(this.partialResultHook,  Object.assign({speech: evt}, this.cs.callInfo));
+        const b3 = this.getTracingPropagation();
+        const httpHeaders = b3 && {b3};
+        this.cs.requestor.request(this.partialResultHook,  Object.assign({speech: evt},
+          this.cs.callInfo, httpHeaders));
      }
    }
  }
  _onEndOfUtterance(cs, ep) {
-    this.logger.info('TaskGather:_onEndOfUtterance');
+    this.logger.debug('TaskGather:_onEndOfUtterance');
    if (this.bargein && this.minBargeinWordCount === 0) {
      this._killAudio(cs);
    }
@@ -385,22 +426,42 @@ class TaskGather extends Task {
    }
  }

+  _onVadDetected(cs, ep) {
+    if (this.bargein && this.minBargeinWordCount === 0) {
+      this.logger.debug('TaskGather:_onVadDetected');
+      this._killAudio(cs);
+      this.emit('vad');
+    }
+  }
+
  _onNoSpeechDetected(cs, ep) {
-    this._resolve('timeout');
+    if (!this.callSession.callGone && !this.killed) {
+      this.logger.debug('TaskGather:_onNoSpeechDetected - listen again');
+      return this._startTranscribing(ep);
+    }
  }

  async _resolve(reason, evt) {
-    if (this.resolved) return;
-    this.resolved = true;
    this.logger.debug(`TaskGather:resolve with reason ${reason}`);
+    if (this.resolved) return;
+
+    this.resolved = true;
    clearTimeout(this.interDigitTimer);

+    this.span.setAttributes({'stt.resolve': reason, 'stt.result': JSON.stringify(evt)});
    if (this.ep && this.ep.connected) {
      this.ep.stopTranscription({vendor: this.vendor})
        .catch((err) => this.logger.error({err}, 'Error stopping transcription'));
    }

    this._clearTimer();
+
+    if (this.callSession && this.callSession.callGone) {
+      this.logger.debug('TaskGather:_resolve - call is gone, not invoking web callback');
+      this.notifyTaskDone();
+      return;
+    }
+
    if (reason.startsWith('dtmf')) {
      if (this.parentTask) this.parentTask.emit('dtmf', evt);
      else {
--- a/lib/tasks/lex.js
+++ b/lib/tasks/lex.js
@@ -289,7 +289,9 @@ class Lex extends Task {
  }

  async _performHook(cs, hook, results) {
-    const json = await this.cs.requestor.request('verb:hook', hook, results);
+    const b3 = this.getTracingPropagation();
+    const httpHeaders = b3 && {b3};
+    const json = await this.cs.requestor.request('verb:hook', hook, results, httpHeaders);
    if (json && Array.isArray(json)) {
      const makeTask = require('./make_task');
      const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
--- a/lib/tasks/listen.js
+++ b/lib/tasks/listen.js
@@ -38,7 +38,12 @@ class TaskListen extends Task {
      if (this.playBeep) await this._playBeep(ep);
      if (this.transcribeTask) {
        this.logger.debug('TaskListen:exec - starting nested transcribe task');
-        this.transcribeTask.exec(cs, ep);
+        const {span, ctx} = this.startChildSpan(`nested:${this.transcribeTask.summary}`);
+        this.transcribeTask.span = span;
+        this.transcribeTask.ctx = ctx;
+        this.transcribeTask.exec(cs, ep)
+          .then((result) => span.end())
+          .catch((err) => span.end());
      }
      await this._startListening(cs, ep);
      await this.awaitTaskDone();
--- a/lib/tasks/play.js
+++ b/lib/tasks/play.js
@@ -13,6 +13,10 @@ class TaskPlay extends Task {

  get name() { return TaskName.Play; }

+  get summary() {
+    return `${this.name}:{url=${this.url}}`;
+  }
+
  async exec(cs, ep) {
    await super.exec(cs);
    this.ep = ep;
--- a/lib/tasks/rasa.js
+++ b/lib/tasks/rasa.js
@@ -31,8 +31,15 @@ class Rasa extends Task {

      /* start the first gather */
      this.gatherTask = this._makeGatherTask(this.prompt);
+      const {span, ctx} = this.startChildSpan(`nested:${this.gatherTask.summary}`);
+      this.gatherTask.span = span;
+      this.gatherTask.ctx = ctx;
      this.gatherTask.exec(cs, ep, this)
-        .catch((err) => this.logger.info({err}, 'Rasa gather task returned error'));
+        .then(() => span.end())
+        .catch((err) => {
+          span.end();
+          this.logger.info({err}, 'Rasa gather task returned error');
+        });

      await this.awaitTaskDone();
    } catch (err) {
@@ -118,8 +125,15 @@ class Rasa extends Task {
      if (botUtterance) {
        this.logger.debug({botUtterance}, 'Rasa:_onTranscription: got user utterance');
        this.gatherTask = this._makeGatherTask(botUtterance);
+        const {span, ctx} = this.startChildSpan(`nested:${this.gatherTask.summary}`);
+        this.gatherTask.span = span;
+        this.gatherTask.ctx = ctx;
        this.gatherTask.exec(cs, ep, this)
-          .catch((err) => this.logger.info({err}, 'Rasa gather task returned error'));
+          .then(() => span.end())
+          .catch((err) => {
+            span.end();
+            this.logger.info({err}, 'Rasa gather task returned error');
+          });
        if (this.eventHook) {
          this.performHook(cs, this.eventHook, {event: 'botMessage', message: response})
            .then((redirected) => {
--- a/lib/tasks/rest_dial.js
+++ b/lib/tasks/rest_dial.js
@@ -48,7 +48,9 @@ class TaskRestDial extends Task {
    cs.setDialog(dlg);

    try {
-      const tasks = await cs.requestor.request('verb:hook', this.call_hook, cs.callInfo);
+      const b3 = this.getTracingPropagation();
+      const httpHeaders = b3 && {b3};
+      const tasks = await cs.requestor.request('verb:hook', this.call_hook, cs.callInfo, httpHeaders);
      if (tasks && Array.isArray(tasks)) {
        this.logger.debug({tasks: tasks}, `TaskRestDial: replacing application with ${tasks.length} tasks`);
        cs.replaceApplication(normalizeJambones(this.logger, tasks).map((tdata) => makeTask(this.logger, tdata)));
--- a/lib/tasks/say.js
+++ b/lib/tasks/say.js
@@ -19,7 +19,7 @@ class TaskSay extends Task {
      if (this.text[i].startsWith('silence_stream')) continue;
      return `${this.name}{text=${this.text[i].slice(0, 15)}${this.text[i].length > 15 ? '...' : ''}}`;
    }
-    return this.text[0];
+    return `${this.name}{${this.text[0]}}`;
  }

  async exec(cs, ep) {
@@ -51,45 +51,64 @@ class TaskSay extends Task {
          alert_type: AlertType.TTS_NOT_PROVISIONED,
          vendor
        }).catch((err) => this.logger.info({err}, 'Error generating alert for no tts'));
+        this.notifyError(`No speech credentials have been provisioned for ${vendor}`);
        throw new Error('no provisioned speech credentials for TTS');
      }
      // synthesize all of the text elements
      let lastUpdated = false;
-      const filepath = (await Promise.all(this.text.map(async(text) => {
+
+      /* produce an audio segment from the provided text */
+      const generateAudio = async(text) => {
        if (this.killed) return;
        if (text.startsWith('silence_stream://')) return text;
-        const {filePath, servedFromCache} = await synthAudio(stats, {
-          text,
-          vendor,
-          language,
-          voice,
-          engine,
-          salt,
-          credentials
-        }).catch((err) => {
-          this.logger.info(err, 'Error synthesizing tts');
+
+        /* otel: trace time for tts */
+        const {span} = this.startChildSpan('tts-generation', {
+          'tts.vendor': vendor,
+          'tts.language': language,
+          'tts.voice': voice
+        });
+        try {
+          const {filePath, servedFromCache} = await synthAudio(stats, {
+            text,
+            vendor,
+            language,
+            voice,
+            engine,
+            salt,
+            credentials
+          });
+          this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
+          if (filePath) cs.trackTmpFile(filePath);
+          if (!servedFromCache && !lastUpdated) {
+            lastUpdated = true;
+            updateSpeechCredentialLastUsed(credentials.speech_credential_sid)
+              .catch(() => {/*already logged error */});
+          }
+          span.setAttributes({'tts.cached': servedFromCache});
+          span.end();
+          return filePath;
+        } catch (err) {
+          this.logger.info({err}, 'Error synthesizing tts');
+          span.end();
          writeAlerts({
            account_sid: cs.accountSid,
            alert_type: AlertType.TTS_NOT_PROVISIONED,
            vendor,
            detail: err.message
-          });
-        }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
-        this.logger.debug(`file ${filePath}, served from cache ${servedFromCache}`);
-        if (filePath) cs.trackTmpFile(filePath);
-        if (!servedFromCache && !lastUpdated) {
-          lastUpdated = true;
-          updateSpeechCredentialLastUsed(credentials.speech_credential_sid)
-            .catch(() => {/*already logged error */});
+          }).catch((err) => this.logger.info({err}, 'Error generating alert for tts failure'));
+          this.notifyError(err.message || err);
+          return;
        }
-        return filePath;
-      }))).filter((fp) => fp && fp.length);
+      };

+      const arr = this.text.map((t) => generateAudio(t));
+      const filepath = (await Promise.all(arr)).filter((fp) => fp && fp.length);
      this.logger.debug({filepath}, 'synthesized files for tts');

      while (!this.killed && (this.loop === 'forever' || this.loop--) && this.ep?.connected) {
        let segment = 0;
-        do {
+        while (!this.killed && segment < filepath.length) {
          if (cs.isInConference) {
            const {memberId, confName, confUuid} = cs;
            await this.playToConfMember(this.ep, memberId, confName, confUuid, filepath[segment]);
@@ -99,7 +118,8 @@ class TaskSay extends Task {
            await ep.play(filepath[segment]);
            this.logger.debug(`Say:exec completed play file ${filepath[segment]}`);
          }
-        } while (!this.killed && ++segment < filepath.length);
+          segment++;
+        }
      }
    } catch (err) {
      this.logger.info(err, 'TaskSay:exec error');
--- a/lib/tasks/sip_refer.js
+++ b/lib/tasks/sip_refer.js
@@ -26,6 +26,12 @@ class TaskSipRefer extends Task {
    try {
      this.notifyHandler = this._handleNotify.bind(this, cs, dlg);
      dlg.on('notify', this.notifyHandler);
+      /* otel: trace time for tts */
+      this.referSpan = this.startSpan('send-refer', {
+        'refer.refer_to': referTo,
+        'refer.referred_by': referredBy
+      });
+
      const response = await dlg.request({
        method: 'REFER',
        headers: {
@@ -35,16 +41,20 @@ class TaskSipRefer extends Task {
        }
      });
      this.referStatus = response.status;
+      this.referSpan.setAttributes({'refer.status_code': response.status});
      this.logger.info(`TaskSipRefer:exec - received ${this.referStatus} to REFER`);

      /* if we fail, fall through to next verb.  If success, we should get BYE from far end */
      if (this.referStatus === 202) {
        await this.awaitTaskDone();
      }
-      else await this.performAction({refer_status: this.referStatus});
+      else {
+        await this.performAction({refer_status: this.referStatus});
+      }
    } catch (err) {
      this.logger.info({err}, 'TaskSipRefer:exec - error sending REFER');
    }
+    this.referSpan?.end();
  }

  async kill(cs) {
@@ -66,9 +76,13 @@ class TaskSipRefer extends Task {
        const status = arr[1];
        this.logger.debug(`TaskSipRefer:_handleNotify: call got status ${status}`);
        if (this.eventHook) {
-          await cs.requestor.request('verb:hook', this.eventHook, {event: 'transfer-status', call_status: status});
+          const b3 = this.getTracingPropagation();
+          const httpHeaders = b3 && {b3};
+          await cs.requestor.request('verb:hook', this.eventHook,
+            {event: 'transfer-status', call_status: status}, httpHeaders);
        }
        if (status >= 200) {
+          this.referSpan.setAttributes({'refer.finalNotify': status});
          await this.performAction({refer_status: 202, final_referred_call_status: status});
          this.notifyTaskDone();
        }
--- a/lib/tasks/specs.json
+++ b/lib/tasks/specs.json
@@ -32,6 +32,7 @@
  "bargeIn": {
    "properties": {
      "enable": "boolean",
+      "sticky": "boolean",
      "actionHook": "object|string",
      "input": "array",
      "finishOnKey": "string",
--- a/lib/tasks/task.js
+++ b/lib/tasks/task.js
@@ -4,6 +4,7 @@ const debug = require('debug')('jambonz:feature-server');
 const assert = require('assert');
 const {TaskPreconditions} = require('../utils/constants');
 const normalizeJambones = require('../utils/normalize-jambones');
+const {trace} = require('@opentelemetry/api');
 const specs = new Map();
 const _specData = require('./specs');
 for (const key in _specData) {specs.set(key, _specData[key]);}
@@ -71,6 +72,34 @@ class Task extends Emitter {
    setImmediate(() => this.parentTask = null);
  }

+  startSpan(name, attributes) {
+    const {srf} = require('../..');
+    const {tracer} = srf.locals.otel;
+    const span = tracer.startSpan(name, undefined, this.ctx);
+    if (attributes) span.setAttributes(attributes);
+    trace.setSpan(this.ctx, span);
+    return span;
+  }
+
+  startChildSpan(name, attributes) {
+    const {srf} = require('../..');
+    const {tracer} = srf.locals.otel;
+    const span = tracer.startSpan(name, undefined, this.ctx);
+    if (attributes) span.setAttributes(attributes);
+    const ctx = trace.setSpan(this.ctx, span);
+    return {span, ctx};
+  }
+
+  getTracingPropagation(encoding, span) {
+    // TODO: support encodings beyond b3 https://github.com/openzipkin/b3-propagation
+    if (span) {
+      return `${span.spanContext().traceId}-${span.spanContext().spanId}-1`;
+    }
+    if (this.span) {
+      return `${this.span.spanContext().traceId}-${this.span.spanContext().spanId}-1`;
+    }
+  }
+
  /**
   * when a subclass Task has completed its work, it should call this method
   */
@@ -108,32 +137,62 @@ class Task extends Emitter {
    return this.callSession.normalizeUrl(url, method, auth);
  }

+  notifyError(errMsg) {
+    const params = {error: errMsg, verb: this.name};
+    this.cs.requestor.request('jambonz:error', '/error', params)
+      .catch((err) => this.logger.info({err}, 'Task:notifyError error sending error'));
+  }
+
  async performAction(results, expectResponse = true) {
    if (this.actionHook) {
      const params = results ? Object.assign(results, this.cs.callInfo.toJSON()) : this.cs.callInfo.toJSON();
-      const json = await this.cs.requestor.request('verb:hook', this.actionHook, params);
-      if (expectResponse && json && Array.isArray(json)) {
-        const makeTask = require('./make_task');
-        const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
-        if (tasks && tasks.length > 0) {
-          this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
-          this.callSession.replaceApplication(tasks);
+      const span = this.startSpan('verb:hook', {'hook.url': this.actionHook});
+      const b3 = this.getTracingPropagation('b3', span);
+      const httpHeaders = b3 && {b3};
+      span.setAttributes({'http.body': JSON.stringify(params)});
+      try {
+        const json = await this.cs.requestor.request('verb:hook', this.actionHook, params, httpHeaders);
+        span.setAttributes({'http.statusCode': 200});
+        span.end();
+        if (expectResponse && json && Array.isArray(json)) {
+          const makeTask = require('./make_task');
+          const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
+          if (tasks && tasks.length > 0) {
+            this.logger.info({tasks: tasks}, `${this.name} replacing application with ${tasks.length} tasks`);
+            this.callSession.replaceApplication(tasks);
+          }
        }
+      } catch (err) {
+        span.setAttributes({'http.statusCode': err.statusCode});
+        span.end();
+        throw err;
      }
    }
  }

  async performHook(cs, hook, results) {
-    const json = await cs.requestor.request('verb:hook', hook, results);
-    if (json && Array.isArray(json)) {
-      const makeTask = require('./make_task');
-      const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
-      if (tasks && tasks.length > 0) {
-        this.redirect(cs, tasks);
-        return true;
+    const span = this.startSpan('verb:hook', {'hook.url': hook});
+    const b3 = this.getTracingPropagation('b3', span);
+    const httpHeaders = b3 && {b3};
+    span.setAttributes({'http.body': JSON.stringify(results)});
+    try {
+      const json = await cs.requestor.request('verb:hook', hook, results, httpHeaders);
+      span.setAttributes({'http.statusCode': 200});
+      span.end();
+      if (json && Array.isArray(json)) {
+        const makeTask = require('./make_task');
+        const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
+        if (tasks && tasks.length > 0) {
+          this.redirect(cs, tasks);
+          return true;
+        }
      }
+      return false;
+    } catch (err) {
+      span.setAttributes({'http.statusCode': err.statusCode});
+      span.end();
+      throw err;
    }
-    return false;
  }

  redirect(cs, tasks) {
--- a/lib/tasks/transcribe.js
+++ b/lib/tasks/transcribe.js
@@ -253,8 +253,16 @@ class TaskTranscribe extends Task {
      evt = newEvent;
    }

+    if (evt.alternatives[0].transcript === '' && !cs.callGone && !this.killed) {
+      this.logger.info({evt}, 'TaskGather:_onTranscription - got empty transcript, listen again');
+      return this._transcribe(ep);
+    }
+
    if (this.transcriptionHook) {
-      this.cs.requestor.request('verb:hook', this.transcriptionHook, Object.assign({speech: evt}, this.cs.callInfo))
+      const b3 = this.getTracingPropagation();
+      const httpHeaders = b3 && {b3};
+      this.cs.requestor.request('verb:hook', this.transcriptionHook,
+        Object.assign({speech: evt}, this.cs.callInfo), httpHeaders)
        .catch((err) => this.logger.info(err, 'TranscribeTask:_onTranscription error'));
    }
    if (this.parentTask) {
--- a/lib/utils/call-tracer.js
+++ b/lib/utils/call-tracer.js
@@ -0,0 +1,78 @@
+const {context, trace} = require('@opentelemetry/api');
+const {Dialog} = require('drachtio-srf');
+class RootSpan {
+  constructor(callType, req) {
+    let tracer, callSid, linkedSpanId;
+
+    if (req instanceof Dialog) {
+      const dlg = req;
+      tracer = dlg.srf.locals.otel.tracer;
+      callSid = dlg.callSid;
+      linkedSpanId = dlg.linkedSpanId;
+    }
+    else {
+      tracer = req.srf.locals.otel.tracer;
+      callSid = req.locals.callSid;
+    }
+    this._span = tracer.startSpan(callType || 'incoming-call');
+    if (req instanceof Dialog) {
+      const dlg = req;
+      this._span.setAttributes({
+        linkedSpanId,
+        callId: dlg.sip.callId
+      });
+    }
+    else {
+      this._span.setAttributes({
+        callSid,
+        accountSid: req.get('X-Account-Sid'),
+        applicationSid: req.locals.application_sid,
+        callId: req.get('Call-ID'),
+        externalCallId: req.get('X-CID')
+      });
+    }
+
+    this._ctx = trace.setSpan(context.active(), this._span);
+    this.tracer = tracer;
+  }
+
+  get context() {
+    return this._ctx;
+  }
+
+  get traceId() {
+    return this._span.spanContext().traceId;
+  }
+
+  get spanId() {
+    return this._span.spanContext().spanId;
+  }
+
+  get traceFlags() {
+    return this._span.spanContext().traceFlags;
+  }
+
+  getTracingPropagation(encoding) {
+    // TODO: support encodings beyond b3 https://github.com/openzipkin/b3-propagation
+    if (this._span  && this.traceId !== '00000000000000000000000000000000') {
+      return `${this.traceId}-${this.spanId}-1`;
+    }
+  }
+
+  setAttributes(attrs) {
+    this._span.setAttributes(attrs);
+  }
+
+  end() {
+    this._span.end();
+  }
+
+  startChildSpan(name, attributes) {
+    const span = this.tracer.startSpan(name, attributes, this._ctx);
+    const ctx = trace.setSpan(context.active(), span);
+    return {span, ctx};
+  }
+}
+
+module.exports = RootSpan;
+
--- a/lib/utils/constants.json
+++ b/lib/utils/constants.json
@@ -59,19 +59,22 @@
    "Transcription": "google_transcribe::transcription",
    "EndOfUtterance": "google_transcribe::end_of_utterance",
    "NoAudioDetected": "google_transcribe::no_audio_detected",
-    "MaxDurationExceeded": "google_transcribe::max_duration_exceeded"
+    "MaxDurationExceeded": "google_transcribe::max_duration_exceeded",
+    "VadDetected": "google_transcribe::vad_detected"
  },
  "AwsTranscriptionEvents": {
    "Transcription": "aws_transcribe::transcription",
    "EndOfTranscript": "aws_transcribe::end_of_transcript",
    "NoAudioDetected": "aws_transcribe::no_audio_detected",
-    "MaxDurationExceeded": "aws_transcribe::max_duration_exceeded"
+    "MaxDurationExceeded": "aws_transcribe::max_duration_exceeded",
+    "VadDetected": "aws_transcribe::vad_detected"
  },
  "AzureTranscriptionEvents": {
    "Transcription": "azure_transcribe::transcription",
    "StartOfUtterance": "azure_transcribe::start_of_utterance",
    "EndOfUtterance": "azure_transcribe::end_of_utterance",
-    "NoSpeechDetected": "azure_transcribe::no_speech_detected"
+    "NoSpeechDetected": "azure_transcribe::no_speech_detected",
+    "VadDetected": "azure_transcribe::vad_detected"
  },
  "ListenEvents": {
    "Connect": "mod_audio_fork::connect",
@@ -112,6 +115,7 @@
    "session:redirect",
    "call:status",
    "queue:status",
+    "dial:confirm",
    "verb:hook",
    "jambonz:error"
  ],
--- a/lib/utils/db-utils.js
+++ b/lib/utils/db-utils.js
@@ -48,6 +48,7 @@ module.exports = (logger, srf) => {
  const pp = pool.promise();

  const lookupAccountDetails = async(account_sid) => {
+
    const [r] = await pp.query({sql: sqlAccountDetails, nestTables: true}, account_sid);
    if (0 === r.length) throw new Error(`invalid accountSid: ${account_sid}`);
    const [r2] = await pp.query(sqlSpeechCredentials, account_sid);
--- a/lib/utils/http-requestor.js
+++ b/lib/utils/http-requestor.js
@@ -49,8 +49,12 @@ class HttpRequestor extends BaseRequestor {
   * @param {string} [hook.password] - if basic auth is protecting the endpoint
   * @param {object} [params] - request parameters
   */
-  async request(type, hook, params) {
+  async request(type, hook, params, httpHeaders = {}) {
+    /* jambonz:error only sent over ws */
+    if (type === 'jambonz:error') return;
+
    assert(HookMsgTypes.includes(type));
+
    const payload = params ? snakeCaseKeys(params, ['customerData', 'sip']) : null;
    const url = hook.url || hook;
    const method = hook.method || 'POST';
@@ -64,8 +68,8 @@ class HttpRequestor extends BaseRequestor {
    let buf;
    try {
      const sigHeader = this._generateSigHeader(payload, this.secret);
-      const headers = {...sigHeader, ...this.authHeader};
-      //this.logger.info({url, headers}, 'send webhook');
+      const headers = {...sigHeader, ...this.authHeader, ...httpHeaders};
+      this.logger.debug({url, headers}, 'send webhook');
      buf = this._isRelativeUrl(url) ?
        await this.post(url, payload, headers) :
        await bent(method, 'buffer', 200, 201, 202)(url, payload, headers);
--- a/lib/utils/install-srf-locals.js
+++ b/lib/utils/install-srf-locals.js
@@ -1,6 +1,5 @@
 const Mrf = require('drachtio-fsmrf');
 const ip = require('ip');
-const localIp = ip.address();
 const PORT = process.env.HTTP_PORT || 3000;
 const assert = require('assert');

@@ -32,6 +31,7 @@ function initMS(logger, wrapper, ms) {
 function installSrfLocals(srf, logger) {
  logger.debug('installing srf locals');
  assert(!srf.locals.dbHelpers);
+  const {tracer} = srf.locals.otel;
  const {getSBC, lifecycleEmitter} = require('./sbc-pinger')(logger);
  const StatsCollector = require('@jambonz/stats-collector');
  const stats = srf.locals.stats = new StatsCollector(logger);
@@ -49,7 +49,11 @@ function installSrfLocals(srf, logger) {
        assert.ok(arr, `Invalid syntax JAMBONES_FREESWITCH: ${process.env.JAMBONES_FREESWITCH}`);
        const opts = {address: arr[1], port: arr[2], secret: arr[3]};
        if (arr.length > 4) opts.advertisedAddress = arr[4];
+        /* NB: originally for testing only, but for now all jambonz deployments
+          have freeswitch installed locally alongside this app
+        */
        if (process.env.NODE_ENV === 'test') opts.listenAddress = '0.0.0.0';
+        else if (process.env.JAMBONES_ESL_LISTEN_ADDRESS) opts.listenAddress = process.env.JAMBONES_ESL_LISTEN_ADDRESS;
        return opts;
      });
    logger.info({fsInventory}, 'freeswitch inventory');
@@ -62,7 +66,7 @@ function installSrfLocals(srf, logger) {
        initMS(logger, val, ms);
      }
      catch (err) {
-        logger.info(`failed connecting to freeswitch at ${fs.address}, will retry shortly`);
+        logger.info({err}, `failed connecting to freeswitch at ${fs.address}, will retry shortly: ${err.message}`);
      }
    }
    // retry to connect to any that were initially offline
@@ -74,7 +78,7 @@ function installSrfLocals(srf, logger) {
            const ms = await mrf.connect(val.opts);
            initMS(logger, val, ms);
          } catch (err) {
-            logger.info(`failed connecting to freeswitch at ${val.opts.address}, will retry shortly`);
+            logger.info({err}, `failed connecting to freeswitch at ${val.opts.address}, will retry shortly`);
          }
        }
      }
@@ -127,7 +131,7 @@ function installSrfLocals(srf, logger) {
    password: process.env.JAMBONES_MYSQL_PASSWORD,
    database: process.env.JAMBONES_MYSQL_DATABASE,
    connectionLimit: process.env.JAMBONES_MYSQL_CONNECTION_LIMIT || 10
-  }, logger);
+  }, logger, tracer);
  const {
    client,
    updateCallStatus,
@@ -152,7 +156,7 @@ function installSrfLocals(srf, logger) {
  } = require('@jambonz/realtimedb-helpers')({
    host: process.env.JAMBONES_REDIS_HOST,
    port: process.env.JAMBONES_REDIS_PORT || 6379
-  }, logger);
+  }, logger, tracer);
  const {
    writeAlerts,
    AlertType
@@ -162,6 +166,13 @@ function installSrfLocals(srf, logger) {
    commitInterval: 'test' === process.env.NODE_ENV ? 7 : 20
  });

+  let localIp;
+  try {
+    localIp = ip.address();
+  } catch (err) {
+    logger.error({err}, 'installSrfLocals - error detecting local ipv4 address');
+  }
+
  srf.locals = {...srf.locals,
    dbHelpers: {
      client,
@@ -196,8 +207,6 @@ function installSrfLocals(srf, logger) {
      getListPosition
    },
    parentLogger: logger,
-    ipv4: localIp,
-    serviceUrl: `http://${localIp}:${PORT}`,
    getSBC,
    getSmpp: () => {
      return process.env.SMPP_URL;
@@ -208,6 +217,11 @@ function installSrfLocals(srf, logger) {
    writeAlerts,
    AlertType
  };
+
+  if (localIp) {
+    srf.locals.ipv4 = localIp;
+    srf.locals.serviceUrl = `http://${localIp}:${PORT}`;
+  }
 }

 module.exports = installSrfLocals;
--- a/lib/utils/place-outdial.js
+++ b/lib/utils/place-outdial.js
@@ -4,15 +4,18 @@ const SipError = require('drachtio-srf').SipError;
 const {TaskPreconditions, CallDirection} = require('../utils/constants');
 const CallInfo = require('../session/call-info');
 const assert = require('assert');
+const normalizeJambones = require('../utils/normalize-jambones');
+const makeTask = require('../tasks/make_task');
 const ConfirmCallSession = require('../session/confirm-call-session');
 const AdultingCallSession = require('../session/adulting-call-session');
 const deepcopy = require('deepcopy');
 const moment = require('moment');
 const stripCodecs = require('./strip-ancillary-codecs');
+const RootSpan = require('./call-tracer');
 const { v4: uuidv4 } = require('uuid');

 class SingleDialer extends Emitter {
-  constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo}) {
+  constructor({logger, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan}) {
    super();
    assert(target.type);

@@ -22,6 +25,8 @@ class SingleDialer extends Emitter {
    this.opts = opts;
    this.application = application;
    this.confirmHook = target.confirmHook;
+    this.rootSpan = rootSpan;
+    this.startSpan = startSpan;

    this.bindings = logger.bindings();

@@ -71,7 +76,7 @@ class SingleDialer extends Emitter {
      };
    }
    this.ms = ms;
-    let uri, to;
+    let uri, to, inviteSpan;
    try {
      switch (this.target.type) {
        case 'phone':
@@ -137,13 +142,24 @@ class SingleDialer extends Emitter {
        localSdp: this.ep.local.sdp
      });
      if (this.target.auth) opts.auth = this.target.auth;
+      inviteSpan = this.startSpan('invite', {
+        'invite.uri': uri,
+        'invite.dest_type': this.target.type
+      });
+
      this.dlg = await srf.createUAC(uri, {...opts,  followRedirects: true, keepUriOnRedirect: true}, {
        cbRequest: (err, req) => {
          if (err) {
            this.logger.error(err, 'SingleDialer:exec Error creating call');
            this.emit('callCreateFail', err);
+            inviteSpan.setAttributes({
+              'invite.status_code': 500,
+              'invite.err': err.message
+            });
+            inviteSpan.end();
            return;
          }
+          inviteSpan.setAttributes({'invite.call_id': req.get('Call-ID')});

          /**
           * INVITE has been sent out
@@ -156,7 +172,8 @@ class SingleDialer extends Emitter {
            parentCallInfo: this.parentCallInfo,
            req,
            to,
-            callSid: this.callSid
+            callSid: this.callSid,
+            traceId: this.rootSpan.traceId
          });
          this.logger = srf.locals.parentLogger.child({
            callSid: this.callSid,
@@ -193,6 +210,9 @@ class SingleDialer extends Emitter {
      });
      this.logger.debug(`SingleDialer:exec call connected: ${this.callSid}`);
      const connectTime = this.dlg.connectTime = moment();
+      inviteSpan.setAttributes({'invite.status_code': 200});
+      inviteSpan.end();
+

      /* race condition: we were killed just as call was answered */
      if (this.killed) {
@@ -246,10 +266,17 @@ class SingleDialer extends Emitter {
        if (err.status === 487) status.callStatus = CallStatus.NoAnswer;
        else if ([486, 600].includes(err.status)) status.callStatus = CallStatus.Busy;
        this.logger.info(`SingleDialer:exec outdial failure ${err.status}`);
+        inviteSpan.setAttributes({'invite.status_code': err.status});
+        inviteSpan.end();
      }
      else {
        this.logger.error(err, 'SingleDialer:exec');
        status.sipStatus = 500;
+        inviteSpan.setAttributes({
+          'invite.status_code': 500,
+          'invite.err': err.message
+        });
+        inviteSpan.end();
      }
      this.emit('callStatusChange', status);
      if (this.ep) this.ep.destroy();
@@ -284,8 +311,8 @@ class SingleDialer extends Emitter {
  async _executeApp(confirmHook) {
    try {
      // retrieve set of tasks
-      const tasks = await this.requestor.request(confirmHook, this.callInfo.toJSON());
-
+      const json = await this.requestor.request('dial:confirm', confirmHook, this.callInfo.toJSON());
+      const tasks = normalizeJambones(this.logger, json).map((tdata) => makeTask(this.logger, tdata));
      // verify it contains only allowed verbs
      const allowedTasks = tasks.filter((task) => {
        return [
@@ -305,7 +332,9 @@ class SingleDialer extends Emitter {
        dlg: this.dlg,
        ep: this.ep,
        callInfo: this.callInfo,
-        tasks
+        accountInfo: this.accountInfo,
+        tasks,
+        rootSpan: this.rootSpan
      });
      await cs.exec();

@@ -319,7 +348,6 @@ class SingleDialer extends Emitter {
  }

  async doAdulting({logger, tasks, application}) {
-    this.logger = logger;
    this.adulting = true;
    this.emit('adulting');
    if (this.ep) {
@@ -330,15 +358,21 @@ class SingleDialer extends Emitter {
    else {
      await this.reAnchorMedia();
    }
+
+    this.dlg.callSid = this.callSid;
+    this.dlg.linkedSpanId = this.rootSpan.traceId;
+    const rootSpan = new RootSpan('outbound-call', this.dlg);
+    const newLogger = logger.child({traceId: rootSpan.traceId});
    const cs = new AdultingCallSession({
-      logger: this.logger,
+      logger: newLogger,
      singleDialer: this,
      application,
      callInfo: this.callInfo,
      accountInfo: this.accountInfo,
-      tasks
+      tasks,
+      rootSpan
    });
-    cs.exec();
+    cs.exec().catch((err) => newLogger.error({err}, 'doAdulting: error executing session'));
    return cs;
  }

@@ -387,9 +421,13 @@ class SingleDialer extends Emitter {
  }
 }

-function placeOutdial({logger, srf, ms, sbcAddress, target, opts, application, callInfo, accountInfo}) {
+function placeOutdial({
+  logger, srf, ms, sbcAddress, target, opts, application, callInfo, accountInfo, rootSpan, startSpan
+}) {
  const myOpts = deepcopy(opts);
-  const sd = new SingleDialer({logger, sbcAddress, target, myOpts, application, callInfo, accountInfo});
+  const sd = new SingleDialer({
+    logger, sbcAddress, target, myOpts, application, callInfo, accountInfo, rootSpan, startSpan
+  });
  sd.exec(srf, ms, myOpts);
  return sd;
 }
--- a/lib/utils/ws-requestor.js
+++ b/lib/utils/ws-requestor.js
@@ -15,6 +15,7 @@ class WsRequestor extends BaseRequestor {
    this.messagesInFlight = new Map();
    this.maliciousClient = false;
    this.closedByUs = false;
+    this.backoffMs = 500;

    assert(this._isAbsoluteUrl(this.url));

@@ -32,7 +33,7 @@ class WsRequestor extends BaseRequestor {
   * @param {string} [hook.password] - if basic auth is protecting the endpoint
   * @param {object} [params] - request parameters
   */
-  async request(type, hook, params) {
+  async request(type, hook, params, httpHeaders = {}) {
    assert(HookMsgTypes.includes(type));
    const url = hook.url || hook;

@@ -47,7 +48,7 @@ class WsRequestor extends BaseRequestor {
    if (this._isAbsoluteUrl(url) && url.startsWith('http')) {
      this.logger.debug({hook}, 'WsRequestor: sending a webhook (HTTP)');
      const requestor = new HttpRequestor(this.logger, this.account_sid, hook, this.secret);
-      return requestor.request(type, hook, params);
+      return requestor.request(type, hook, params, httpHeaders);
    }

    /* connect if necessary */
@@ -72,12 +73,14 @@ class WsRequestor extends BaseRequestor {
    assert.ok(url, 'WsRequestor:request url was not provided');

    const msgid = short.generate();
+    const b3 = httpHeaders?.b3 ? {b3: httpHeaders.b3} : {};
    const obj = {
      type,
      msgid,
      call_sid: this.call_sid,
      hook: type === 'verb:hook' ? url : undefined,
-      data: {...payload}
+      data: {...payload},
+      ...b3
    };

    //this.logger.debug({obj}, `websocket: sending (${url})`);
@@ -102,6 +105,7 @@ class WsRequestor extends BaseRequestor {
      /* save the message info for reply */
      const startAt = process.hrtime();
      this.messagesInFlight.set(msgid, {
+        timer,
        success: (response) => {
          clearTimeout(timer);
          const rtt = this._roundTrip(startAt);
@@ -123,15 +127,17 @@ class WsRequestor extends BaseRequestor {
  }

  close() {
-    this.logger.info('WsRequestor:close closing socket');
    this.closedByUs = true;
    try {
      if (this.ws) {
+        this.logger.info('WsRequestor:close closing socket');
        this.ws.close();
        this.ws.removeAllListeners();
      }

      for (const [msgid, obj] of this.messagesInFlight) {
+        const {timer} = obj;
+        clearTimeout(timer);
        obj.failure(`abandoning msgid ${msgid} since we have closed the socket`);
      }
      this.messagesInFlight.clear();
@@ -144,10 +150,13 @@ class WsRequestor extends BaseRequestor {
  _connect() {
    assert(!this.ws);
    return new Promise((resolve, reject) => {
+      const handshakeTimeout = process.env.JAMBONES_WS_HANDSHAKE_TIMEOUT_MS ?
+        parseInt(process.env.JAMBONES_WS_HANDSHAKE_TIMEOUT_MS) :
+        1500;
      let opts = {
        followRedirects: true,
        maxRedirects: 2,
-        handshakeTimeout: 1000,
+        handshakeTimeout,
        maxPayload: 8096,
      };
      if (this.username && this.password) opts = {...opts, auth: `${this.username}:${this.password}`};
@@ -156,7 +165,7 @@ class WsRequestor extends BaseRequestor {
        .once('ready', (ws) => {
          this.ws = ws;
          this.removeAllListeners('not-ready');
-          if (this.connections++ > 0) this.request('session:reconnect', this.url);
+          if (this.connections > 0) this.request('session:reconnect', this.url);
          resolve();
        })
        .once('not-ready', (err) => {
@@ -213,8 +222,10 @@ class WsRequestor extends BaseRequestor {

  _onSocketClosed() {
    this.ws = null;
-    if (this.connections > 0 && this.connections < MAX_RECONNECTS && !this.closedByUs) {
-      setTimeout(this._connect.bind(this), 500);
+    this.emit('connection-dropped');
+    if (this.connections++ > 0 && this.connections < MAX_RECONNECTS && !this.closedByUs) {
+      setTimeout(this._connect.bind(this), this.backoffMs);
+      this.backoffMs = this.backoffMs < 2000 ? this.backoffMs * 2 : (this.backoffMs + 2000);
    }
  }

@@ -229,7 +240,7 @@ class WsRequestor extends BaseRequestor {
    /* messages must be JSON format */
    try {
      const obj = JSON.parse(content);
-      const {type, msgid, command, queueCommand = false, data} = obj;
+      const {type, msgid, command, call_sid = this.call_sid, queueCommand = false, data} = obj;

      this.logger.debug({obj}, 'WsRequestor:request websocket: received');
      assert.ok(type, 'type property not supplied');
@@ -243,14 +254,14 @@ class WsRequestor extends BaseRequestor {
        case 'command':
          assert.ok(command, 'command property not supplied');
          assert.ok(data, 'data property not supplied');
-          this._recvCommand(msgid, command, queueCommand, data);
+          this._recvCommand(msgid, command, call_sid, queueCommand, data);
          break;

        default:
          assert.ok(false, `invalid type property: ${type}`);
      }
    } catch (err) {
-      this.logger.info({err}, 'WsRequestor:_onMessage - invalid incoming message');
+      this.logger.info({err, content}, 'WsRequestor:_onMessage - invalid incoming message');
    }
  }

@@ -266,10 +277,10 @@ class WsRequestor extends BaseRequestor {
    success && success(data);
  }

-  _recvCommand(msgid, command, queueCommand, data) {
+  _recvCommand(msgid, command, call_sid, queueCommand, data) {
    // TODO: validate command
-    this.logger.info({msgid, command, queueCommand, data}, 'received command');
-    this.emit('command', {msgid, command, queueCommand, data});
+    this.logger.info({msgid, command, call_sid, queueCommand, data}, 'received command');
+    this.emit('command', {msgid, command, call_sid, queueCommand, data});
  }
 }

--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
  "name": "jambonz-feature-server",
-  "version": "v0.7.4",
+  "version": "v0.7.5",
  "main": "app.js",
  "engines": {
    "node": ">= 10.16.0"
@@ -21,7 +21,7 @@
  },
  "scripts": {
    "start": "node app",
-    "test": "NODE_ENV=test JAMBONES_HOSTING=1 DRACHTIO_HOST=127.0.0.1 DRACHTIO_PORT=9060 DRACHTIO_SECRET=cymru JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=127.0.0.1 JAMBONES_REDIS_PORT=16379 JAMBONES_LOGLEVEL=error ENABLE_METRICS=0 HTTP_PORT=3000 JAMBONES_SBCS=172.38.0.10 JAMBONES_FREESWITCH=127.0.0.1:8022:ClueCon:docker-host JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_NETWORK_CIDR=172.38.0.0/16 node test/ ",
+    "test": "NODE_ENV=test JAMBONES_HOSTING=1 DRACHTIO_HOST=127.0.0.1 DRACHTIO_PORT=9060 DRACHTIO_SECRET=cymru JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=127.0.0.1 JAMBONES_REDIS_PORT=16379 JAMBONES_LOGLEVEL=info ENABLE_METRICS=0 HTTP_PORT=3000 JAMBONES_SBCS=172.38.0.10 JAMBONES_FREESWITCH=127.0.0.1:8022:ClueCon:docker-host JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_NETWORK_CIDR=172.38.0.0/16 node test/ ",
    "coverage": "./node_modules/.bin/nyc --reporter html --report-dir ./coverage npm run test",
    "jslint": "eslint app.js lib"
  },
@@ -33,6 +33,18 @@
    "@jambonz/realtimedb-helpers": "^0.4.26",
    "@jambonz/stats-collector": "^0.1.6",
    "@jambonz/time-series": "^0.1.6",
+    "@opentelemetry/api": "^1.1.0",
+    "@opentelemetry/exporter-collector": "^0.25.0",
+    "@opentelemetry/exporter-jaeger": "^1.1.0",
+    "@opentelemetry/exporter-zipkin": "^1.1.0",
+    "@opentelemetry/instrumentation": "^0.27.0",
+    "@opentelemetry/instrumentation-express": "^0.28.0",
+    "@opentelemetry/instrumentation-http": "^0.27.0",
+    "@opentelemetry/instrumentation-pino": "^0.28.1",
+    "@opentelemetry/resources": "^1.1.0",
+    "@opentelemetry/sdk-trace-base": "^1.1.0",
+    "@opentelemetry/sdk-trace-node": "^1.1.0",
+    "@opentelemetry/semantic-conventions": "^1.1.0",
    "aws-sdk": "^2.1073.0",
    "bent": "^7.3.12",
    "cidr-matcher": "^2.1.1",
@@ -43,9 +55,9 @@
    "express": "^4.17.1",
    "helmet": "^5.0.2",
    "ip": "^1.1.5",
-    "moment": "^2.29.1",
+    "moment": "^2.29.2",
    "parse-url": "^5.0.7",
-    "pino": "^6.14.0",
+    "pino": "^6.13.4",
    "short-uuid": "^4.2.0",
    "to-snake-case": "^1.0.0",
    "uuid": "^8.3.2",
--- a/tracer.js
+++ b/tracer.js
@@ -0,0 +1,61 @@
+const opentelemetry = require('@opentelemetry/api');
+const { registerInstrumentations } = require('@opentelemetry/instrumentation');
+const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
+const { Resource } = require('@opentelemetry/resources');
+const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');
+const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base');
+const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
+const { ZipkinExporter } = require('@opentelemetry/exporter-zipkin');
+const { CollectorTraceExporter } =  require('@opentelemetry/exporter-collector');
+//const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http');
+//const { ExpressInstrumentation } = require('@opentelemetry/instrumentation-express');
+//const { PinoInstrumentation } = require('@opentelemetry/instrumentation-pino');
+
+module.exports = (serviceName) => {
+  if (process.env.JAMBONES_OTEL_ENABLED) {
+    const {version} = require('./package.json');
+    const provider = new NodeTracerProvider({
+      resource: new Resource({
+        [SemanticResourceAttributes.SERVICE_NAME]: serviceName,
+        [SemanticResourceAttributes.SERVICE_VERSION]: version,
+      }),
+    });
+
+    let exporter;
+    if (process.env.OTEL_EXPORTER_JAEGER_AGENT_HOST) {
+      exporter = new JaegerExporter();
+    }
+    else if (process.env.OTEL_EXPORTER_ZIPKIN_URL) {
+      exporter = new ZipkinExporter({url:process.env.OTEL_EXPORTER_ZIPKIN_URL});
+    }
+    else {
+      exporter = new CollectorTraceExporter({
+        url: process.OTEL_EXPORTER_COLLECTOR_URL
+      });
+    }
+
+    provider.addSpanProcessor(new BatchSpanProcessor(exporter, {
+      // The maximum queue size. After the size is reached spans are dropped.
+      maxQueueSize: 100,
+      // The maximum batch size of every export. It must be smaller or equal to maxQueueSize.
+      maxExportBatchSize: 10,
+      // The interval between two consecutive exports
+      scheduledDelayMillis: 500,
+      // How long the export can run before it is cancelled
+      exportTimeoutMillis: 30000,
+    }));
+
+    // Initialize the OpenTelemetry APIs to use the NodeTracerProvider bindings
+    provider.register();
+    registerInstrumentations({
+      instrumentations: [
+        //new HttpInstrumentation(),
+        //new ExpressInstrumentation(),
+        //new PinoInstrumentation()
+      ],
+    });
+  }
+
+  return opentelemetry.trace.getTracer(serviceName);
+};
+
Author	SHA1	Message	Date
Dave Horton	de9f2ce5ca	bugfix: handle error if we cannot get our own ipv4	2022-04-21 19:09:23 -04:00
Dave Horton	36c97e9562	simplify error message	2022-04-21 14:43:09 -04:00
Dave Horton	13ea559cb1	send error notification over websocket if tts fails	2022-04-21 14:33:49 -04:00
Dave Horton	698d12a95f	clean up error handling in say verb	2022-04-21 10:27:33 -04:00
Dave Horton	359cb82d80	per recommendation from microsoft, do NOT sort transcripts by confidence: first transcript in the returned list is 'best'	2022-04-17 17:53:16 -04:00
Dave Horton	29dec24095	bugfix: azure stt - if we get no speech detected, listen again	2022-04-13 12:07:30 -04:00
Dave Horton	6330b0d443	Dockerfile update	2022-04-12 16:12:29 -04:00
Dave Horton	24a0bc547f	gather: dont restart transcribing if task has been killed	2022-04-11 21:13:49 -04:00
Dave Horton	db5486de27	gather bugfix: dont start transcribing after call is gone	2022-04-10 15:48:35 -04:00
Dave Horton	41d6c74c8e	send application defaults for speech in initial webhook	2022-04-09 11:38:31 -04:00
Dave Horton	92ca40c9b3	add feature flag env JAMBONES_INJECT_CONTENT (#98 )	2022-04-06 15:54:59 -04:00
Dave Horton	3fa913215f	bump version	2022-04-06 08:19:33 -04:00
Snyk bot	0b132411c1	fix: package.json & package-lock.json to reduce vulnerabilities (#97 ) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JS-MOMENT-2440688	2022-04-06 07:30:11 -04:00
Dave Horton	077d34dc9e	gather: resolve with reason killed prevented task from ending	2022-04-05 08:15:30 -04:00
Dave Horton	49a75a3e3a	minor logging improvement	2022-04-04 14:02:09 -04:00
Dave Horton	6f214a66e8	AdultingSession: just create new child logger (simpler)	2022-04-04 13:49:30 -04:00
Dave Horton	3456c51118	AdultingSession: change bindings on logger to include new traceId	2022-04-04 13:38:38 -04:00
Dave Horton	13c38a9875	AdultingCallSession: constructor now requires rootSpan	2022-04-04 13:23:08 -04:00
Dave Horton	4f87cf9b38	dial: include http b3 header	2022-04-04 13:07:29 -04:00
Dave Horton	bf21a1f9a4	config: fixes from bargein testing	2022-04-04 12:40:18 -04:00
Dave Horton	81f6163aca	confirmCallSession: pass accountInfo	2022-04-03 22:46:04 -04:00
Dave Horton	547ca0281f	fix prev commit	2022-04-03 22:30:47 -04:00
Dave Horton	3281a213c8	proper creation of confirmHook tasks	2022-04-03 22:27:37 -04:00
Dave Horton	4f2fc70383	add new type dial:confirm	2022-04-03 22:12:14 -04:00
Dave Horton	f72e8e654c	bugfix: confirmHook	2022-04-03 22:04:24 -04:00
Dave Horton	cf2100f925	another fix for confirmHook	2022-04-03 21:52:09 -04:00
Dave Horton	5a584f50da	bugfix: implement confirmHook for dial	2022-04-03 21:41:23 -04:00
Dave Horton	befe910503	logging fix	2022-04-03 20:02:51 -04:00
Dave Horton	040ec0db9b	logging fix	2022-04-03 19:42:31 -04:00
Dave Horton	8459376f88	fix bug in prev checkin	2022-04-03 19:15:00 -04:00
Dave Horton	775a317821	rest createCall: include accountSid and traceId in logging	2022-04-03 19:02:14 -04:00
Dave Horton	9004f654ff	bugfix: yet another tracing fix on rest outdial	2022-04-03 18:52:32 -04:00
Dave Horton	6163657845	bugfix: another tracing fix on rest outdial	2022-04-03 18:44:27 -04:00
Dave Horton	398daa87d5	remove tracing lib that is not needed	2022-04-03 18:34:46 -04:00
Dave Horton	4f5ab7d146	bugfix: tracing-related exception on rest createCall	2022-04-03 18:29:37 -04:00
Dave Horton	70f7775893	dial: fix tracing attribute	2022-04-03 15:36:06 -04:00
Dave Horton	a950f9f738	Feature/trace propagation (#96 ) * add b3 header for trace propagation on initial webhook * logging * add tracing context to all webhooks * Add span parameter to Task.getTracingPropagation. Pass proper span to getTracingPropagation calls in Task methods to propagate the proper spanId (#91) * some tracing cleanup * bugfix: azure stt results need to be ordered by confidence level before processing (#92) * fix assertion * bugfix: vad was not enabled on config verb, restart STT on empty transcript in gather * gather: dont send webhook if call is gone * rest outdial: handle 302 redirect so we can later cancel request if needed (#95) * gather: restart if we get an empty transcript (looking at you, Azure) Co-authored-by: javibookline <98887695+javibookline@users.noreply.github.com>	2022-04-01 14:48:27 -04:00
Dave Horton	ff8d7f3648	bugfix: create spans for nested tasks in gather, rasa, and dial; fix gather bug not starting transcribe after say completes	2022-03-29 15:44:55 -04:00
Dave Horton	6e4ae69cb7	logging	2022-03-29 09:48:18 -04:00
Dave Horton	23eae34888	add env JAMBONES_ESL_LISTEN_ADDRESS	2022-03-29 09:33:39 -04:00
Dave Horton	aaf94006db	explicitly bind esl socket to ipv4 interface (digital ocean k8s defaults to ipv6 which causes in ECONNREFUSED from freeswitch)	2022-03-29 09:19:38 -04:00
Dave Horton	86b030db93	logging	2022-03-29 08:57:32 -04:00
Dave Horton	6abfdafe05	Feature/opentelemetry (#89 ) * initial adds for otel tracing * initial basic testing * basic tracing for incoming calls * linting * add traceId to the webhook params * trace webhook calls * tracing: add new commands as tags when receiving async commands over websocket * tracing new commands * add summary for config verb * trace async commands * bugfix: undefined ref * tracing: give time for final webhooks before closing root span * tracing bugfix: span for background gather was not ended * tracing - minor tag changes * tracing - add span atttribute for reason call ended * trace call status webhooks, add app version to trace output * config: add support for automatically re-enabling * env var to customize service name in tracing UI * config: change to use 'sticky' attribute to re-enable bargein automatically * fix warnings * when adulting create a new root span * when background gather triggers bargein via vad clear queue of tasks * additional trace attributes for dial and refer * fix dial tracing * add better summary for dial * fix prev commit * add exponential backoff to WsRequestor reconnection logic * add calling number to log metadata, as this will be frequently the key data given for troubleshooting * add accountSid to log metadata * make handshake timeout for ws connections configurable with default 1.5 secs * rename env var * fix bug prev checkin * logging fixes * consistent env naming	2022-03-28 15:38:28 -04:00
Snyk bot	f1f83598ca	fix: Dockerfile to reduce vulnerabilities (#84 ) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-DEBIAN11-GNUTLS28-2419151 - https://snyk.io/vuln/SNYK-DEBIAN11-OPENSSL-2388380 - https://snyk.io/vuln/SNYK-DEBIAN11-OPENSSL-2426309 - https://snyk.io/vuln/SNYK-DEBIAN11-UTILLINUX-2401081 - https://snyk.io/vuln/SNYK-DEBIAN11-UTILLINUX-2401081	2022-03-18 07:55:42 -04:00
Dave Horton	3dd703411c	kill audio on vad when bargein is true and minBargeinWordCount is zero	2022-03-17 08:51:44 -04:00
Dave Horton	8c5cdd374b	ws command can have call_id	2022-03-10 10:52:48 -05:00
Dave Horton	15d784a4b0	bugfix: sip_refer sending body	2022-03-10 06:45:27 -05:00