Feature/opentelemetry (#89)

* initial adds for otel tracing

* initial basic testing

* basic tracing for incoming calls

* linting

* add traceId to the webhook params

* trace webhook calls

* tracing: add new commands as tags when receiving async commands over websocket

* tracing new commands

* add summary for config verb

* trace async commands

* bugfix: undefined ref

* tracing: give time for final webhooks before closing root span

* tracing bugfix: span for background gather was not ended

* tracing - minor tag changes

* tracing - add span atttribute for reason call ended

* trace call status webhooks, add app version to trace output

* config: add support for automatically re-enabling

* env var to customize service name in tracing UI

* config: change to use 'sticky' attribute to re-enable bargein automatically

* fix warnings

* when adulting create a new root span

* when background gather triggers bargein via vad clear queue of tasks

* additional trace attributes for dial and refer

* fix dial tracing

* add better summary for dial

* fix prev commit

* add exponential backoff to WsRequestor reconnection logic

* add calling number to log metadata, as this will be frequently the key data given for troubleshooting

* add accountSid to log metadata

* make handshake timeout for ws connections configurable with default 1.5 secs

* rename env var

* fix bug prev checkin

* logging fixes

* consistent env naming
This commit is contained in:
Dave Horton
2022-03-28 15:38:28 -04:00
committed by GitHub
parent f1f83598ca
commit 6abfdafe05
24 changed files with 1852 additions and 87 deletions

View File

@@ -7,6 +7,8 @@ const makeTask = require('./tasks/make_task');
const parseUri = require('drachtio-srf').parseUri;
const normalizeJambones = require('./utils/normalize-jambones');
const dbUtils = require('./utils/db-utils');
const RootSpan = require('./utils/call-tracer');
const listTaskNames = require('./utils/summarize-tasks');
module.exports = function(srf, logger) {
const {
@@ -17,15 +19,18 @@ module.exports = function(srf, logger) {
lookupAppByTeamsTenant
} = srf.locals.dbHelpers;
const {lookupAccountDetails} = dbUtils(logger, srf);
function initLocals(req, res, next) {
if (!req.has('X-Account-Sid')) {
logger.info('getAccountDetails - rejecting call due to missing X-Account-Sid header');
return res.send(500);
}
const callSid = req.has('X-Retain-Call-Sid') ? req.get('X-Retain-Call-Sid') : uuidv4();
req.locals = {
callSid,
logger: logger.child({callId: req.get('Call-ID'), callSid})
};
const account_sid = req.get('X-Account-Sid');
req.locals = {callSid, account_sid};
if (req.has('X-Application-Sid')) {
const application_sid = req.get('X-Application-Sid');
req.locals.logger.debug(`got application from X-Application-Sid header: ${application_sid}`);
logger.debug(`got application from X-Application-Sid header: ${application_sid}`);
req.locals.application_sid = application_sid;
}
if (req.has('X-Authenticated-User')) req.locals.originatingUser = req.get('X-Authenticated-User');
@@ -34,19 +39,50 @@ module.exports = function(srf, logger) {
next();
}
function createRootSpan(req, res, next) {
const {callSid, account_sid} = req.locals;
const rootSpan = new RootSpan('incoming-call', req);
const traceId = rootSpan.traceId;
req.locals = {
...req.locals,
traceId,
logger: logger.child({
callId: req.get('Call-ID'),
callSid,
accountSid: account_sid,
callingNumber: req.callingNumber,
calledNumber: req.calledNumber,
traceId}),
rootSpan
};
/**
* end the span on final failure or cancel from caller;
* otherwise it will be closed when sip dialog is destroyed
*/
req.once('cancel', () => {
rootSpan.setAttributes({finalStatus: 487});
rootSpan.end();
});
res.once('finish', () => {
rootSpan.setAttributes({finalStatus: res.statusCode});
res.statusCode >= 300 && rootSpan.end();
});
next();
}
/**
* retrieve account information for the incoming call
*/
async function getAccountDetails(req, res, next) {
const {rootSpan, account_sid} = req.locals;
if (!req.has('X-Account-Sid')) {
logger.info('getAccountDetails - rejecting call due to missing X-Account-Sid header');
return res.send(500);
}
const account_sid = req.locals.account_sid = req.get('X-Account-Sid');
const {span} = rootSpan.startChildSpan('lookupAccountDetails');
try {
req.locals.accountInfo = await lookupAccountDetails(account_sid);
span.end();
if (!req.locals.accountInfo.account.is_active) {
logger.info(`Account is inactive or suspended ${account_sid}`);
// TODO: alert
@@ -55,6 +91,7 @@ module.exports = function(srf, logger) {
logger.debug({accountInfo: req.locals?.accountInfo?.account}, `retrieved account info for ${account_sid}`);
next();
} catch (err) {
span.end();
logger.info({err}, `Error retrieving account details for account ${account_sid}`);
res.send(503, {headers: {'X-Reason': `No Account exists for sid ${account_sid}`}});
}
@@ -86,7 +123,8 @@ module.exports = function(srf, logger) {
*/
async function retrieveApplication(req, res, next) {
const logger = req.locals.logger;
const {accountInfo, account_sid} = req.locals;
const {accountInfo, account_sid, rootSpan} = req.locals;
const {span} = rootSpan.startChildSpan('lookupApplication');
try {
let app;
if (req.locals.application_sid) app = await lookupAppBySid(req.locals.application_sid);
@@ -130,6 +168,11 @@ module.exports = function(srf, logger) {
}
}
span.setAttributes({
'app.hook': app?.call_hook?.url,
'application_sid': req.locals.application_sid
});
span.end();
if (!app || !app.call_hook || !app.call_hook.url) {
logger.info(`rejecting call to ${req.locals.calledNumber}: no application or webhook url`);
return res.send(480, {
@@ -163,9 +206,15 @@ module.exports = function(srf, logger) {
// eslint-disable-next-line no-unused-vars
const {call_hook, call_status_hook, ...appInfo} = obj; // mask sensitive data like user/pass on webhook
logger.info({app: appInfo}, `retrieved application for incoming call to ${req.locals.calledNumber}`);
req.locals.callInfo = new CallInfo({req, app, direction: CallDirection.Inbound});
req.locals.callInfo = new CallInfo({
req,
app,
direction: CallDirection.Inbound,
traceId: rootSpan.traceId
});
next();
} catch (err) {
span.end();
logger.error(err, `${req.get('Call-ID')} Error looking up application for ${req.calledNumber}`);
res.send(500);
}
@@ -176,7 +225,8 @@ module.exports = function(srf, logger) {
*/
async function invokeWebCallback(req, res, next) {
const logger = req.locals.logger;
const app = req.locals.application;
const {rootSpan, application:app} = req.locals;
let span;
try {
if (app.tasks) {
@@ -187,11 +237,20 @@ module.exports = function(srf, logger) {
/* retrieve the application to execute for this inbound call */
const params = Object.assign(['POST', 'WS'].includes(app.call_hook.method) ? {sip: req.msg} : {},
req.locals.callInfo);
const obj = rootSpan.startChildSpan('performAppWebhook');
span = obj.span;
const json = await app.requestor.request('session:new', app.call_hook, params);
app.tasks = normalizeJambones(logger, json).map((tdata) => makeTask(logger, tdata));
span.setAttributes({
'http.statusCode': 200,
'app.tasks': listTaskNames(app.tasks)
});
span.end();
if (0 === app.tasks.length) throw new Error('no application provided');
next();
} catch (err) {
span?.setAttributes({webhookStatus: err.statusCode});
span?.end();
logger.info({err}, `Error retrieving or parsing application: ${err?.message}`);
res.send(480, {headers: {'X-Reason': err?.message || 'unknown'}});
app.requestor.close();
@@ -200,6 +259,7 @@ module.exports = function(srf, logger) {
return {
initLocals,
createRootSpan,
getAccountDetails,
normalizeNumbers,
retrieveApplication,