add APIs to retrieve pcaps from homer

.gitignore

@@ -43,4 +43,6 @@ create_db.sql
 .vscode
 
 .env.*
-.env
+.env
+
+postgres-data/*

lib/routes/api/recent-calls.js

@@ -1,7 +1,7 @@
 const router = require('express').Router();
 const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
-
+const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
   if (arr) return arr[1];
@@ -34,4 +34,42 @@ router.get('/', async(req, res) => {
   }
 });
 
+router.get('/:call_id', async(req, res) => {
+  const {logger} = req.app.locals;
+  try {
+    const token = await getHomerApiKey(logger);
+    if (!token) return res.sendStatus(400, {msg: 'Failed to get Homer API token; check server config'});
+    const obj = await getHomerSipTrace(logger, token, req.params.call_id);
+    if (!obj) {
+      logger.info(`/RecentCalls: unable to get sip traces from Homer for ${req.params.call_id}`);
+      return res.sendStatus(404);
+    }
+    res.status(200).json(obj);
+  } catch (err) {
+    logger.error({err}, '/RecentCalls error retrieving sip traces from homer');
+    res.sendStatus(err.statusCode || 500);
+  }
+});
+
+router.get('/:call_id/pcap', async(req, res) => {
+  const {logger} = req.app.locals;
+  try {
+    const token = await getHomerApiKey(logger);
+    if (!token) return res.sendStatus(400, {msg: 'getHomerApiKey: Failed to get Homer API token; check server config'});
+    const stream = await getHomerPcap(logger, token, [req.params.call_id]);
+    if (!stream) {
+      logger.info(`getHomerApiKey: unable to get sip traces from Homer for ${req.params.call_id}`);
+      return res.sendStatus(404);
+    }
+    res.set({
+      'Content-Type': 'application/octet-stream',
+      'Content-Disposition': `attachment; filename=callid-${req.params.call_id}.pcap`
+    });
+    stream.pipe(res);
+  } catch (err) {
+    logger.error({err}, 'getHomerApiKey error retrieving sip traces from homer');
+    res.sendStatus(err.statusCode || 500);
+  }
+});
+
 module.exports = router;

lib/routes/api/utils.js

@@ -74,12 +74,13 @@ const createTestCdrs = async(writeCdrs,  account_sid) => {
   for (let i = 0 ; i < points; i++) {
     const attempted_at = new Date(start.getTime() + (i * increment));
     const failed = 0 === i % 5;
+    const sip_callid = `685cd008-0a66-4974-b37a-bdd6d9a3c4a-${i % 2}`;
     data.push({
       call_sid: 'b6f48929-8e86-4d62-ae3b-64fb574d91f6',
       from: '15083084809',
       to: '18882349999',
       answered:  !failed,
-      sip_callid: '685cd008-0a66-4974-b37a-bdd6d9a3c4aa@192.168.1.100',
+      sip_callid,
       sip_status: 200,
       duration: failed ? 0 : 45,
       attempted_at: attempted_at.getTime(),

lib/swagger/swagger.yaml

@@ -2635,6 +2635,56 @@ paths:
                           - duration
         404:
           description: account not found
+  /Accounts/{AccountSid}/RecentCalls/{CallId}:
+    parameters:
+      - name: AccountSid
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+      - name: CallId
+        in: path
+        required: true
+        schema:
+          type: string
+    get:
+      summary: retrieve sip trace detail for a call
+      operationId: getRecentCallTrace
+      responses:
+        200:
+          description: retrieve sip trace data
+          content:
+            application/json:
+              schema:
+                type: object
+        404:
+          description: account or call not found
+  /Accounts/{AccountSid}/RecentCalls/{CallId}/pcap:
+    parameters:
+      - name: AccountSid
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+      - name: CallId
+        in: path
+        required: true
+        schema:
+          type: string
+    get:
+      summary: retrieve pcap for a call
+      operationId: getRecentCallTrace
+      responses:
+        200:
+          description: retrieve sip trace data
+          content:
+            application/octet-stream:
+              schema:
+                type: object
+        404:
+          description: account or call not found
   /Accounts/{AccountSid}/Alerts:
     parameters:
       - name: AccountSid

lib/utils/homer-utils.js

@@ -0,0 +1,93 @@
+const debug = require('debug')('jambonz:api-server');
+const bent = require('bent');
+const basicAuth = (apiKey) => {
+  const header = `Bearer ${apiKey}`;
+  return {Authorization: header};
+};
+const postJSON = bent(process.env.HOMER_BASE_URL || 'http://127.0.0.1', 'POST', 'json', 200, 201);
+const postPcap = bent(process.env.HOMER_BASE_URL || 'http://127.0.0.1', 'POST', 200, {
+  'Content-Type': 'application/json',
+  'Accept': 'application/json, text/plain, */*',
+});
+const SEVEN_DAYS_IN_MS = (1000 * 3600 * 24 * 7);
+
+const getHomerApiKey = async(logger) => {
+  if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
+    logger.debug('getHomerApiKey: Homer integration not installed');
+  }
+
+  try {
+    const obj = await postJSON('/api/v3/auth', {
+      username: process.env.HOMER_USERNAME,
+      password: process.env.HOMER_PASSWORD
+    });
+    debug(obj);
+    logger.debug({obj}, `getHomerApiKey for user ${process.env.HOMER_USERNAME}`);
+    return obj.token;
+  } catch (err) {
+    debug(err);
+    logger.info({err}, `getHomerApiKey: Error retrieving apikey for user ${process.env.HOMER_USERNAME}`);
+  }
+};
+
+const getHomerSipTrace = async(logger, apiKey, callId) => {
+  if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
+    logger.debug('getHomerSipTrace: Homer integration not installed');
+  }
+  try {
+    const now = Date.now();
+    const obj = await postJSON('/api/v3/call/transaction', {
+      param: {
+        transaction: {
+          call: true
+        },
+        search: {
+          '1_call': {
+            callid: [callId]
+          }
+        },
+      },
+      timestamp: {
+        from: now - SEVEN_DAYS_IN_MS,
+        to: now
+      }
+    }, basicAuth(apiKey));
+    return obj;
+  } catch (err) {
+    logger.info({err}, `getHomerSipTrace: Error retrieving messages for callid ${callId}`);
+  }
+};
+
+const getHomerPcap = async(logger, apiKey, callIds) => {
+  if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
+    logger.debug('getHomerPcap: Homer integration not installed');
+  }
+  try {
+    const now = Date.now();
+    const stream = await postPcap('/api/v3/export/call/messages/pcap', {
+      param: {
+        transaction: {
+          call: true
+        },
+        search: {
+          '1_call': {
+            callid: callIds
+          }
+        },
+      },
+      timestamp: {
+        from: now - SEVEN_DAYS_IN_MS,
+        to: now
+      }
+    }, basicAuth(apiKey));
+    return stream;
+  } catch (err) {
+    logger.info({err}, `getHomerPcap: Error retrieving messages for callid ${callIds}`);
+  }
+};
+
+module.exports = {
+  getHomerApiKey,
+  getHomerSipTrace,
+  getHomerPcap
+};

package.json

@@ -28,17 +28,12 @@
     "debug": "^4.3.1",
     "express": "^4.17.1",
     "form-data": "^2.3.3",
-    "form-urlencoded": "^4.2.1",
-    "google-libphonenumber": "^3.2.15",
     "jsonwebtoken": "^8.5.1",
     "mailgun.js": "^3.3.0",
     "mysql2": "^2.2.5",
     "passport": "^0.4.1",
     "passport-http-bearer": "^1.0.1",
     "pino": "^5.17.0",
-    "qs": "^6.7.0",
-    "request": "^2.88.2",
-    "request-debug": "^0.2.0",
     "short-uuid": "^4.1.0",
     "stripe": "^8.138.0",
     "swagger-ui-express": "^4.1.6",
@@ -50,6 +45,7 @@
     "eslint-plugin-promise": "^4.2.1",
     "nyc": "^15.1.0",
     "request-promise-native": "^1.0.9",
+    "request": "^2.88.2",
     "tape": "^5.2.2"
   }
 }

test/docker-compose-testbed.yaml

@@ -1,5 +1,13 @@
 version: '3'
 
+networks:
+  jambonz-api:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.58.0.0/16
+
+
 services:  
   mysql:
     image: mysql:5.7
@@ -10,7 +18,11 @@ services:
     healthcheck:
       test: ["CMD", "mysqladmin" ,"ping", "-h", "127.0.0.1", "--protocol", "tcp"]
       timeout: 5s
-      retries: 10    
+      retries: 10
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.2
+  
   redis:
     image: redis:5-alpine
     ports:
@@ -18,8 +30,101 @@ services:
     depends_on:
       mysql:
         condition: service_healthy
-
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.3
+    
   influxdb:
     image: influxdb:1.8-alpine
     ports:
       - "8086:8086"
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.4
+  
+  db:
+    image: postgres:11-alpine
+    restart: always
+    environment:
+      POSTGRES_PASSWORD: homerSeven
+      POSTGRES_USER: root
+    expose:
+      - 5432
+    restart: unless-stopped
+    volumes:
+      - ./postgresql/init-user-db.sh:/docker-entrypoint-initdb.d/init-user-db.sh
+      - ./postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "psql -h 'localhost' -U 'root' -c '\\l'"]
+      interval: 1s
+      timeout: 3s
+      retries: 30
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.5
+  
+  heplify-server:
+    image: sipcapture/heplify-server
+    container_name: heplify-server
+    ports:
+      - "9069:9060"
+      - "9060:9060/udp"
+      - "9061:9061/tcp"
+    command:
+      - './heplify-server'
+    environment:
+      - "HEPLIFYSERVER_HEPADDR=0.0.0.0:9060"
+      - "HEPLIFYSERVER_HEPTCPADDR=0.0.0.0:9061"
+      - "HEPLIFYSERVER_DBDRIVER=postgres"
+      - "HEPLIFYSERVER_DBSHEMA=homer7"
+      - "HEPLIFYSERVER_DBADDR=db:5432"
+      - "HEPLIFYSERVER_DBUSER=root"
+      - "HEPLIFYSERVER_DBPASS=homerSeven"
+      - "HEPLIFYSERVER_DBDATATABLE=homer_data"
+      - "HEPLIFYSERVER_DBROTATE=true"
+      - "HEPLIFYSERVER_LOGLVL=debug"
+      - "HEPLIFYSERVER_LOGSTD=true"
+      - "HEPLIFYSERVER_DBDROPDAYS=7"
+      - "HEPLIFYSERVER_ALEGIDS=X-CID"
+    restart: unless-stopped
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.6
+    
+  homer-webapp:
+    container_name: homer-webapp
+    image: sipcapture/webapp
+    environment:
+      - "DB_HOST=db"
+      - "DB_USER=root"
+      - "DB_PASS=homerSeven"
+    ports:
+      - "9090:80"
+    expose:
+      - 80
+    restart: unless-stopped
+    volumes:
+      - ./bootstrap:/app/bootstrap
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.7
+    
+  drachtio:
+    container_name: drachtio
+    image: drachtio/drachtio-server:latest
+    command: drachtio --contact "sip:*;transport=udp" --loglevel debug --sofia-loglevel 9 --homer 172.58.0.6:9060 --homer-id 10
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.8
+    depends_on:
+      db:
+        condition: service_healthy
+          
+      
+      

test/homer.js

@@ -0,0 +1,45 @@
+const test = require('tape') ;
+const noopLogger = {debug: () => {}, info: () => {}, error: () => {}};
+const fs = require('fs');
+
+test('homer tests', async(t, done) => {
+  //const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../lib/utils/homer-utils');
+  if (process.env.HOMER_BASE_URL && process.env.HOMER_USERNAME && process.env.HOMER_PASSWORD) {
+    try {
+      /* get a token */
+      /*
+      let token = await getHomerApiKey(noopLogger);
+      console.log(token);
+      t.ok(token, 'successfully created an api key for homer');
+      const result = await getHomerSipTrace(noopLogger, token, '224f0f24-69aa-123a-eaa6-0ea24be4d211');
+      console.log(`got trace: ${JSON.stringify(result)}`);
+
+      var writeStream = fs.createWriteStream('./call.pcap');
+      const stream = await getHomerPcap(noopLogger, token, ['224f0f24-69aa-123a-eaa6-0ea24be4d211']);
+      stream.pipe(writeStream);
+      stream.on('end', () => {
+        console.log('finished writing');
+        done();
+      });
+      */
+
+      let result = await request.get('/RecentCalls/224f0f24-69aa-123a-eaa6-0ea24be4d211', {
+        resolveWithFullResponse: true,
+        auth: authAdmin,
+        json: true,
+        body: {
+          service_provider_sid,
+          account_sid,
+          tenant_fqdn: 'foo.bar.baz'
+        }
+      });
+      t.ok(result.statusCode === 201, 'successfully added ms teams tenant');
+  
+    }
+    catch (err) {
+      console.error(err);
+      t.end(err);
+    }  
+  }
+});
+

test/index.js

@@ -13,4 +13,5 @@ require('./ms-teams');
 require('./speech-credentials');
 require('./recent-calls');
 require('./webapp_tests');
+//require('./homer');
 require('./docker_stop');

test/oauth/gh-get-user.js

@@ -1,7 +1,6 @@
 const bent = require('bent');
 const getJSON = bent('GET', 200);
 const request = require('request');
-require('request-debug')(request);
 
 const test = async() => {
   request.get('https://api.github.com/user', {

test/postgresql/init-user-db.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+	CREATE DATABASE homer_config;
+EOSQL

test/recent-calls.js

@@ -66,6 +66,22 @@ test('recent calls tests', async(t) => {
       json: true,
     });
 
+    /* pull sip traces and pcap from homer */
+    /*
+    result = await request.get(`/Accounts/${account_sid}/RecentCalls/224f0f24-69aa-123a-eaa6-0ea24be4d211`, {
+      auth: authUser,
+      json: true
+    });
+    console.log(result);
+
+    const writeStream = fs.createWriteStream('./call.pcap');
+    const ret = await request.get(`/Accounts/${account_sid}/RecentCalls/224f0f24-69aa-123a-eaa6-0ea24be4d211/pcap`, {
+      auth: authUser,
+      resolveWithFullResponse: true
+    });
+    writeStream.write(ret.body);
+    */
+   
     await deleteObjectBySid(request, '/Accounts', account_sid);
     await deleteObjectBySid(request, '/ServiceProviders', service_provider_sid);
 

test/scenarios/uac.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<!DOCTYPE scenario SYSTEM "sipp.dtd">
+
+<scenario name="UAC with media">
+  <send retrans="500">
+    <![CDATA[
+
+      INVITE sip:+15083871234@echo.sip.jambonz.org SIP/2.0
+      Via: SIP/2.0/[transport] [local_ip]:[local_port];branch=[branch]
+      From: sipp <sip:sipp@[local_ip]:[local_port]>;tag=[pid]SIPpTag09[call_number]
+      To: <sip:15083871234@echo.sip.jambonz.org>
+      Call-ID: 685cd008-0a66-4974-b37a-bdd6d9a3c4a-0
+      CSeq: 1 INVITE
+      Contact: sip:sipp@[local_ip]:[local_port]
+      Max-Forwards: 70
+      Content-Type: application/sdp
+      Content-Length: [len]
+
+      v=0
+      o=user1 53655765 2353687637 IN IP[local_ip_type] [local_ip]
+      s=-
+      c=IN IP[local_ip_type] [local_ip]
+      t=0 0
+      m=audio [auto_media_port] RTP/AVP 8 101
+      a=rtpmap:8 PCMA/8000
+      a=rtpmap:101 telephone-event/8000
+      a=fmtp:101 0-11,16
+
+    ]]>
+  </send>
+
+  <recv response="100" optional="true">
+  </recv>
+
+
+  <!-- By adding rrs="true" (Record Route Sets), the route sets         -->
+  <!-- are saved and used for following messages sent. Useful to test   -->
+  <!-- against stateful SIP proxies/B2BUAs.                             -->
+  <recv response="503" rtd="true" crlf="true">
+  </recv>
+
+  <!-- Packet lost can be simulated in any send/recv message by         -->
+  <!-- by adding the 'lost = "10"'. Value can be [1-100] percent.       -->
+  <send>
+    <![CDATA[
+
+      ACK sip:15083871234@echo.sip.jambonz.org SIP/2.0
+      [last_Via]
+      From: sipp <sip:sipp@[local_ip]:[local_port]>;tag=[pid]SIPpTag09[call_number]
+      To: <sip:15083871234@echo.sip.jambonz.org>[peer_tag_param]
+      Call-ID: 685cd008-0a66-4974-b37a-bdd6d9a3c4a-0
+      CSeq: 1 ACK
+      Max-Forwards: 70
+      Subject: uac-pcap-carrier-max-call-limit
+      Content-Length: 0
+
+    ]]>
+  </send>
+
+
+  <!-- definition of the response time repartition table (unit is ms)   -->
+  <ResponseTimeRepartition value="10, 20, 30, 40, 50, 100, 150, 200"/>
+
+  <!-- definition of the call length repartition table (unit is ms)     -->
+  <CallLengthRepartition value="10, 50, 100, 500, 1000, 5000, 10000"/>
+
+</scenario>
+

test/serve-integration.js

@@ -1,4 +1,5 @@
 const exec = require('child_process').exec ;
+const { sippUac } = require('./sipp')('test_jambonz-api');
 let stopping = false;
 
 process.on('SIGINT', async() => {
@@ -66,6 +67,14 @@ const resetAdminPassword = () => {
   });
 };
 
+const generateSipTrace = async() => {
+  try {
+    await sippUac('uac.xml', '172.58.0.30');
+  } catch (err) {
+    console.log(err);
+  }
+};
+
 const stopDocker = () => {
   return new Promise((resolve, reject) => {
     console.log('stopping docker network..')
@@ -81,6 +90,7 @@ startDocker()
   .then(createSchema)
   .then(seedDb)
   .then(resetAdminPassword)
+  .then(generateSipTrace)
   .then(() => {
     console.log('ready for testing!');
     require('..');

test/sipp.js

@@ -0,0 +1,68 @@
+const { spawn } = require('child_process');
+const debug = require('debug')('jambonz:ci');
+let network;
+const obj = {};
+let output = '';
+let idx = 1;
+
+function clearOutput() {
+  output = '';
+}
+
+function addOutput(str) {
+  for (let i = 0; i < str.length; i++) {
+    if (str.charCodeAt(i) < 128) output += str.charAt(i);
+  }
+}
+
+module.exports = (networkName) => {
+  network = networkName ;
+  return obj;
+};
+
+obj.output = () => {
+  return output;
+};
+
+obj.sippUac = (file, bindAddress) => {
+  const cmd = 'docker';
+  const args = [
+    'run', '--rm', '--net', `${network}`,
+    '-v', `${__dirname}/scenarios:/tmp/scenarios`,
+    'drachtio/sipp', 'sipp', '-sf', `/tmp/scenarios/${file}`,
+    '-m', '1',
+    '-sleep', '250ms',
+    '-nostdin',
+    '-cid_str', `%u-%p@%s-${idx++}`,
+    'drachtio'
+  ];
+
+  if (bindAddress) args.splice(4, 0, '--ip', bindAddress);
+
+  //console.log(args.join(' '));
+  clearOutput();
+
+  return new Promise((resolve, reject) => {
+    const child_process = spawn(cmd, args, {stdio: ['inherit', 'pipe', 'pipe']});
+
+    child_process.on('exit', (code, signal) => {
+      if (code === 0) {
+        return resolve();
+      }
+      console.log(`sipp exited with non-zero code ${code} signal ${signal}`);
+      reject(code);
+    });
+    child_process.on('error', (error) => {
+      console.log(`error spawing child process for docker: ${args}`);
+    });
+
+    child_process.stdout.on('data', (data) => {
+      debug(`stderr: ${data}`);
+      addOutput(data.toString());
+    });
+    child_process.stderr.on('data', (data) => {
+      debug(`stderr: ${data}`);
+      addOutput(data.toString());
+    });
+  });
+};