Feature/retrieve registered users (#243)

* add GET /Accounts/:sid/RegisteredSipUsers

* fix vulnerabilities

.github/workflows/docker-publish-dbcreate.yml

@@ -2,16 +2,10 @@ name: Docker
 
 on:
   push:
-    # Publish `main` as Docker `latest` image.
     branches:
       - main
-
-    # Publish `v1.2.3` tags as releases.
     tags:
-      - v*
-
-env:
-  IMAGE_NAME: db-create
+      - '*'
 
 jobs:
   push:
@@ -20,32 +14,42 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build . --file Dockerfile.db-create --tag $IMAGE_NAME
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Push image
+      - name: prepare tag 
+        id: prepare_tag
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+            IMAGE_ID=jambonz/db-create
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
 
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
 
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "main" ] && VERSION=latest
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
 
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.db-create
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.github/workflows/docker-publish.yml

@@ -2,6 +2,8 @@ name: Docker
 
 on:
   push:
+    branches:
+      - main
     tags:
       - '*'
 
@@ -18,7 +20,7 @@ jobs:
       - name: prepare tag 
         id: prepare_tag
         run: |
-            IMAGE_ID=$GITHUB_REPOSITORY
+            IMAGE_ID=jambonz/api-server
 
             # Strip git ref prefix from version
             VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')

README.md

@@ -33,6 +33,8 @@ Configuration is provided via environment variables:
 |K8S| service running as kubernetes service |no|
 |K8S_FEATURE_SERVER_SERVICE_NAME| feature server name(required for K8S) |no|
 |K8S_FEATURE_SERVER_SERVICE_PORT| feature server port(required for K8S) |no|
+|JAMBONZ_RECORD_WS_USERNAME| recording websocket username|no|
+|JAMBONZ_RECORD_WS_PASSWORD| recording websocket password|no|
 
 #### Database dependency
 A mysql database is used to store long-lived objects such as Accounts, Applications, etc. To create the database schema, use or review the scripts in the 'db' folder, particularly:

app.js

@@ -8,12 +8,18 @@ const rateLimit = require('express-rate-limit');
 const cors = require('cors');
 const passport = require('passport');
 const routes = require('./lib/routes');
+const Registrar = require('@jambonz/mw-registrar');
 
 assert.ok(process.env.JAMBONES_MYSQL_HOST &&
   process.env.JAMBONES_MYSQL_USER &&
   process.env.JAMBONES_MYSQL_PASSWORD &&
   process.env.JAMBONES_MYSQL_DATABASE, 'missing JAMBONES_MYSQL_XXX env vars');
-assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+if (process.env.JAMBONES_REDIS_SENTINELS) {
+  assert.ok(process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+    'missing JAMBONES_REDIS_SENTINEL_MASTER_NAME env var, JAMBONES_REDIS_SENTINEL_PASSWORD env var is optional');
+} else {
+  assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+}
 assert.ok(process.env.JAMBONES_TIME_SERIES_HOST, 'missing JAMBONES_TIME_SERIES_HOST env var');
 assert.ok(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET, 'missing ENCRYPTION_SECRET env var');
 assert.ok(process.env.JWT_SECRET, 'missing JWT_SECRET env var');
@@ -30,20 +36,24 @@ const {
   logger, process.env.JAMBONES_TIME_SERIES_HOST
 );
 const {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
   retrieveKey,
   deleteKey,
-  incrKey
+  incrKey,
+  JAMBONES_REDIS_SENTINELS
 } = require('./lib/helpers/realtimedb-helpers');
 const {
-  getTtsVoices
-} = require('@jambonz/speech-utils')({
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache
+} = require('@jambonz/speech-utils')(JAMBONES_REDIS_SENTINELS || {
   host: process.env.JAMBONES_REDIS_HOST,
   port: process.env.JAMBONES_REDIS_PORT || 6379
 }, logger);
@@ -66,17 +76,19 @@ const {
 const PORT = process.env.HTTP_PORT || 3000;
 const authStrategy = require('./lib/auth')(logger, retrieveKey);
 const {delayLoginMiddleware} = require('./lib/middleware');
+const Websocket = require('ws');
 
 passport.use(authStrategy);
 
 app.locals = app.locals || {};
 app.locals = {
   ...app.locals,
+  registrar: new Registrar(logger, client),
   logger,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
@@ -84,6 +96,8 @@ app.locals = {
   retrieveKey,
   deleteKey,
   getTtsVoices,
+  getTtsSize,
+  purgeTtsCache,
   lookupAppBySid,
   lookupAccountBySid,
   lookupAccountByPhoneNumber,
@@ -114,6 +128,12 @@ const limiter = rateLimit({
   legacyHeaders: false, // Disable the `X-RateLimit-*` headers
 });
 
+// Setup websocket for recording audio
+const recordWsServer = require('./lib/record');
+const wsServer = new Websocket.Server({ noServer: true });
+wsServer.setMaxListeners(0);
+wsServer.on('connection', recordWsServer.bind(null, logger));
+
 if (process.env.JAMBONES_TRUST_PROXY) {
   const proxyCount = parseInt(process.env.JAMBONES_TRUST_PROXY);
   if (!isNaN(proxyCount) && proxyCount > 0) {
@@ -154,7 +174,52 @@ app.use((err, req, res, next) => {
   });
 });
 logger.info(`listening for HTTP traffic on port ${PORT}`);
-app.listen(PORT);
+const server = app.listen(PORT);
+
+
+const isValidWsKey = (hdr) => {
+  const username = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
+  const password = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
+  if (username && password) {
+    if (!hdr) {
+      // auth header is missing
+      return false;
+    }
+    const token = Buffer.from(`${username}:${password}`).toString('base64');
+    const arr = /^Basic (.*)$/.exec(hdr);
+    if (!Array.isArray(arr)) {
+      // malformed auth header
+      return false;
+    }
+    return arr[1] === token;
+  }
+  return true;
+};
+
+server.on('upgrade', (request, socket, head) => {
+  logger.debug({
+    url: request.url,
+    headers: request.headers,
+  }, 'received upgrade request');
+
+  /* verify the path starts with /transcribe */
+  if (!request.url.includes('/record/')) {
+    logger.info(`unhandled path: ${request.url}`);
+    return socket.write('HTTP/1.1 404 Not Found \r\n\r\n', () => socket.destroy());
+  }
+
+  /* verify the api key */
+  if (!isValidWsKey(request.headers['authorization'])) {
+    logger.info(`invalid auth header: ${request.headers['authorization'] || 'authorization header missing'}`);
+    return socket.write('HTTP/1.1 403 Forbidden \r\n\r\n', () => socket.destroy());
+  }
+
+  /* complete the upgrade */
+  wsServer.handleUpgrade(request, socket, head, (ws) => {
+    logger.info(`upgraded to websocket, url: ${request.url}`);
+    wsServer.emit('connection', ws, request.url);
+  });
+});
 
 // purge old calls from active call set every 10 mins
 async function purge() {

db/jambones-sql.sql

@@ -13,6 +13,8 @@ DROP TABLE IF EXISTS beta_invite_codes;
 
 DROP TABLE IF EXISTS call_routes;
 
+DROP TABLE IF EXISTS clients;
+
 DROP TABLE IF EXISTS dns_records;
 
 DROP TABLE IF EXISTS lcr;
@@ -127,6 +129,16 @@ application_sid CHAR(36) NOT NULL,
 PRIMARY KEY (call_route_sid)
 ) COMMENT='a regex-based pattern match for call routing';
 
+CREATE TABLE clients
+(
+client_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+is_active BOOLEAN NOT NULL DEFAULT 1,
+username VARCHAR(64),
+password VARCHAR(1024),
+PRIMARY KEY (client_sid)
+);
+
 CREATE TABLE dns_records
 (
 dns_record_sid CHAR(36) NOT NULL UNIQUE ,
@@ -322,6 +334,7 @@ last_tested DATETIME,
 tts_tested_ok BOOLEAN,
 stt_tested_ok BOOLEAN,
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+label VARCHAR(64),
 PRIMARY KEY (speech_credential_sid)
 );
 
@@ -411,7 +424,7 @@ PRIMARY KEY (smpp_gateway_sid)
 CREATE TABLE phone_numbers
 (
 phone_number_sid CHAR(36) UNIQUE ,
-number VARCHAR(132) NOT NULL UNIQUE ,
+number VARCHAR(132) NOT NULL,
 voip_carrier_sid CHAR(36),
 account_sid CHAR(36),
 application_sid CHAR(36),
@@ -424,11 +437,12 @@ CREATE TABLE sip_gateways
 sip_gateway_sid CHAR(36),
 ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
 netmask INTEGER NOT NULL DEFAULT 32,
-port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+port INTEGER COMMENT 'sip signaling port',
 inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+pad_crypto BOOLEAN NOT NULL DEFAULT 0,
 protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
@@ -466,9 +480,20 @@ app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
 speech_synthesis_voice VARCHAR(64),
+speech_synthesis_label VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
+speech_recognizer_label VARCHAR(64),
+use_for_fallback_speech BOOLEAN DEFAULT false,
+fallback_speech_synthesis_vendor VARCHAR(64),
+fallback_speech_synthesis_language VARCHAR(12),
+fallback_speech_synthesis_voice VARCHAR(64),
+fallback_speech_synthesis_label VARCHAR(64),
+fallback_speech_recognizer_vendor VARCHAR(64),
+fallback_speech_recognizer_language VARCHAR(64),
+fallback_speech_recognizer_label VARCHAR(64),
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
 ) COMMENT='A defined set of behaviors to be applied to phone calls ';
 
@@ -506,6 +531,9 @@ subspace_client_secret VARCHAR(255),
 subspace_sip_teleport_id VARCHAR(255),
 subspace_sip_teleport_destinations VARCHAR(255),
 siprec_hook_sid CHAR(36),
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
+record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
+bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
 PRIMARY KEY (account_sid)
 ) COMMENT='An enterprise that uses the platform for comm services';
 
@@ -526,6 +554,9 @@ ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERE
 
 ALTER TABLE call_routes ADD FOREIGN KEY application_sid_idxfk (application_sid) REFERENCES applications (application_sid);
 
+CREATE INDEX client_sid_idx ON clients (client_sid);
+ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid);
+
 CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
 ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
 
@@ -590,8 +621,6 @@ CREATE INDEX smpp_address_sid_idx ON smpp_addresses (smpp_address_sid);
 CREATE INDEX service_provider_sid_idx ON smpp_addresses (service_provider_sid);
 ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-CREATE UNIQUE INDEX speech_credentials_idx_1 ON speech_credentials (vendor,account_sid);
-
 CREATE INDEX speech_credential_sid_idx ON speech_credentials (speech_credential_sid);
 CREATE INDEX service_provider_sid_idx ON speech_credentials (service_provider_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
@@ -628,6 +657,8 @@ CREATE INDEX smpp_gateway_sid_idx ON smpp_gateways (smpp_gateway_sid);
 CREATE INDEX voip_carrier_sid_idx ON smpp_gateways (voip_carrier_sid);
 ALTER TABLE smpp_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
+CREATE UNIQUE INDEX phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid);
+
 CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
 CREATE INDEX number_idx ON phone_numbers (number);
 CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);
@@ -682,5 +713,4 @@ ALTER TABLE accounts ADD FOREIGN KEY queue_event_hook_sid_idxfk (queue_event_hoo
 ALTER TABLE accounts ADD FOREIGN KEY device_calling_application_sid_idxfk (device_calling_application_sid) REFERENCES applications (application_sid);
 
 ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid);
-
-SET FOREIGN_KEY_CHECKS=1;
+SET FOREIGN_KEY_CHECKS=1;

db/jambones.sqs

@@ -87,7 +87,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[49E56AF4-4E40-49B6-BA88-4E378F1E6C18]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[11]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[12]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[0507BD46-ACAC-48A3-841E-4DEC2FEDCB72]]></uid>
     </SQLTable>
@@ -148,7 +148,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[6E651E52-F91E-4086-9A1E-FB3425476B2F]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[21]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[22]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[0A95311F-47FA-429F-BAF9-1442C6EE0C0E]]></uid>
     </SQLTable>
@@ -225,7 +225,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[D1019218-F1FC-4BC5-A890-F8DBB7153375]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[14]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[15]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[0AC2FD20-B22F-48DB-8611-801CEF6BFA12]]></uid>
     </SQLTable>
@@ -279,7 +279,7 @@
             <uid><![CDATA[755D10B0-F60D-4250-8971-C8E4FDB0E0CD]]></uid>
             <unique><![CDATA[1]]></unique>
         </SQLField>
-        <labelWindowIndex><![CDATA[18]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[19]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[16B9E383-E044-4D71-AB46-FEB86A46A298]]></uid>
     </SQLTable>
@@ -316,7 +316,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[C4EEBFF0-C3CB-4897-8720-12D14DBA93A5]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[5]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[6]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[1A80FB9B-419E-483A-86FF-B44A00A44D7F]]></uid>
     </SQLTable>
@@ -471,7 +471,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[601FA05B-78A5-4E7E-9983-39BB0E6D18EB]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[25]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[26]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[2A735FAB-592C-42E5-9C8B-06B109314799]]></uid>
     </SQLTable>
@@ -537,7 +537,7 @@
             <indexed><![CDATA[1]]></indexed>
             <uid><![CDATA[365FB018-429D-4DA4-AC33-D9D106EA97E5]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[10]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[11]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[311D99B2-DC8B-4C4A-A1FC-4AFAA1F450F3]]></uid>
     </SQLTable>
@@ -737,7 +737,7 @@
             <type><![CDATA[VARCHAR(4096)]]></type>
             <uid><![CDATA[7C7DFE92-D7AC-4447-A1C2-E0F10C1EA26A]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[26]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[27]]></labelWindowIndex>
         <objectComment><![CDATA[A Carrier or customer PBX that can send or receive calls]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[3D3136A7-AFC0-4A70-AEC3-68577955CA2E]]></uid>
@@ -819,7 +819,7 @@
             <defaultValue><![CDATA[CURRENT_TIMESTAMP]]></defaultValue>
             <uid><![CDATA[C84C9B6A-80B5-4B0B-8C14-EB02F7421BBE]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[35]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[36]]></labelWindowIndex>
         <objectComment><![CDATA[An authorization token that is used to access the REST api]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[3EDF89A0-FD38-4DF9-BB65-E0FCD0A678BE]]></uid>
@@ -872,7 +872,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[EA4C1A7E-68ED-41D5-9EE9-345DD61F00C7]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[7]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[8]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[4893A0F0-BE1B-4322-9034-644528E802DE]]></uid>
     </SQLTable>
@@ -884,7 +884,7 @@
             <y>958.00</y>
         </location>
         <size>
-            <width>368.00</width>
+            <width>302.00</width>
             <height>280.00</height>
         </size>
         <zorder>14</zorder>
@@ -981,25 +981,12 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[8860648C-4790-4A01-9E2E-60DC52A287FA]]></uid>
         </SQLField>
-        <SQLIndex>
-            <name><![CDATA[speech_credentials_idx_1]]></name>
-            <fieldName><![CDATA[vendor]]></fieldName>
-            <fieldName><![CDATA[account_sid]]></fieldName>
-            <SQLIndexEntry>
-                <name><![CDATA[vendor]]></name>
-                <prefixSize><![CDATA[]]></prefixSize>
-                <fieldUid><![CDATA[9D8FCF55-D68E-44D3-90DF-27B5ABD1D0BE]]></fieldUid>
-            </SQLIndexEntry>
-            <SQLIndexEntry>
-                <name><![CDATA[account_sid]]></name>
-                <prefixSize><![CDATA[]]></prefixSize>
-                <fieldUid><![CDATA[7E964ED2-EC2E-4BCB-8DEC-C455B87FAC07]]></fieldUid>
-            </SQLIndexEntry>
-            <indexNamePrefix><![CDATA[speech_credentials]]></indexNamePrefix>
-            <indexType><![CDATA[UNIQUE]]></indexType>
-            <uid><![CDATA[554ABEC2-3E1B-41B1-BF07-25F403D5E3B4]]></uid>
-        </SQLIndex>
-        <labelWindowIndex><![CDATA[20]]></labelWindowIndex>
+        <SQLField>
+            <name><![CDATA[label]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <uid><![CDATA[0D42A22C-DF14-42A1-BDE2-A53AC8B0D8D6]]></uid>
+        </SQLField>
+        <labelWindowIndex><![CDATA[21]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[49A68E1C-DEE2-446C-A4EB-9850E16155CC]]></uid>
     </SQLTable>
@@ -1020,7 +1007,7 @@
             <type><![CDATA[VARCHAR(16)]]></type>
             <uid><![CDATA[1EA572BD-FF6B-43CC-9EBB-33A735781429]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[8]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[9]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[518AC592-D3E6-4032-8A33-15A3DB72B060]]></uid>
     </SQLTable>
@@ -1081,7 +1068,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[6B2F726C-48A6-49D9-B7B1-8850DD6FB3EC]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[13]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[14]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[51A02EFE-AA51-46EF-8671-E8B2F1FC5F8D]]></uid>
     </SQLTable>
@@ -1133,7 +1120,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[2EA3A57F-7EF7-4958-B06B-62B0279BB87E]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[6]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[7]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[5784AC2F-BEBC-466F-9818-F9A7D227A5B5]]></uid>
     </SQLTable>
@@ -1183,7 +1170,7 @@
             <type><![CDATA[VARCHAR(255)]]></type>
             <uid><![CDATA[04BB457A-D532-4780-8A58-5900094171EC]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[27]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[28]]></labelWindowIndex>
         <objectComment><![CDATA[An HTTP callback]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[64D64CB9-0990-4C68-BE71-F9FD43C2BE19]]></uid>
@@ -1283,7 +1270,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[9A75A20B-1EFD-4E16-994A-5376C650EAB5]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[9]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[10]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[6511AF7D-91FD-40C7-9F73-B8E9E66DC249]]></uid>
     </SQLTable>
@@ -1352,7 +1339,7 @@
             <uid><![CDATA[9B4208B5-9E3B-4B76-B7F7-4E5D36B99BF2]]></uid>
             <unsigned><![CDATA[0]]></unsigned>
         </SQLField>
-        <labelWindowIndex><![CDATA[34]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[35]]></labelWindowIndex>
         <objectComment><![CDATA[a regex-based pattern match for call routing]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[78584D93-2CD7-4495-9C5E-893C7B869133]]></uid>
@@ -1389,7 +1376,7 @@
             <noQuoteDefault><![CDATA[1]]></noQuoteDefault>
             <uid><![CDATA[4D2F7B02-F183-4239-8CE8-3E98206708AE]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[12]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[13]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[86FAB0AB-DC68-4ADF-8A08-BBAF61BA1840]]></uid>
     </SQLTable>
@@ -1427,7 +1414,7 @@
             <type><![CDATA[VARCHAR(255)]]></type>
             <uid><![CDATA[673137EA-B74C-4BA7-AD25-1B71360A2E26]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[4]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[5]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[87F254ED-D381-48E3-8E8F-C0F3D99CC01C]]></uid>
     </SQLTable>
@@ -1459,7 +1446,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[8998AAD6-A21C-4697-9660-8DC5005AED07]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[3]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[4]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[8E75DB2D-9078-40E6-88BF-7DDED5033362]]></uid>
     </SQLTable>
@@ -1536,7 +1523,7 @@
             <uid><![CDATA[1DDAD1A1-942D-4487-89C8-D496B7F82274]]></uid>
             <unique><![CDATA[1]]></unique>
         </SQLField>
-        <labelWindowIndex><![CDATA[24]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[25]]></labelWindowIndex>
         <objectComment><![CDATA[A Microsoft Teams customer tenant]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[92FD042A-5AEC-4D8F-AB94-C73C0F566F75]]></uid>
@@ -1607,7 +1594,7 @@
             <objectComment><![CDATA[lower priority carriers are attempted first]]></objectComment>
             <uid><![CDATA[01F61C68-799B-49B0-9E6A-0E2162EE5A54]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[30]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[31]]></labelWindowIndex>
         <objectComment><![CDATA[An entry in the LCR routing list]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[956025F5-0798-47F7-B76C-457814C7B52E]]></uid>
@@ -1622,7 +1609,7 @@
         </location>
         <size>
             <width>318.00</width>
-            <height>440.00</height>
+            <height>500.00</height>
         </size>
         <zorder>4</zorder>
         <SQLField>
@@ -1800,7 +1787,28 @@
             <referencesTableUID><![CDATA[E97EE4F0-7ED7-4E8C-862E-D98192D6EAE0]]></referencesTableUID>
             <uid><![CDATA[4B8283B4-5E16-4846-A79D-12C6B2E73C86]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[32]]></labelWindowIndex>
+        <SQLField>
+            <name><![CDATA[record_all_calls]]></name>
+            <type><![CDATA[BOOLEAN]]></type>
+            <defaultValue><![CDATA[false]]></defaultValue>
+            <notNull><![CDATA[1]]></notNull>
+            <uid><![CDATA[F4154533-AE5B-48FE-9435-A37FA2632752]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[record_format]]></name>
+            <type><![CDATA[VARCHAR(16)]]></type>
+            <defaultValue><![CDATA[mp3]]></defaultValue>
+            <notNull><![CDATA[1]]></notNull>
+            <uid><![CDATA[F03BFF28-69EA-4555-9E5E-B0CA7B095408]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[bucket_credential]]></name>
+            <type><![CDATA[VARCHAR(8192)]]></type>
+            <forcedUnique><![CDATA[0]]></forcedUnique>
+            <objectComment><![CDATA[credential used to authenticate with storage service]]></objectComment>
+            <uid><![CDATA[E81859C0-DCBD-4FF3-BEEF-FA575394326B]]></uid>
+        </SQLField>
+        <labelWindowIndex><![CDATA[33]]></labelWindowIndex>
         <objectComment><![CDATA[An enterprise that uses the platform for comm services]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[985D6997-B1A7-4AB3-80F4-4D59B45480C8]]></uid>
@@ -1839,7 +1847,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[D0BF7D36-E40C-4385-9BA5-2099B49A1042]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[19]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[20]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[A8ED2178-3CC4-4174-A8FC-C2B58FD28214]]></uid>
     </SQLTable>
@@ -1931,7 +1939,7 @@
             <type><![CDATA[VARCHAR(32)]]></type>
             <uid><![CDATA[CE2015BC-8538-4FB0-B4D9-454436FAB1D9]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[15]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[16]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[AF34726D-EDFD-414E-9B44-5243DA9D9497]]></uid>
     </SQLTable>
@@ -2001,7 +2009,7 @@
             <indexed><![CDATA[1]]></indexed>
             <uid><![CDATA[EA936343-89D3-4E3F-BD92-9F8967DC27C4]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[2]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[3]]></labelWindowIndex>
         <objectComment><![CDATA[An LCR (least cost routing) table that is used by a service provider or account to make decisions about routing outbound calls when multiple carriers are available.]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[AFD51FD4-44C8-4442-94C2-0DFA93CD48AE]]></uid>
@@ -2010,8 +2018,8 @@
         <name><![CDATA[dns_records]]></name>
         <schema><![CDATA[]]></schema>
         <location>
-            <x>960.00</x>
-            <y>1155.00</y>
+            <x>1270.00</x>
+            <y>1425.00</y>
         </location>
         <size>
             <width>262.00</width>
@@ -2053,7 +2061,7 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[73092A7A-9F3F-4C49-8478-39CE5DAF5ADD]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[17]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[18]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[B10C0DE3-03CD-4C5A-B1FB-F9F81ED14A40]]></uid>
     </SQLTable>
@@ -2066,8 +2074,8 @@
             <y>1128.00</y>
         </location>
         <size>
-            <width>265.00</width>
-            <height>140.00</height>
+            <width>522.00</width>
+            <height>160.00</height>
         </size>
         <zorder>2</zorder>
         <SQLField>
@@ -2087,7 +2095,7 @@
             <indexed><![CDATA[1]]></indexed>
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[159B82ED-C6B0-4FC6-957B-5C354AF9E783]]></uid>
-            <unique><![CDATA[1]]></unique>
+            <unique><![CDATA[0]]></unique>
         </SQLField>
         <SQLField>
             <name><![CDATA[voip_carrier_sid]]></name>
@@ -2149,11 +2157,87 @@
             <objectComment><![CDATA[if not null, this number is a test number for the associated service provider]]></objectComment>
             <uid><![CDATA[D2D46B75-F9C7-42A5-8D8C-4A8412C75ECA]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[33]]></labelWindowIndex>
+        <SQLIndex>
+            <name><![CDATA[phone_numbers_unique_idx_voip_carrier_number]]></name>
+            <fieldName><![CDATA[number]]></fieldName>
+            <fieldName><![CDATA[voip_carrier_sid]]></fieldName>
+            <SQLIndexEntry>
+                <name><![CDATA[number]]></name>
+                <prefixSize><![CDATA[]]></prefixSize>
+                <fieldUid><![CDATA[159B82ED-C6B0-4FC6-957B-5C354AF9E783]]></fieldUid>
+            </SQLIndexEntry>
+            <SQLIndexEntry>
+                <name><![CDATA[voip_carrier_sid]]></name>
+                <prefixSize><![CDATA[]]></prefixSize>
+                <fieldUid><![CDATA[58A047B9-C1C9-4697-9FE8-08E3BFF91660]]></fieldUid>
+            </SQLIndexEntry>
+            <indexNamePrefix><![CDATA[phone_numbers]]></indexNamePrefix>
+            <indexType><![CDATA[UNIQUE]]></indexType>
+            <uid><![CDATA[4E84523A-7F30-4A5D-A0A8-578652102BD0]]></uid>
+        </SQLIndex>
+        <labelWindowIndex><![CDATA[34]]></labelWindowIndex>
         <objectComment><![CDATA[A phone number that has been assigned to an account]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[BA650DDC-AC7B-4DFE-A5E5-828C75607807]]></uid>
     </SQLTable>
+    <SQLTable>
+        <name><![CDATA[clients]]></name>
+        <schema><![CDATA[]]></schema>
+        <location>
+            <x>974.00</x>
+            <y>1496.00</y>
+        </location>
+        <size>
+            <width>228.00</width>
+            <height>120.00</height>
+        </size>
+        <zorder>36</zorder>
+        <SQLField>
+            <name><![CDATA[client_sid]]></name>
+            <type><![CDATA[CHAR(36)]]></type>
+            <primaryKey>1</primaryKey>
+            <indexed><![CDATA[1]]></indexed>
+            <notNull><![CDATA[1]]></notNull>
+            <uid><![CDATA[0CD7B8BF-0C4C-49CC-8A9B-27E03C70255D]]></uid>
+            <unique><![CDATA[1]]></unique>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[account_sid]]></name>
+            <type><![CDATA[CHAR(36)]]></type>
+            <referencesField>account_sid</referencesField>
+            <referencesTable>accounts</referencesTable>
+            <referencesField><![CDATA[account_sid]]></referencesField>
+            <referencesTable><![CDATA[accounts]]></referencesTable>
+            <sourceCardinality>4</sourceCardinality>
+            <destinationCardinality>1</destinationCardinality>
+            <referencesFieldUID><![CDATA[1342FAFA-C15C-429B-809B-C6C55F9FA5B6]]></referencesFieldUID>
+            <referencesTableUID><![CDATA[985D6997-B1A7-4AB3-80F4-4D59B45480C8]]></referencesTableUID>
+            <foreignKeyName><![CDATA[account_sid_idxfk_13]]></foreignKeyName>
+            <notNull><![CDATA[1]]></notNull>
+            <uid><![CDATA[AEB7E64F-A890-483D-946E-B0EFC640B017]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[is_active]]></name>
+            <type><![CDATA[BOOLEAN]]></type>
+            <defaultValue><![CDATA[1]]></defaultValue>
+            <notNull><![CDATA[1]]></notNull>
+            <uid><![CDATA[EEE9DF14-8CA6-40BA-9737-C37D702FCB92]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[username]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <uid><![CDATA[9F6D877B-3985-4DF0-9E42-41FF87140C05]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[password]]></name>
+            <type><![CDATA[VARCHAR(1024)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[2963DB28-7248-4D58-92E6-F21F2EB9E680]]></uid>
+        </SQLField>
+        <labelWindowIndex><![CDATA[1]]></labelWindowIndex>
+        <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
+        <uid><![CDATA[C2CDB45C-3F50-40B3-B8CE-2F093EB0D517]]></uid>
+    </SQLTable>
     <SQLTable>
         <name><![CDATA[sip_gateways]]></name>
         <schema><![CDATA[]]></schema>
@@ -2164,7 +2248,7 @@
         </location>
         <size>
             <width>281.00</width>
-            <height>220.00</height>
+            <height>240.00</height>
         </size>
         <zorder>7</zorder>
         <SQLField>
@@ -2190,8 +2274,7 @@
         <SQLField>
             <name><![CDATA[port]]></name>
             <type><![CDATA[INTEGER]]></type>
-            <defaultValue><![CDATA[5060]]></defaultValue>
-            <notNull><![CDATA[1]]></notNull>
+            <notNull><![CDATA[0]]></notNull>
             <objectComment><![CDATA[sip signaling port]]></objectComment>
             <uid><![CDATA[26B20F1E-4DB0-48C0-90F7-CA90A06A1070]]></uid>
         </SQLField>
@@ -2232,6 +2315,13 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[27D4A5BD-8093-4ADD-B5B5-D546844206F9]]></uid>
         </SQLField>
+        <SQLField>
+            <name><![CDATA[pad_crypto]]></name>
+            <type><![CDATA[BOOLEAN]]></type>
+            <defaultValue><![CDATA[0]]></defaultValue>
+            <notNull><![CDATA[1]]></notNull>
+            <uid><![CDATA[C5C0043B-100A-4476-BF01-BE0777AE27C0]]></uid>
+        </SQLField>
         <SQLIndex>
             <name><![CDATA[sip_gateway_idx_hostport]]></name>
             <fieldName><![CDATA[ipv4]]></fieldName>
@@ -2256,7 +2346,7 @@
             <objectComment><![CDATA[Outbound call protocol]]></objectComment>
             <uid><![CDATA[30661D66-96EC-4B02-995C-5E7EB8A3BD70]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[29]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[30]]></labelWindowIndex>
         <objectComment><![CDATA[A whitelisted sip gateway used for origination/termination]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[D8A564E2-DA41-4217-8ACE-06CF77E9BEC1]]></uid>
@@ -2266,12 +2356,12 @@
         <schema><![CDATA[]]></schema>
         <comment><![CDATA[A defined set of behaviors to be applied to phone calls ]]></comment>
         <location>
-            <x>841.00</x>
-            <y>824.00</y>
+            <x>840.00</x>
+            <y>917.00</y>
         </location>
         <size>
             <width>345.00</width>
-            <height>320.00</height>
+            <height>540.00</height>
         </size>
         <zorder>0</zorder>
         <SQLField>
@@ -2390,6 +2480,12 @@
             <notNull><![CDATA[0]]></notNull>
             <uid><![CDATA[929D66F0-64B9-4D7C-AB4B-24F131E1178F]]></uid>
         </SQLField>
+        <SQLField>
+            <name><![CDATA[speech_synthesis_label]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[BFA24DF2-9CF5-47B0-848D-8B685B7C6750]]></uid>
+        </SQLField>
         <SQLField>
             <name><![CDATA[speech_recognizer_vendor]]></name>
             <type><![CDATA[VARCHAR(64)]]></type>
@@ -2404,14 +2500,76 @@
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[A03AFB7B-492F-48E3-AE3C-B1416D5B6B12]]></uid>
         </SQLField>
+        <SQLField>
+            <name><![CDATA[speech_recognizer_label]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[A247A784-CCD6-40B4-9D0A-2F0EF8F8AFD2]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[use_for_fallback_speech]]></name>
+            <type><![CDATA[BOOLEAN]]></type>
+            <defaultValue><![CDATA[false]]></defaultValue>
+            <uid><![CDATA[DDA48DD6-4B0F-4AD5-9B32-D508BBA1A8EE]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[fallback_speech_synthesis_vendor]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[26BBDEEF-E179-4280-9917-6F2BD6367459]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[fallback_speech_synthesis_language]]></name>
+            <type><![CDATA[VARCHAR(12)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[E008D6D7-9BB7-4372-8B46-F92C0EB15082]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[fallback_speech_synthesis_voice]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[6A0E92C9-32B9-4179-A893-3DADF5DD7728]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[fallback_speech_synthesis_label]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[8576DEF6-D81A-4D4D-8980-00580779D164]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[fallback_speech_recognizer_vendor]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[14ECF5EA-81C5-4EAE-9575-9785CEB672E6]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[fallback_speech_recognizer_language]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[EC792500-6B2B-4E54-AA89-43E7A0FD8642]]></uid>
+        </SQLField>
+        <SQLField>
+            <name><![CDATA[fallback_speech_recognizer_label]]></name>
+            <type><![CDATA[VARCHAR(64)]]></type>
+            <notNull><![CDATA[0]]></notNull>
+            <uid><![CDATA[65AA5173-6523-49F7-9D95-78C4B3A7C7E6]]></uid>
+        </SQLField>
         <SQLField>
             <name><![CDATA[created_at]]></name>
             <type><![CDATA[DATETIME]]></type>
             <defaultValue><![CDATA[CURRENT_TIMESTAMP]]></defaultValue>
+            <forcedUnique><![CDATA[0]]></forcedUnique>
             <noQuoteDefault><![CDATA[1]]></noQuoteDefault>
             <notNull><![CDATA[1]]></notNull>
             <uid><![CDATA[C09B1BDB-8390-4B8A-B70A-642EC5E12899]]></uid>
         </SQLField>
+        <SQLField>
+            <name><![CDATA[record_all_calls]]></name>
+            <type><![CDATA[BOOLEAN]]></type>
+            <defaultValue><![CDATA[false]]></defaultValue>
+            <notNull><![CDATA[1]]></notNull>
+            <uid><![CDATA[0AAA3997-9E40-47CC-976D-5EC1B82AD290]]></uid>
+        </SQLField>
         <SQLIndex>
             <name><![CDATA[applications_idx_name]]></name>
             <fieldName><![CDATA[account_sid]]></fieldName>
@@ -2430,7 +2588,7 @@
             <indexType><![CDATA[UNIQUE]]></indexType>
             <uid><![CDATA[3FDDDF3B-375D-4DE4-B759-514438845F7D]]></uid>
         </SQLIndex>
-        <labelWindowIndex><![CDATA[31]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[32]]></labelWindowIndex>
         <objectComment><![CDATA[A defined set of behaviors to be applied to phone calls ]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[E97EE4F0-7ED7-4E8C-862E-D98192D6EAE0]]></uid>
@@ -2538,7 +2696,7 @@
             <type><![CDATA[VARBINARY(52)]]></type>
             <uid><![CDATA[B4793720-635C-4E25-A306-62E7416541C4]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[16]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[17]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[EB4BC5F9-CC10-4C8C-AB31-6D942256AEFB]]></uid>
     </SQLTable>
@@ -2569,7 +2727,7 @@
             <type><![CDATA[VARCHAR(255)]]></type>
             <uid><![CDATA[0A8DB34E-76C9-4D40-9E31-786E0228DCEE]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[1]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[2]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[F0C2DC80-CBA7-4BE7-8856-D10B062A7B17]]></uid>
     </SQLTable>
@@ -2654,7 +2812,7 @@
             <type><![CDATA[DATETIME]]></type>
             <uid><![CDATA[CD43B91B-F34E-4422-9C0F-A4B92E2E7B95]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[23]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[24]]></labelWindowIndex>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[F0EE651E-DBF6-4CAC-A517-AC85BCC2D3AF]]></uid>
     </SQLTable>
@@ -2714,7 +2872,7 @@
             <uid><![CDATA[B73773BA-AB1B-47AA-B995-2D2FE006198F]]></uid>
             <unique><![CDATA[0]]></unique>
         </SQLField>
-        <labelWindowIndex><![CDATA[28]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[29]]></labelWindowIndex>
         <objectComment><![CDATA[An ordered list of  digit patterns in an LCR table.  The patterns are tested in sequence until one matches]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[F283D572-F670-4571-91FD-A665A9D3E15D]]></uid>
@@ -2781,7 +2939,7 @@
             <type><![CDATA[VARCHAR(255)]]></type>
             <uid><![CDATA[FA39B463-61C7-4654-BE9C-D1AC39AB1B97]]></uid>
         </SQLField>
-        <labelWindowIndex><![CDATA[22]]></labelWindowIndex>
+        <labelWindowIndex><![CDATA[23]]></labelWindowIndex>
         <objectComment><![CDATA[A partition of the platform used by one service provider]]></objectComment>
         <ui.treeExpanded><![CDATA[1]]></ui.treeExpanded>
         <uid><![CDATA[F294B51E-F867-47CA-BC1F-F70BDF8170FF]]></uid>
@@ -2855,17 +3013,17 @@
         <overviewPanelHidden><![CDATA[0]]></overviewPanelHidden>
         <pageBoundariesVisible><![CDATA[0]]></pageBoundariesVisible>
         <PageGridVisible><![CDATA[0]]></PageGridVisible>
-        <RightSidebarWidth><![CDATA[1873.000000]]></RightSidebarWidth>
+        <RightSidebarWidth><![CDATA[1235.000000]]></RightSidebarWidth>
         <sidebarIndex><![CDATA[2]]></sidebarIndex>
         <snapToGrid><![CDATA[0]]></snapToGrid>
-        <SourceSidebarWidth><![CDATA[312.000000]]></SourceSidebarWidth>
+        <SourceSidebarWidth><![CDATA[0.000000]]></SourceSidebarWidth>
         <SQLEditorFileFormatVersion><![CDATA[4]]></SQLEditorFileFormatVersion>
         <uid><![CDATA[58C99A00-06C9-478C-A667-C63842E088F3]]></uid>
         <windowHeight><![CDATA[1055.000000]]></windowHeight>
-        <windowLocationX><![CDATA[1728.000000]]></windowLocationX>
-        <windowLocationY><![CDATA[37.000000]]></windowLocationY>
-        <windowScrollOrigin><![CDATA[{0, 0}]]></windowScrollOrigin>
-        <windowWidth><![CDATA[1874.000000]]></windowWidth>
+        <windowLocationX><![CDATA[0.000000]]></windowLocationX>
+        <windowLocationY><![CDATA[24.000000]]></windowLocationY>
+        <windowScrollOrigin><![CDATA[{90, 0}]]></windowScrollOrigin>
+        <windowWidth><![CDATA[1682.000000]]></windowWidth>
     </SQLDocumentInfo>
     <AllowsIndexRenamingOnInsert><![CDATA[1]]></AllowsIndexRenamingOnInsert>
     <defaultLabelExpanded><![CDATA[1]]></defaultLabelExpanded>

db/seed-integration-test.sql

@@ -22,7 +22,7 @@ values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9
 
 -- create one service provider and one account
 insert into service_providers (service_provider_sid, name, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.us');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.cloud');
 
 insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
 values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'wh_secret_cJqgtMDPzDhhnjmaJH6Mtk');
@@ -38,9 +38,9 @@ values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fs', '38700987-c7a4-4685-a5bb-af378f9
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES

db/seed-production-database-open-source.sql

@@ -24,10 +24,9 @@ values ('09e92f3c-9d73-4303-b63f-3668574862ce', '1cf2f4f4-64c4-4249-9a3e-5bb4cb5
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
-
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES
 ('7087fe50-8acb-4f3b-b820-97b573723aab', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'hello world', 'd31568d0-b193-4a05-8ff6-778369bc6efe', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US'),
@@ -68,9 +67,6 @@ VALUES
 ('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
 ('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
 ('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
-('48b108e3-1ce7-4f18-a4cb-e41e63688bdf', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.193', 30, 5060, 1, 0),
-('d9131a69-fe44-4c2a-ba82-4adc81f628dd', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.194', 30, 5060, 1, 0),
-('34a6a311-4bd6-49ca-aa77-edd3cb92c6e1', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.195', 30, 5060, 1, 0),
 ('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
 ('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),

db/upgrade-jambonz-db.js

@@ -138,6 +138,43 @@ const sql = {
     'CREATE INDEX account_sid_idx ON lcr (account_sid)',
     'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY lcr_route_sid_idxfk (lcr_route_sid) REFERENCES lcr_routes (lcr_route_sid)',
     'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_3 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid)',
+    'SET FOREIGN_KEY_CHECKS=1',
+  ],
+  8004: [
+    'alter table accounts add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table accounts add column bucket_credential VARCHAR(8192)',
+    'alter table accounts add column record_format VARCHAR(16) NOT NULL DEFAULT \'mp3\'',
+    'alter table applications add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table phone_numbers DROP INDEX number',
+    'create unique index phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid)',
+    `CREATE TABLE clients
+    (
+    client_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    username VARCHAR(64),
+    password VARCHAR(1024),
+    PRIMARY KEY (client_sid)
+    )`,
+    'CREATE INDEX client_sid_idx ON clients (client_sid)',
+    'ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid)',
+    'ALTER TABLE sip_gateways ADD COLUMN protocol ENUM(\'udp\',\'tcp\',\'tls\', \'tls/srtp\') DEFAULT \'udp\''
+  ],
+  8005: [
+    'DROP INDEX speech_credentials_idx_1 ON speech_credentials',
+    'ALTER TABLE speech_credentials ADD COLUMN label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN use_for_fallback_speech BOOLEAN DEFAULT false',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_language VARCHAR(12)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_voice VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_language VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE sip_gateways ADD COLUMN pad_crypto BOOLEAN NOT NULL DEFAULT 0',
+    'ALTER TABLE sip_gateways MODIFY port INTEGER'
   ]
 };
 
@@ -168,6 +205,8 @@ const doIt = async() => {
         if (val < 7007) upgrades.push(...sql['7007']);
         if (val < 8000) upgrades.push(...sql['8000']);
         if (val < 8003) upgrades.push(...sql['8003']);
+        if (val < 8004) upgrades.push(...sql['8004']);
+        if (val < 8005) upgrades.push(...sql['8005']);
 
         // perform all upgrades
         logger.info({upgrades}, 'applying schema upgrades..');

db/webapp-tests.sql

@@ -2,7 +2,7 @@ SET FOREIGN_KEY_CHECKS=0;
 
 -- create one service provider
 insert into service_providers (service_provider_sid, name, description, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.us', 'jambonz.us service provider', 'sip.yakeeda.com');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.cloud', 'jambonz.cloud service provider', 'sip.yakeeda.com');
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
 
@@ -19,8 +19,8 @@ insert into sip_gateways (sip_gateway_sid, voip_carrier_sid, ipv4, port, inbound
 values ('46b727eb-c7dc-44fa-b063-96e48d408e4a', '5145b436-2f38-4029-8d4c-fd8c67831c7a', '3.3.3.3', 5060, 1, 1, 1);
 
 -- create the test application and test phone number
-insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.us/testCall', 'POST');
-insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.us/callStatus', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.cloud/testCall', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.cloud/callStatus', 'POST');
 insert into applications (application_sid, name, service_provider_sid, call_hook_sid, call_status_hook_sid, 
 speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 values ('7a489343-02ed-471e-8df0-fc5e1b98ce8f', 'Test application', '2708b1b3-2736-40ea-b502-c53d8396247f', 

lib/helpers/realtimedb-helpers.js

@@ -1,10 +1,35 @@
 const logger = require('../logger');
 
+const JAMBONES_REDIS_SENTINELS = process.env.JAMBONES_REDIS_SENTINELS ? {
+  sentinels: process.env.JAMBONES_REDIS_SENTINELS.split(',').map((sentinel) => {
+    let host, port = 26379;
+    if (sentinel.includes(':')) {
+      const arr = sentinel.split(':');
+      host = arr[0];
+      port = parseInt(arr[1], 10);
+    } else {
+      host = sentinel;
+    }
+    return {host, port};
+  }),
+  name: process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+  ...(process.env.JAMBONES_REDIS_SENTINEL_PASSWORD && {
+    password: process.env.JAMBONES_REDIS_SENTINEL_PASSWORD
+  }),
+  ...(process.env.JAMBONES_REDIS_SENTINEL_USERNAME && {
+    username: process.env.JAMBONES_REDIS_SENTINEL_USERNAME
+  }),
+  ...(process.env.JAMBONES_REDIS_SENTINEL_SENTINAL_PASSWORD && {
+    sentinelPassword: process.env.JAMBONES_REDIS_SENTINEL_SENTINAL_PASSWORD
+  }),
+} : null;
+
 const {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
@@ -12,21 +37,23 @@ const {
   deleteKey,
   incrKey,
   client: redisClient,
-} = require('@jambonz/realtimedb-helpers')({
+} = require('@jambonz/realtimedb-helpers')(JAMBONES_REDIS_SENTINELS || {
   host: process.env.JAMBONES_REDIS_HOST || 'localhost',
   port: process.env.JAMBONES_REDIS_PORT || 6379
 }, logger);
 
 module.exports = {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
   retrieveKey,
   deleteKey,
   redisClient,
-  incrKey
+  incrKey,
+  JAMBONES_REDIS_SENTINELS
 };

lib/logger.js

@@ -1,11 +1,7 @@
-const opts = Object.assign({
-  timestamp: () => {
-    return `, "time": "${new Date().toISOString()}"`;
-  }
-}, {
+const opts = {
   level: process.env.JAMBONES_LOGLEVEL || 'info'
-});
-
-const logger = require('pino')(opts);
+};
+const pino = require('pino');
+const logger = pino(opts, pino.destination(1, {sync: false}));
 
 module.exports = logger;

lib/models/account.js

@@ -4,7 +4,7 @@ const {getMysqlConnection} = require('../db');
 const {promisePool} = require('../db');
 const { v4: uuid } = require('uuid');
 
-const {encrypt} = require('../utils/encrypt-decrypt');
+const {encrypt, decrypt} = require('../utils/encrypt-decrypt');
 
 const retrieveSql = `SELECT * from accounts acc
 LEFT JOIN webhooks AS rh
@@ -34,7 +34,7 @@ AND effective_end_date IS NULL
 AND pending=0`;
 
 const updatePaymentInfoSql = `UPDATE account_subscriptions 
-SET last4 = ?, exp_month = ?, exp_year = ?, card_type = ? 
+SET last4 = ?, stripe_payment_method_id=?, exp_month = ?, exp_year = ?, card_type = ? 
 WHERE account_sid = ? 
 AND effective_end_date IS NULL`;
 
@@ -55,6 +55,13 @@ WHERE account_sid = ?
 AND effective_end_date IS NULL 
 AND pending = 0`;
 
+const extractBucketCredential = (obj) => {
+  const {bucket_credential} = obj;
+  if (bucket_credential) {
+    obj.bucket_credential = JSON.parse(decrypt(bucket_credential));
+  }
+};
+
 function transmogrifyResults(results) {
   return results.map((row) => {
     const obj = row.acc;
@@ -75,6 +82,8 @@ function transmogrifyResults(results) {
     else obj.queue_event_hook = null;
     delete obj.queue_event_hook_sid;
 
+    extractBucketCredential(obj);
+
     return obj;
   });
 }
@@ -197,10 +206,10 @@ class Account extends Model {
   }
 
   static async updatePaymentInfo(logger, account_sid, pm) {
-    const {card} = pm;
+    const {id, card} = pm;
     const last4_encrypted = encrypt(card.last4);
     await promisePool.execute(updatePaymentInfoSql,
-      [last4_encrypted, card.exp_month, card.exp_year, card.brand, account_sid]);
+      [last4_encrypted, id, card.exp_month, card.exp_year, card.brand, account_sid]);
   }
 
   static async provisionPendingSubscription(logger, account_sid, products, payment_method, subscription_id) {
@@ -238,7 +247,6 @@ class Account extends Model {
     }));
     return account_subscription_sid;
   }
-
 }
 
 Account.table = 'accounts';
@@ -319,7 +327,15 @@ Account.fields = [
     type: 'string',
   },
   {
-    name: 'lcr_sid',
+    name: 'record_all_calls',
+    type: 'number'
+  },
+  {
+    name: 'record_format',
+    type: 'string'
+  },
+  {
+    name: 'bucket_credential',
     type: 'string'
   }
 ];

lib/models/application.js

@@ -120,6 +120,10 @@ Application.fields = [
   {
     name: 'messaging_hook_sid',
     type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number',
   }
 ];
 

lib/models/client.js

@@ -0,0 +1,58 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Client extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(service_provider_sid) {
+    const sql = `SELECT c.client_sid, c.account_sid, c.is_active, c.username, c.hashed_password
+      FROM ${this.table} AS c LEFT JOIN accounts AS acc ON c.account_sid = acc.account_sid
+      LEFT JOIN service_providers AS sp ON sp.service_provider_sid = accs.service_provider_sid
+      WHERE sp.service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, service_provider_sid);
+    return rows;
+  }
+
+  static async retrieveByAccountSidAndUserName(account_sid, username) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ? AND username = ?`;
+    const [rows] = await promisePool.query(sql, [account_sid, username]);
+    return rows;
+  }
+}
+
+Client.table = 'clients';
+Client.fields = [
+  {
+    name: 'client_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+  {
+    name: 'username',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'password',
+    type: 'string'
+  }
+];
+
+module.exports = Client;

lib/models/phone-number.js

@@ -1,6 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
-const sql = 'SELECT * from phone_numbers WHERE account_sid = ?';
+const sqlRetrieveAll = 'SELECT * from phone_numbers WHERE account_sid = ? ORDER BY number';
+const sqlRetrieveOne = 'SELECT * from phone_numbers WHERE phone_number_sid = ? AND account_sid = ? ORDER BY number';
 const sqlSP = `SELECT * 
 FROM phone_numbers 
 WHERE account_sid IN 
@@ -8,7 +9,7 @@ WHERE account_sid IN
   SELECT account_sid 
   FROM accounts 
   WHERE service_provider_sid = ?
-)`;
+) ORDER BY number`;
 
 class PhoneNumber extends Model {
   constructor() {
@@ -16,8 +17,8 @@ class PhoneNumber extends Model {
   }
 
   static async retrieveAll(account_sid) {
-    if (!account_sid) return super.retrieveAll();
-    const [rows] = await promisePool.query(sql, account_sid);
+    if (!account_sid) return await super.retrieveAll();
+    const [rows] = await promisePool.query(sqlRetrieveAll, account_sid);
     return rows;
   }
   static async retrieveAllForSP(service_provider_sid) {
@@ -30,7 +31,7 @@ class PhoneNumber extends Model {
    */
   static async retrieve(sid, account_sid) {
     if (!account_sid) return super.retrieve(sid);
-    const [rows] = await promisePool.query(`${sql} AND phone_number_sid = ?`, [account_sid, sid]);
+    const [rows] = await promisePool.query(sqlRetrieveOne, [sid, account_sid]);
     return rows;
   }
 }

lib/models/sip-gateway.js

@@ -51,6 +51,10 @@ SipGateway.fields = [
     name: 'is_active',
     type: 'number'
   },
+  {
+    name: 'pad_crypto',
+    type: 'number'
+  },
   {
     name: 'account_sid',
     type: 'string'

lib/models/speech-credential.js

@@ -1,7 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
 const retrieveSql = 'SELECT * from speech_credentials WHERE account_sid = ?';
-const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ?';
+const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ? and account_sid is null';
 
 class SpeechCredential extends Model {
   constructor() {
@@ -20,6 +20,18 @@ class SpeechCredential extends Model {
     return rows;
   }
 
+  static async getSpeechCredentialsByVendorAndLabel(service_provider_sid, account_sid, vendor, label) {
+    let sql;
+    if (account_sid) {
+      sql = `SELECT * FROM speech_credentials WHERE account_sid = ? AND vendor = ? ${label ? 'AND label = ?' : ''}`;
+    } else {
+      sql = `SELECT * FROM speech_credentials WHERE service_provider_sid = ? AND vendor = ?
+        ${label ? 'AND label = ?' : ''}`;
+    }
+    const [rows] = await promisePool.query(sql, [account_sid ? account_sid : service_provider_sid, vendor, label]);
+    return rows;
+  }
+
   static async disableStt(account_sid) {
     await promisePool.execute('UPDATE speech_credentials SET use_for_stt = 0 WHERE account_sid = ?', [account_sid]);
   }
@@ -86,6 +98,10 @@ SpeechCredential.fields = [
   {
     name: 'last_tested',
     type: 'date'
+  },
+  {
+    name: 'label',
+    type: 'string'
   }
 ];
 

lib/record/azure-storage.js

@@ -0,0 +1,41 @@
+const { Writable } = require('stream');
+const { BlobServiceClient } = require('@azure/storage-blob');
+const { v4: uuidv4 } = require('uuid');
+
+class AzureStorageUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+    this.blockBlobClient = blobServiceClient.getContainerClient(opts.bucketName).getBlockBlobClient(opts.Key);
+    this.metadata = opts.metadata;
+    this.blocks = [];
+  }
+
+  async _write(chunk, encoding, callback) {
+    const blockID = uuidv4().replace(/-/g, '');
+    this.blocks.push(blockID);
+    try {
+      await this.blockBlobClient.stageBlock(blockID, chunk, chunk.length);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this.blockBlobClient.commitBlockList(this.blocks);
+      // remove all null/undefined props
+      const filteredObj = Object.entries(this.metadata).reduce((acc, [key, val]) => {
+        if (val !== undefined && val !== null) acc[key] = val;
+        return acc;
+      }, {});
+      await this.blockBlobClient.setMetadata(filteredObj);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = AzureStorageUploadStream;

lib/record/encoder.js

@@ -0,0 +1,61 @@
+const { Transform } = require('stream');
+const lamejs = require('@jambonz/lamejs');
+
+class PCMToMP3Encoder extends Transform {
+  constructor(options, logger) {
+    super(options);
+
+    const channels = options.channels || 1;
+    const sampleRate = options.sampleRate || 8000;
+    const bitRate = options.bitRate || 128;
+
+    this.encoder = new lamejs.Mp3Encoder(channels, sampleRate, bitRate);
+    this.channels = channels;
+    this.logger = logger;
+  }
+
+  _transform(chunk, encoding, callback) {
+    try {
+      // Convert chunk buffer into Int16Array for lamejs
+      const samples = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
+
+      // Split input samples into left and right channel arrays if stereo
+      let leftChannel, rightChannel;
+      if (this.channels === 2) {
+        leftChannel = new Int16Array(samples.length / 2);
+        rightChannel = new Int16Array(samples.length / 2);
+
+        for (let i = 0; i < samples.length; i += 2) {
+          leftChannel[i / 2] = samples[i];
+          rightChannel[i / 2] = samples[i + 1];
+        }
+      } else {
+        leftChannel = samples;
+      }
+
+      // Encode the input data
+      const mp3Data = this.encoder.encodeBuffer(leftChannel, rightChannel);
+
+      if (mp3Data.length > 0) {
+        this.push(Buffer.from(mp3Data));
+      }
+      callback();
+    } catch (err) {
+      this.logger.error(
+        { err },
+        'Error while mp3 transform');
+    }
+  }
+
+  _flush(callback) {
+    // Finalize encoding and flush the internal buffers
+    const mp3Data = this.encoder.flush();
+
+    if (mp3Data.length > 0) {
+      this.push(Buffer.from(mp3Data));
+    }
+    callback();
+  }
+}
+
+module.exports = PCMToMP3Encoder;

lib/record/google-storage.js

@@ -0,0 +1,41 @@
+const { Storage } = require('@google-cloud/storage');
+const { Writable } = require('stream');
+
+class GoogleStorageUploadStream extends Writable {
+
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.metadata = opts.metadata;
+
+    const storage = new Storage(opts.bucketCredential);
+    this.gcsFile = storage.bucket(opts.bucketName).file(opts.Key);
+    this.writeStream = this.gcsFile.createWriteStream();
+
+    this.writeStream.on('error', (err) => this.logger.error(err));
+    this.writeStream.on('finish', () => {
+      this.logger.info('google storage Upload completed.');
+      this._addMetadata();
+    });
+  }
+
+  _write(chunk, encoding, callback) {
+    this.writeStream.write(chunk, encoding, callback);
+  }
+
+  _final(callback) {
+    this.writeStream.end();
+    this.writeStream.once('finish', callback);
+  }
+
+  async _addMetadata() {
+    try {
+      await this.gcsFile.setMetadata({metadata: this.metadata});
+      this.logger.info('Google storage Upload and metadata setting completed.');
+    } catch (err) {
+      this.logger.error(err, 'Google storage  An error occurred while setting metadata');
+    }
+  }
+}
+
+module.exports = GoogleStorageUploadStream;

lib/record/index.js

@@ -0,0 +1,6 @@
+
+async function record(logger, socket) {
+  return require('./upload')(logger, socket);
+}
+
+module.exports = record;

lib/record/s3-multipart-upload-stream.js

@@ -0,0 +1,103 @@
+const { Writable } = require('stream');
+const {
+  S3Client,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+} = require('@aws-sdk/client-s3');
+
+class S3MultipartUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.bucketName = opts.bucketName;
+    this.objectKey = opts.objectKey;
+    this.uploadId = null;
+    this.partNumber = 1;
+    this.multipartETags = [];
+    this.buffer = Buffer.alloc(0);
+    this.minPartSize = 5 * 1024 * 1024; // 5 MB
+    this.s3 = new S3Client(opts.bucketCredential);
+    this.metadata = opts.metadata;
+  }
+
+  async _initMultipartUpload() {
+    const command = new CreateMultipartUploadCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      Metadata: this.metadata
+    });
+    const response = await this.s3.send(command);
+    return response.UploadId;
+  }
+
+  async _uploadBuffer() {
+    const uploadPartCommand = new UploadPartCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      PartNumber: this.partNumber,
+      UploadId: this.uploadId,
+      Body: this.buffer,
+    });
+
+    const uploadPartResponse = await this.s3.send(uploadPartCommand);
+    this.multipartETags.push({
+      ETag: uploadPartResponse.ETag,
+      PartNumber: this.partNumber,
+    });
+    this.partNumber += 1;
+  }
+
+  async _write(chunk, encoding, callback) {
+    try {
+      if (!this.uploadId) {
+        this.uploadId = await this._initMultipartUpload();
+      }
+
+      this.buffer = Buffer.concat([this.buffer, chunk]);
+
+      if (this.buffer.length >= this.minPartSize) {
+        await this._uploadBuffer();
+        this.buffer = Buffer.alloc(0);
+      }
+
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _finalize(err) {
+    try {
+      if (this.buffer.length > 0) {
+        await this._uploadBuffer();
+      }
+
+      const completeMultipartUploadCommand = new CompleteMultipartUploadCommand({
+        Bucket: this.bucketName,
+        Key: this.objectKey,
+        MultipartUpload: {
+          Parts: this.multipartETags.sort((a, b) => a.PartNumber - b.PartNumber),
+        },
+        UploadId: this.uploadId,
+      });
+
+      await this.s3.send(completeMultipartUploadCommand);
+      this.logger.info('Finished upload to S3');
+    } catch (error) {
+      this.logger.error('Error completing multipart upload:', error);
+      throw error;
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this._finalize();
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = S3MultipartUploadStream;

lib/record/upload.js

@@ -0,0 +1,99 @@
+const Account = require('../models/account');
+const Websocket = require('ws');
+const PCMToMP3Encoder = require('./encoder');
+const wav = require('wav');
+const { getUploader } = require('./utils');
+
+async function upload(logger, socket) {
+  socket._recvInitialMetadata = false;
+  socket.on('message', async function(data, isBinary) {
+    try {
+      if (!isBinary && !socket._recvInitialMetadata) {
+        socket._recvInitialMetadata = true;
+        logger.debug(`initial metadata: ${data}`);
+        const obj = JSON.parse(data.toString());
+        logger.info({ obj }, 'received JSON message from jambonz');
+        const { sampleRate, accountSid, callSid, direction, from, to,
+          callId, applicationSid, originatingSipIp, originatingSipTrunkName } = obj;
+        const account = await Account.retrieve(accountSid);
+        if (account && account.length && account[0].bucket_credential) {
+          const obj = account[0].bucket_credential;
+          // add tags to metadata
+          const metadata = {
+            accountSid,
+            callSid,
+            direction,
+            from,
+            to,
+            callId,
+            applicationSid,
+            originatingSipIp,
+            originatingSipTrunkName,
+            sampleRate: `${sampleRate}`
+          };
+          if (obj.tags && obj.tags.length) {
+            obj.tags.forEach((tag) => {
+              metadata[tag.Key] = tag.Value;
+            });
+          }
+          // create S3 path
+          const day = new Date();
+          let key = `${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
+          key += `/${day.getDate().toString().padStart(2, '0')}/${callSid}.${account[0].record_format}`;
+
+          // Uploader
+          const uploadStream = getUploader(key, metadata, obj, logger);
+          if (!uploadStream) {
+            logger.info('There is no available record uploader, close the socket.');
+            socket.close();
+          }
+
+          /**encoder */
+          let encoder;
+          if (account[0].record_format === 'wav') {
+            encoder = new wav.Writer({ channels: 2, sampleRate, bitDepth: 16 });
+          } else {
+            // default is mp3
+            encoder = new PCMToMP3Encoder({
+              channels: 2,
+              sampleRate: sampleRate,
+              bitrate: 128
+            }, logger);
+          }
+          const handleError = (err, streamType) => {
+            logger.error(
+              { err },
+              `Error while streaming for vendor: ${obj.vendor}, pipe: ${streamType}: ${err.message}`
+            );
+          };
+
+          /* start streaming data */
+          const duplex = Websocket.createWebSocketStream(socket);
+          duplex
+            .on('error', (err) => handleError(err, 'duplex'))
+            .pipe(encoder)
+            .on('error', (err) => handleError(err, 'encoder'))
+            .pipe(uploadStream)
+            .on('error', (err) => handleError(err, 'uploadStream'));
+
+        } else {
+          logger.info(`account ${accountSid} does not have any bucket credential, close the socket`);
+          socket.close();
+        }
+      }
+    } catch (err) {
+      logger.error({ err, data }, 'error parsing message during connection');
+    }
+  });
+  socket.on('error', function(err) {
+    logger.error({ err }, 'record upload: error');
+  });
+  socket.on('close', (data) => {
+    logger.info({ data }, 'record upload: close');
+  });
+  socket.on('end', function(err) {
+    logger.error({ err }, 'record upload: socket closed from jambonz');
+  });
+}
+
+module.exports = upload;

lib/record/utils.js

@@ -0,0 +1,58 @@
+const AzureStorageUploadStream = require('./azure-storage');
+const GoogleStorageUploadStream = require('./google-storage');
+const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
+
+const getUploader = (key, metadata, bucket_credential, logger) => {
+  const uploaderOpts = {
+    bucketName: bucket_credential.name,
+    objectKey: key,
+    metadata
+  };
+  try {
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+        uploaderOpts.bucketCredential = {
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: bucket_credential.region || 'us-east-1'
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 's3_compatible':
+        uploaderOpts.bucketCredential = {
+          endpoint: bucket_credential.endpoint,
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: 'us-east-1',
+          forcePathStyle: true
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 'google':
+        const serviceKey = JSON.parse(bucket_credential.service_key);
+        uploaderOpts.bucketCredential = {
+          projectId: serviceKey.project_id,
+          credentials: {
+            client_email: serviceKey.client_email,
+            private_key: serviceKey.private_key
+          }
+        };
+        return new GoogleStorageUploadStream(logger, uploaderOpts);
+      case 'azure':
+        uploaderOpts.connection_string = bucket_credential.connection_string;
+        return new AzureStorageUploadStream(logger, uploaderOpts);
+      default:
+        logger.error(`unknown bucket vendor: ${bucket_credential.vendor}`);
+        break;
+    }
+  } catch (err) {
+    logger.error(`Error creating uploader, vendor: ${bucket_credential.vendor}, reason: ${err.message}`);
+  }
+  return null;
+};
+
+module.exports = {
+  getUploader
+};

lib/routes/api/accounts.js

@@ -1,4 +1,5 @@
 const router = require('express').Router();
+const assert = require('assert');
 const request = require('request');
 const {DbErrorBadRequest, DbErrorForbidden, DbErrorUnprocessableRequest} = require('../../utils/errors');
 const Account = require('../../models/account');
@@ -22,6 +23,8 @@ const {
 } = require('./utils');
 const short = require('short-uuid');
 const VoipCarrier = require('../../models/voip-carrier');
+const { encrypt } = require('../../utils/encrypt-decrypt');
+const { testS3Storage, testGoogleStorage, testAzureStorage } = require('../../utils/storage-utils');
 const translator = short();
 
 let idx = 0;
@@ -38,20 +41,14 @@ const getFsUrl = async(logger, retrieveSet, setName) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/createCall`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/createCall`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
-};
-
 const validateRequest = async(req, account_sid) => {
   try {
     if (req.user.hasScope('admin')) {
@@ -89,6 +86,7 @@ router.use('/:sid/Charges', hasAccountPermissions, require('./charges'));
 router.use('/:sid/SipRealms', hasAccountPermissions, require('./sip-realm'));
 router.use('/:sid/PredefinedCarriers', hasAccountPermissions, require('./add-from-predefined-carrier'));
 router.use('/:sid/Limits', hasAccountPermissions, require('./limits'));
+router.use('/:sid/TtsCache', hasAccountPermissions, require('./tts-cache'));
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
@@ -147,6 +145,21 @@ router.post('/:sid/VoipCarriers', async(req, res) => {
     sysError(logger, res, err);
   }
 });
+router.get('/:sid/RegisteredSipUsers', async(req, res) => {
+  const {logger, registrar} = req.app.locals;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const result = await Account.retrieve(account_sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${account_sid}`);
+    }
+    const users = await registrar.getRegisteredUsersForRealm(result[0].sip_realm);
+    res.status(200).json(users.map((u) => `${u}@${result[0].sip_realm}`));
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
 
 function coerceNumbers(callInfo) {
   if (Array.isArray(callInfo)) {
@@ -178,6 +191,7 @@ function validateUpdateCall(opts) {
     'child_call_hook',
     'call_status',
     'listen_status',
+    'transcribe_status',
     'conf_hold_status',
     'conf_mute_status',
     'mute_status',
@@ -218,8 +232,8 @@ function validateUpdateCall(opts) {
     throw new DbErrorBadRequest('invalid conf_mute_status');
   }
   if (opts.sip_request &&
-    (!opts.sip_request.method && !opts.sip_request.content_type ||  !opts.sip_request.content_type)) {
-    throw new DbErrorBadRequest('sip_request requires content_type and content properties');
+    (!opts.sip_request.method || !opts.sip_request.content_type ||  !opts.sip_request.content)) {
+    throw new DbErrorBadRequest('sip_request requires method, content_type and content properties');
   }
   if (opts.record && !opts.record.action) {
     throw new DbErrorBadRequest('record requires action property');
@@ -263,6 +277,7 @@ async function validateCreateCall(logger, sid, req) {
       const application = await lookupAppBySid(obj.application_sid);
       Object.assign(obj, {
         call_hook: application.call_hook,
+        app_json: application.app_json,
         call_status_hook: application.call_status_hook,
         speech_synthesis_vendor: application.speech_synthesis_vendor,
         speech_synthesis_language: application.speech_synthesis_language,
@@ -449,7 +464,6 @@ router.get('/:sid', async(req, res) => {
   try {
     const account_sid = parseAccountSid(req);
     await validateRequest(req, account_sid);
-
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
     const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
@@ -535,6 +549,58 @@ router.delete('/:sid/SubspaceTeleport', async(req, res) => {
   }
 });
 
+function encryptBucketCredential(obj) {
+  if (!obj.bucket_credential) return;
+  const {
+    vendor,
+    region,
+    name,
+    access_key_id,
+    secret_access_key,
+    tags,
+    service_key,
+    connection_string,
+    endpoint
+  } = obj.bucket_credential;
+
+  switch (vendor) {
+    case 'aws_s3':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(region, 'invalid aws bucket region: region is required');
+      const awsData = JSON.stringify({vendor, region, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(awsData);
+      break;
+    case 's3_compatible':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(endpoint, 'invalid endpoint uri: endpoint is required');
+      const s3Data = JSON.stringify({vendor, endpoint, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(s3Data);
+      break;
+    case 'google':
+      assert(service_key, 'invalid google cloud storage credential: service_key is required');
+      const googleData = JSON.stringify({vendor, name, service_key, tags});
+      obj.bucket_credential = encrypt(googleData);
+      break;
+    case 'azure':
+      assert(name, 'invalid azure container name: name is required');
+      assert(connection_string, 'invalid azure cloud storage credential: connection_string is required');
+      const azureData = JSON.stringify({vendor, name, connection_string, tags});
+      obj.bucket_credential = encrypt(azureData);
+      break;
+    case 'none':
+      obj.bucket_credential = null;
+      break;
+    default:
+      throw new DbErrorBadRequest(`unknown storage vendor: ${vendor}`);
+  }
+}
+
 /**
  * update
  */
@@ -581,6 +647,8 @@ router.put('/:sid', async(req, res) => {
       delete obj.registration_hook;
       delete obj.queue_event_hook;
 
+      encryptBucketCredential(obj);
+
       const rowsAffected = await Account.update(sid, obj);
       if (rowsAffected === 0) {
         return res.status(404).end();
@@ -673,6 +741,44 @@ account_subscriptions WHERE account_sid = ?)
   }
 });
 
+/* Test Bucket credential Keys */
+router.post('/:sid/BucketCredentialTest', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const {vendor, name, region, access_key_id, secret_access_key, service_key, connection_string, endpoint} = req.body;
+    const ret = {
+      status: 'not tested'
+    };
+
+    switch (vendor) {
+      case 'aws_s3':
+        await testS3Storage(logger, {vendor, name, region, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 's3_compatible':
+        await testS3Storage(logger, {vendor, name, endpoint, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 'google':
+        await testGoogleStorage(logger, {vendor, name, service_key});
+        ret.status = 'ok';
+        break;
+      case 'azure':
+        await testAzureStorage(logger, {vendor, name, connection_string});
+        ret.status = 'ok';
+        break;
+      default:
+        throw new DbErrorBadRequest(`Does not support test for ${vendor}`);
+    }
+    return res.status(200).json(ret);
+  }
+  catch (err) {
+    return res.status(200).json({status: 'failed', reason: err.message});
+  }
+});
+
 /**
  * retrieve account level api keys
  */
@@ -696,7 +802,7 @@ router.get('/:sid/ApiKeys', async(req, res) => {
  */
 router.post('/:sid/Calls', async(req, res) => {
   const {retrieveSet, logger} = req.app.locals;
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
+  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
   const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
 
   if (!serviceUrl) {
@@ -857,7 +963,7 @@ router.post('/:sid/Messages', async(req, res) => {
     const account_sid = parseAccountSid(req);
     await validateRequest(req, account_sid);
 
-    const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
+    const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
     const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
     if (!serviceUrl) res.json({msg: 'no available feature servers at this time'}).status(480);
     await validateCreateMessage(logger, account_sid, req);
@@ -894,12 +1000,12 @@ router.post('/:sid/Messages', async(req, res) => {
  * retrieve info for a group of queues under an account
  */
 router.get('/:sid/Queues', async(req, res) => {
-  const {logger, listQueues} = req.app.locals;
+  const {logger, listSortedSets} = req.app.locals;
   const { search } = req.query || {};
   try {
     const accountSid = parseAccountSid(req);
     await validateRequest(req, accountSid);
-    const queues = search ? await listQueues(accountSid, search) : await listQueues(accountSid);
+    const queues = search ? await listSortedSets(accountSid, search) : await listSortedSets(accountSid);
     logger.debug(`retrieved ${queues.length} queues for account sid ${accountSid}`);
     res.status(200).json(queues);
     updateLastUsed(logger, accountSid, req).catch((err) => {});

lib/routes/api/clients.js

@@ -0,0 +1,88 @@
+const router = require('express').Router();
+const decorate = require('./decorate');
+const sysError = require('../error');
+const Client = require('../../models/client');
+const Account = require('../../models/account');
+const { DbErrorBadRequest, DbErrorForbidden } = require('../../utils/errors');
+const { encrypt, decrypt, obscureKey } = require('../../utils/encrypt-decrypt');
+
+const commonCheck = async(req) => {
+  if (req.user.hasAccountAuth) {
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth && req.body.account_sid) {
+    const accounts = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
+    if (accounts.length === 0) {
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  }
+
+  if (req.body.password) {
+    req.body.password = encrypt(req.body.password);
+  }
+};
+
+const validateAdd = async(req) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length && clients[0].client_sid !== sid) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, Client, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Client.retrieveAll() : req.user.hasAccountAuth ?
+        await Client.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Client.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+    const ret = results.map((c) => {
+      c.password = obscureKey(decrypt(c.password), 1);
+      return c;
+    });
+    res.status(200).json(ret);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Client.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const client = results[0];
+    client.password = obscureKey(decrypt(client.password), 1);
+    if (req.user.hasAccountAuth && client.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth) {
+      const accounts = await Account.retrieve(client.account_sid, req.user.service_provider_sid);
+      if (!accounts.length) {
+        return res.sendStatus(404);
+      }
+    }
+    return res.status(200).json(client);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/forgot-password.js

@@ -6,6 +6,7 @@ const {validateEmail, emailSimpleText} = require('../../utils/email-utils');
 const {promisePool} = require('../../db');
 const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
+const assert = require('assert');
 const sql = `SELECT * from users user 
 LEFT JOIN accounts AS acc 
 ON acc.account_sid = user.account_sid 
@@ -26,7 +27,8 @@ function createOauthEmailText(provider) {
 }
 
 function createResetEmailText(link) {
-  const baseUrl = 'http://localhost:3001';
+  assert(process.env.JAMBONZ_BASE_URL, 'process.env.JAMBONZ_BASE_URL is missing');
+  const baseUrl = process.env.JAMBONZ_BASE_URL;
 
   return `Hi there!
 

lib/routes/api/index.js

@@ -17,6 +17,7 @@ const isAdminScope = (req, res, next) => {
 
 api.use('/BetaInviteCodes', isAdminScope, require('./beta-invite-codes'));
 api.use('/SystemInformation', isAdminScope, require('./system-information'));
+api.use('/TtsCache', isAdminScope, require('./tts-cache'));
 api.use('/ServiceProviders', require('./service-providers'));
 api.use('/VoipCarriers', require('./voip-carriers'));
 api.use('/Webhooks', require('./webhooks'));
@@ -50,6 +51,7 @@ api.use('/PasswordSettings', require('./password-settings'));
 api.use('/Lcrs', require('./lcrs'));
 api.use('/LcrRoutes', require('./lcr-routes'));
 api.use('/LcrCarrierSetEntries', require('./lcr-carrier-set-entries'));
+api.use('/Clients', require('./clients'));
 
 // messaging
 api.use('/Smpps', require('./smpps'));   // our smpp server info

lib/routes/api/phone-numbers.js

@@ -94,9 +94,9 @@ decorate(router, PhoneNumber, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = req.user.hasAdminAuth ?
-      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null) :
-      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid);
+    const results = req.user.hasServiceProviderAuth ?
+      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid) :
+      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -120,6 +120,9 @@ router.get('/:sid', async(req, res) => {
         throw new DbErrorBadRequest('insufficient privileges');
       }
     }
+    if (req.user.hasAccountAuth && results.length > 1) {
+      return res.status(200).json(results.filter((r) => r.phone_number_sid === sid)[0]);
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/recent-calls.js

@@ -3,6 +3,15 @@ const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
 const {getJaegerTrace} = require('../../utils/jaeger-utils');
+const Account = require('../../models/account');
+const {
+  getS3Object,
+  getGoogleStorageObject,
+  getAzureStorageObject,
+  deleteS3Object,
+  deleteGoogleStorageObject,
+  deleteAzureStorageObject
+} = require('../../utils/storage-utils');
 
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
@@ -20,7 +29,7 @@ router.get('/', async(req, res) => {
     logger.debug({opts: req.query}, 'GET /RecentCalls');
     const account_sid = parseAccountSid(req.originalUrl);
     const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
-    const {page, count, trunk, direction, days, answered, start, end} = req.query || {};
+    const {page, count, trunk, direction, days, answered, start, end, filter} = req.query || {};
     if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
     if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
 
@@ -35,6 +44,7 @@ router.get('/', async(req, res) => {
         answered,
         start: days ? undefined : start,
         end: days ? undefined : end,
+        filter
       });
       res.status(200).json(data);
     }
@@ -49,6 +59,7 @@ router.get('/', async(req, res) => {
         answered,
         start: days ? undefined : start,
         end: days ? undefined : end,
+        filter
       });
       res.status(200).json(data);
     }
@@ -74,12 +85,12 @@ router.get('/:call_id', async(req, res) => {
   }
 });
 
-router.get('/:call_id/pcap', async(req, res) => {
+router.get('/:call_id/:method/pcap', async(req, res) => {
   const {logger} = req.app.locals;
   try {
     const token = await getHomerApiKey(logger);
     if (!token) return res.sendStatus(400, {msg: 'getHomerApiKey: Failed to get Homer API token; check server config'});
-    const stream = await getHomerPcap(logger, token, [req.params.call_id]);
+    const stream = await getHomerPcap(logger, token, [req.params.call_id], req.params.method);
     if (!stream) {
       logger.info(`getHomerApiKey: unable to get sip traces from Homer for ${req.params.call_id}`);
       return res.sendStatus(404);
@@ -111,4 +122,84 @@ router.get('/trace/:trace_id', async(req, res) => {
   }
 });
 
+router.get('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+    const getOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+    let stream;
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        stream = await getS3Object(logger, getOptions);
+        break;
+      case 'google':
+        stream = await getGoogleStorageObject(logger, getOptions);
+        break;
+      case 'azure':
+        stream = await getAzureStorageObject(logger, getOptions);
+        break;
+      default:
+        logger.error(`There is no handler for fetching record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.set({
+      'Content-Type': `audio/${format || 'mp3'}`
+    });
+    if (stream) {
+      stream.pipe(res);
+    } else {
+      return res.sendStatus(404);
+    }
+  }  catch (err) {
+    logger.error({err}, ` error retrieving recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
+router.delete('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+
+    const deleteOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        await deleteS3Object(logger, deleteOptions);
+        break;
+      case 'google':
+        await deleteGoogleStorageObject(logger, deleteOptions);
+        break;
+      case 'azure':
+        await deleteAzureStorageObject(logger, deleteOptions);
+        break;
+      default:
+        logger.error(`There is no handler for deleting record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.sendStatus(204);
+  }  catch (err) {
+    logger.error({err}, ` error deleting recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
 module.exports = router;

lib/routes/api/register.js

@@ -16,8 +16,9 @@ const insertUserSql = `INSERT into users
 (user_sid, account_sid, name, email, provider, provider_userid, email_validated) 
 values (?, ?, ?, ?, ?, ?, 1)`;
 const insertUserLocalSql = `INSERT into users 
-(user_sid, account_sid, name, email, email_activation_code, email_validated, provider, hashed_password) 
-values (?, ?, ?, ?, ?, 0, 'local', ?)`;
+(user_sid, account_sid, name, email, email_activation_code, email_validated, provider,
+hashed_password, service_provider_sid) 
+values (?, ?, ?, ?, ?, 0, 'local', ?, ?)`;
 const insertAccountSql = `INSERT into accounts 
 (account_sid, service_provider_sid, name, is_active, webhook_secret, trial_end_date) 
 values (?, ?, ?, ?, ?, CURDATE() + INTERVAL 21 DAY)`;
@@ -36,7 +37,7 @@ const insertSignupHistorySql = `INSERT into signup_history
 values (?, ?)`;
 
 const addLocalUser = async(logger, user_sid, account_sid,
-  name, email, email_activation_code, passwordHash) => {
+  name, email, email_activation_code, passwordHash, service_provider_sid) => {
   const [r] = await promisePool.execute(insertUserLocalSql,
     [
       user_sid,
@@ -44,7 +45,8 @@ const addLocalUser = async(logger, user_sid, account_sid,
       name,
       email,
       email_activation_code,
-      passwordHash
+      passwordHash,
+      service_provider_sid
     ]);
   debug({r}, 'Result from adding user');
 };
@@ -145,7 +147,7 @@ router.post('/', async(req, res) => {
       const user = await doGithubAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from github');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         email_validated: user.email_validated,
         avatar_url: user.avatar_url,
@@ -157,7 +159,7 @@ router.post('/', async(req, res) => {
       const user = await doGoogleAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from google');
       Object.assign(userProfile, {
-        name: user.name || user.email,
+        name: user.email || user.email,
         email: user.email,
         email_validated: user.verified_email,
         picture: user.picture,
@@ -170,7 +172,7 @@ router.post('/', async(req, res) => {
       logger.info({user}, 'retrieved user details for local provider');
       debug({user}, 'retrieved user details for local provider');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         provider: 'local',
         email_activation_code: user.email_activation_code
@@ -280,7 +282,8 @@ router.post('/', async(req, res) => {
         const passwordHash = await generateHashedPassword(req.body.password);
         debug(`hashed password: ${passwordHash}`);
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
-          userProfile.name, userProfile.email, userProfile.email_activation_code, passwordHash);
+          userProfile.name, userProfile.email, userProfile.email_activation_code,
+          passwordHash, req.body.service_provider_sid);
         debug('added local user');
       }
       else {
@@ -293,17 +296,25 @@ router.post('/', async(req, res) => {
       const callStatusSid = uuid();
       const helloWordSid = uuid();
       const dialTimeSid = uuid();
+      const echoSid = uuid();
 
-      /* 3 webhooks */
-      await promisePool.execute(insertWebookSql, [callStatusSid, 'https://public-apps.jambonz.us/call-status', 'POST']);
-      await promisePool.execute(insertWebookSql, [helloWordSid, 'https://public-apps.jambonz.us/hello-world', 'POST']);
-      await promisePool.execute(insertWebookSql, [dialTimeSid, 'https://public-apps.jambonz.us/dial-time', 'POST']);
+      /* 4 webhooks */
+      await promisePool.execute(insertWebookSql,
+        [callStatusSid, 'https://public-apps.jambonz.cloud/call-status', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [helloWordSid, 'https://public-apps.jambonz.cloud/hello-world', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [dialTimeSid, 'https://public-apps.jambonz.cloud/dial-time', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [echoSid, 'https://public-apps.jambonz.cloud/echo', 'POST']);
 
       /* 2 applications */
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'hello world',
         helloWordSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'dial time clock',
         dialTimeSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
+      await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'simple echo test',
+        echoSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
 
       Object.assign(userProfile, {
         pristine: true,
@@ -327,7 +338,7 @@ router.post('/', async(req, res) => {
 
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
           userProfile.name, userProfile.email, userProfile.email_activation_code,
-          passwordHash);
+          passwordHash, req.body.service_provider_sid);
 
         /* note: we deactivate the old user once the new email is validated */
       }
@@ -349,6 +360,8 @@ router.post('/', async(req, res) => {
     const token = jwt.sign({
       user_sid: userProfile.user_sid,
       account_sid: userProfile.account_sid,
+      service_provider_sid: req.body.service_provider_sid,
+      scope: 'account',
       email: userProfile.email,
       name: userProfile.name
     }, process.env.JWT_SECRET, { expiresIn });

lib/routes/api/sbcs.js

@@ -28,20 +28,13 @@ router.get('/', async(req, res) => {
 
     if (req.user.hasAccountAuth) {
       const [r] = await promisePool.query('SELECT * from accounts WHERE account_sid = ?', req.user.account_sid);
-      if (0 === r.length) throw new Error('invalid account_sid');
+      if (0 === r.length) throw new DbErrorBadRequest('invalid account_sid');
 
       service_provider_sid = r[0].service_provider_sid;
     }
 
     if (req.user.hasServiceProviderAuth) {
-      const [r] = await promisePool.query(
-        'SELECT * from service_providers where service_provider_sid = ?',
-        service_provider_sid);
-      if (0 === r.length) throw new Error('invalid account_sid');
-
-      service_provider_sid = r[0].service_provider_sid;
-
-      if (!service_provider_sid) throw new DbErrorBadRequest('missing service_provider_sid in query');
+      service_provider_sid = req.user.service_provider_sid;
     }
 
     /** generally, we have a global set of SBCs that all accounts use.

lib/routes/api/sms-inbound.js

@@ -14,20 +14,14 @@ const getFsUrl = async(logger, retrieveSet, setName, provider) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/messaging/${provider}`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/messaging/${provider}`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
-};
-
 const doSendResponse = async(res, respondFn, body) => {
   if (typeof respondFn === 'number') res.sendStatus(respondFn);
   else if (typeof respondFn !== 'function') res.sendStatus(200);
@@ -44,7 +38,7 @@ router.post('/:provider', async(req, res) => {
     lookupAppByPhoneNumber,
     logger
   } = req.app.locals;
-  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:active-fs`;
+  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:fs-service-url`;
   logger.debug({path: req.path, body: req.body}, 'incomingSMS from carrier');
 
   // search for provider module

lib/routes/api/speech-credentials.js

@@ -3,7 +3,7 @@ const assert = require('assert');
 const Account = require('../../models/account');
 const SpeechCredential = require('../../models/speech-credential');
 const sysError = require('../error');
-const {decrypt, encrypt} = require('../../utils/encrypt-decrypt');
+const {decrypt, encrypt, obscureKey} = require('../../utils/encrypt-decrypt');
 const {parseAccountSid, parseServiceProviderSid, parseSpeechCredentialSid} = require('./utils');
 const {DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const {
@@ -19,8 +19,10 @@ const {
   testDeepgramStt,
   testSonioxStt,
   testIbmTts,
-  testIbmStt
+  testIbmStt,
+  testElevenlabs
 } = require('../../utils/speech-utils');
+const bent = require('bent');
 const {promisePool} = require('../../db');
 
 const validateAdd = async(req) => {
@@ -99,17 +101,6 @@ const validateTest = async(req, speech_credentials) => {
   }
 };
 
-const obscureKey = (key) => {
-  const key_spoiler_length = 6;
-  const key_spoiler_char = 'X';
-
-  if (key.length <= key_spoiler_length) {
-    return key;
-  }
-
-  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
-};
-
 const encryptCredential = (obj) => {
   const {
     vendor,
@@ -125,8 +116,10 @@ const encryptCredential = (obj) => {
     nuance_stt_uri,
     use_custom_tts,
     custom_tts_endpoint,
+    custom_tts_endpoint_url,
     use_custom_stt,
     custom_stt_endpoint,
+    custom_stt_endpoint_url,
     tts_api_key,
     tts_region,
     stt_api_key,
@@ -135,7 +128,9 @@ const encryptCredential = (obj) => {
     instance_id,
     custom_stt_url,
     custom_tts_url,
-    auth_token = ''
+    auth_token = '',
+    cobalt_server_uri,
+    model_id
   } = obj;
 
   switch (vendor) {
@@ -158,15 +153,19 @@ const encryptCredential = (obj) => {
       return encrypt(awsData);
 
     case 'microsoft':
-      assert(region, 'invalid azure speech credential: region is required');
-      assert(api_key, 'invalid azure speech credential: api_key is required');
+      if (!custom_tts_endpoint_url && !custom_stt_endpoint_url) {
+        assert(region, 'invalid azure speech credential: region is required');
+        assert(api_key, 'invalid azure speech credential: api_key is required');
+      }
       const azureData = JSON.stringify({
-        region,
-        api_key,
+        ...(region && {region}),
+        ...(api_key && {api_key}),
         use_custom_tts,
         custom_tts_endpoint,
+        custom_tts_endpoint_url,
         use_custom_stt,
-        custom_stt_endpoint
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
       });
       return encrypt(azureData);
 
@@ -201,6 +200,17 @@ const encryptCredential = (obj) => {
       const sonioxData = JSON.stringify({api_key});
       return encrypt(sonioxData);
 
+    case 'cobalt':
+      assert(cobalt_server_uri, 'invalid cobalt speech credential: cobalt_server_uri is required');
+      const cobaltData = JSON.stringify({cobalt_server_uri});
+      return encrypt(cobaltData);
+
+    case 'elevenlabs':
+      assert(api_key, 'invalid elevenLabs speech credential: api_key is required');
+      assert(model_id, 'invalid elevenLabs speech credential: model_id is required');
+      const elevenlabsData = JSON.stringify({api_key, model_id});
+      return encrypt(elevenlabsData);
+
     default:
       if (vendor.startsWith('custom:')) {
         const customData = JSON.stringify({auth_token, custom_stt_url, custom_tts_url});
@@ -218,6 +228,7 @@ router.post('/', async(req, res) => {
       use_for_stt,
       use_for_tts,
       vendor,
+      label
     } = req.body;
     const account_sid = req.user.account_sid || req.body.account_sid;
     const service_provider_sid = req.user.service_provider_sid ||
@@ -232,11 +243,21 @@ router.post('/', async(req, res) => {
       }
     }
 
+    // Check if vendor and label is already used for account or SP
+    if (label) {
+      const existingSpeech = await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+        service_provider_sid, account_sid, vendor, label);
+      if (existingSpeech.length > 0) {
+        throw new DbErrorUnprocessableRequest(`Label ${label} is already in use for another speech credential`);
+      }
+    }
+
     const encrypted_credential = encryptCredential(req.body);
     const uuid = await SpeechCredential.make({
       account_sid,
       service_provider_sid,
       vendor,
+      label,
       use_for_tts,
       use_for_stt,
       credential: encrypted_credential
@@ -247,6 +268,76 @@ router.post('/', async(req, res) => {
   }
 });
 
+function decryptCredential(obj, credential, logger) {
+  if ('google' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    const key_header = '-----BEGIN PRIVATE KEY-----\n';
+    const obscured = {
+      ...o,
+      private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
+    };
+    obj.service_key = obscured;
+  }
+  else if ('aws' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.access_key_id = o.access_key_id;
+    obj.secret_access_key = obscureKey(o.secret_access_key);
+    obj.aws_region = o.aws_region;
+    logger.info({obj, o}, 'retrieving aws speech credential');
+  }
+  else if ('microsoft' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = obscureKey(o.api_key);
+    obj.region = o.region;
+    obj.use_custom_tts = o.use_custom_tts;
+    obj.custom_tts_endpoint = o.custom_tts_endpoint;
+    obj.custom_tts_endpoint_url = o.custom_tts_endpoint_url;
+    obj.use_custom_stt = o.use_custom_stt;
+    obj.custom_stt_endpoint = o.custom_stt_endpoint;
+    obj.custom_stt_endpoint_url = o.custom_stt_endpoint_url;
+    logger.info({obj, o}, 'retrieving azure speech credential');
+  }
+  else if ('wellsaid' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = obscureKey(o.api_key);
+  }
+  else if ('nuance' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.client_id = o.client_id;
+    obj.secret = o.secret ? obscureKey(o.secret) : null;
+  }
+  else if ('deepgram' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = obscureKey(o.api_key);
+  }
+  else if ('ibm' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.tts_api_key = obscureKey(o.tts_api_key);
+    obj.tts_region = o.tts_region;
+    obj.stt_api_key = obscureKey(o.stt_api_key);
+    obj.stt_region = o.stt_region;
+    obj.instance_id = o.instance_id;
+  } else if ('nvidia' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.riva_server_uri = o.riva_server_uri;
+  } else if ('cobalt' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.cobalt_server_uri = o.cobalt_server_uri;
+  } else if ('soniox' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = obscureKey(o.api_key);
+  } else if ('elevenlabs' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = obscureKey(o.api_key);
+    obj.model_id = o.model_id;
+  } else if (obj.vendor.startsWith('custom:')) {
+    const o = JSON.parse(decrypt(credential));
+    obj.auth_token = obscureKey(o.auth_token);
+    obj.custom_stt_url = o.custom_stt_url;
+    obj.custom_tts_url = o.custom_tts_url;
+  }
+}
+
 /**
  * retrieve all speech credentials for an account
  */
@@ -273,66 +364,7 @@ router.get('/', async(req, res) => {
     res.status(200).json(creds.map((c) => {
       const {credential, ...obj} = c;
 
-      if ('google' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        const key_header = '-----BEGIN PRIVATE KEY-----\n';
-        const obscured = {
-          ...o,
-          private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
-        };
-        obj.service_key = obscured;
-      }
-      else if ('aws' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.access_key_id = o.access_key_id;
-        obj.secret_access_key = obscureKey(o.secret_access_key);
-        obj.aws_region = o.aws_region;
-        logger.info({obj, o}, 'retrieving aws speech credential');
-      }
-      else if ('microsoft' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-        obj.region = o.region;
-        obj.use_custom_tts = o.use_custom_tts;
-        obj.custom_tts_endpoint = o.custom_tts_endpoint;
-        obj.use_custom_stt = o.use_custom_stt;
-        obj.custom_stt_endpoint = o.custom_stt_endpoint;
-        logger.info({obj, o}, 'retrieving azure speech credential');
-      }
-      else if ('wellsaid' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if ('nuance' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.client_id = o.client_id;
-        obj.secret = o.secret ? obscureKey(o.secret) : null;
-      }
-      else if ('deepgram' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if ('ibm' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.tts_api_key = obscureKey(o.tts_api_key);
-        obj.tts_region = o.tts_region;
-        obj.stt_api_key = obscureKey(o.stt_api_key);
-        obj.stt_region = o.stt_region;
-        obj.instance_id = o.instance_id;
-      } else if ('nvidia' == obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.riva_server_uri = o.riva_server_uri;
-      }
-      else if ('soniox' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if (obj.vendor.startsWith('custom:')) {
-        const o = JSON.parse(decrypt(credential));
-        obj.auth_token = obscureKey(o.auth_token);
-        obj.custom_stt_url = o.custom_stt_url;
-        obj.custom_tts_url = o.custom_tts_url;
-      }
+      decryptCredential(obj, credential, logger);
 
       if (req.user.hasAccountAuth && obj.account_sid === null) {
         delete obj.api_key;
@@ -362,66 +394,7 @@ router.get('/:sid', async(req, res) => {
     await validateRetrieveUpdateDelete(req, cred);
 
     const {credential, ...obj} = cred[0];
-    if ('google' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      const key_header = '-----BEGIN PRIVATE KEY-----\n';
-      const obscured = {
-        ...o,
-        private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
-      };
-      obj.service_key = JSON.stringify(obscured);
-    }
-    else if ('aws' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.access_key_id = o.access_key_id;
-      obj.secret_access_key = obscureKey(o.secret_access_key);
-      obj.aws_region = o.aws_region;
-    }
-    else if ('microsoft' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-      obj.region = o.region;
-      obj.use_custom_tts = o.use_custom_tts;
-      obj.custom_tts_endpoint = o.custom_tts_endpoint;
-      obj.use_custom_stt = o.use_custom_stt;
-      obj.custom_stt_endpoint = o.custom_stt_endpoint;
-    }
-    else if ('wellsaid' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if ('nuance' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.client_id = o.client_id;
-      obj.secret = o.secret ? obscureKey(o.secret) : null;
-      obj.nuance_tts_uri = o.nuance_tts_uri;
-      obj.nuance_stt_uri = o.nuance_stt_uri;
-    }
-    else if ('deepgram' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if ('ibm' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.tts_api_key = obscureKey(o.tts_api_key);
-      obj.tts_region = o.tts_region;
-      obj.stt_api_key = obscureKey(o.stt_api_key);
-      obj.stt_region = o.stt_region;
-      obj.instance_id = o.instance_id;
-    } else if ('nvidia' == obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.riva_server_uri = o.riva_server_uri;
-    }
-    else if ('soniox' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if (obj.vendor.startsWith('custom:')) {
-      const o = JSON.parse(decrypt(credential));
-      obj.auth_token = obscureKey(o.auth_token);
-      obj.custom_stt_url = o.custom_stt_url;
-      obj.custom_tts_url = o.custom_tts_url;
-    }
+    decryptCredential(obj, credential, logger);
 
     if (req.user.hasAccountAuth && obj.account_sid === null) {
       delete obj.api_key;
@@ -488,8 +461,13 @@ router.put('/:sid', async(req, res) => {
         const {
           use_custom_tts,
           custom_tts_endpoint,
+          custom_tts_endpoint_url,
           use_custom_stt,
-          custom_stt_endpoint
+          custom_stt_endpoint,
+          custom_stt_endpoint_url,
+          custom_stt_url,
+          custom_tts_url,
+          cobalt_server_uri,
         } = req.body;
 
         const newCred = {
@@ -499,13 +477,18 @@ router.put('/:sid', async(req, res) => {
           aws_region,
           use_custom_tts,
           custom_tts_endpoint,
+          custom_tts_endpoint_url,
           use_custom_stt,
           custom_stt_endpoint,
+          custom_stt_endpoint_url,
           stt_region,
           tts_region,
           riva_server_uri,
           nuance_stt_uri,
-          nuance_tts_uri
+          nuance_tts_uri,
+          custom_stt_url,
+          custom_tts_url,
+          cobalt_server_uri
         };
         logger.info({o, newCred}, 'updating speech credential with this new credential');
         obj.credential = encryptCredential(newCred);
@@ -618,8 +601,10 @@ router.get('/:sid/test', async(req, res) => {
         region,
         use_custom_tts,
         custom_tts_endpoint,
+        custom_tts_endpoint_url,
         use_custom_stt,
-        custom_stt_endpoint
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
       } = credential;
       if (cred.use_for_tts) {
         try {
@@ -628,8 +613,10 @@ router.get('/:sid/test', async(req, res) => {
             region,
             use_custom_tts,
             custom_tts_endpoint,
+            custom_tts_endpoint_url,
             use_custom_stt,
-            custom_stt_endpoint
+            custom_stt_endpoint,
+            custom_stt_endpoint_url
           });
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
@@ -758,6 +745,18 @@ router.get('/:sid/test', async(req, res) => {
           SpeechCredential.sttTestResult(sid, false);
         }
       }
+    } else if (cred.vendor === 'elevenlabs') {
+      const {api_key, model_id} = credential;
+      if (cred.use_for_tts) {
+        try {
+          await testElevenlabs(logger, {api_key, model_id});
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
     }
 
     res.status(200).json(results);
@@ -767,4 +766,93 @@ router.get('/:sid/test', async(req, res) => {
   }
 });
 
+/**
+ * Fetch speech voices and languages
+ */
+
+router.post('/voices', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {vendor, label} = req.body;
+  const account_sid = req.user.account_sid || req.body.account_sid;
+  const service_provider_sid = req.user.service_provider_sid ||
+    req.body.service_provider_sid || parseServiceProviderSid(req);
+  try {
+    res.status(200).json(await getTtsVoices(vendor, label, service_provider_sid, account_sid));
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.post('/languages', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {vendor, label} = req.body;
+  const account_sid = req.user.account_sid || req.body.account_sid;
+  const service_provider_sid = req.user.service_provider_sid ||
+    req.body.service_provider_sid || parseServiceProviderSid(req);
+  try {
+    res.status(200).json(await getTtsLanguages(vendor, label, service_provider_sid, account_sid));
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+const getTtsVoices = async(vendor, label, service_provider_sid, account_sid) => {
+  const credentials = await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+    service_provider_sid, account_sid, vendor, label);
+  const tmp = credentials && credentials.length > 0 ? credentials[0] : null;
+  const cred = tmp ? JSON.parse(decrypt(tmp.credential)) : null;
+  if (vendor === 'elevenlabs') {
+    const get = bent('https://api.elevenlabs.io', 'GET', 'json', {
+      ...(cred && {
+        'xi-api-key' : cred.api_key
+      })
+    });
+    const resp = await get('/v1/voices');
+    return resp ? resp.voices.map((v) => {
+      let name = `${v.name}${v.category !== 'premade' ? ` (${v.category})` : ''} -
+${v.labels.accent ? ` ${v.labels.accent},` : ''}
+${v.labels.description ? ` ${v.labels.description},` : ''}
+${v.labels.age ? ` ${v.labels.age},` : ''}
+${v.labels.gender ? ` ${v.labels.gender},` : ''}
+${v.labels['use case'] ? ` ${v.labels['use case']},` : ''}
+`;
+      const lastIndex = name.lastIndexOf(',');
+      if (lastIndex !== -1) {
+        name = name.substring(0, lastIndex);
+      }
+      return {
+        value: v.voice_id,
+        name
+      };
+    }).sort((a, b) => a.name.localeCompare(b.name)) : [];
+  }
+  return [];
+};
+
+const getTtsLanguages = async(vendor, label, service_provider_sid, account_sid) => {
+  const credentials = await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+    service_provider_sid, account_sid, vendor, label);
+  const tmp = credentials && credentials.length > 0 ? credentials[0] : null;
+  const cred = tmp ? JSON.parse(decrypt(tmp.credential)) : null;
+  if (vendor === 'elevenlabs') {
+    if (!cred) {
+      return [];
+    }
+    const get = bent('https://api.elevenlabs.io', 'GET', 'json', {
+      'xi-api-key' : cred.api_key
+    });
+    const resp = await get('/v1/models');
+    if (!resp || resp.length === 0) {
+      return [];
+    }
+    const model = resp.find((m) => m.model_id === cred.model_id);
+    return model ? model.languages.map((l) => {
+      return {
+        value: l.language_id,
+        name: l.name
+      };
+    }).sort((a, b) => a.name.localeCompare(b.name)) : [];
+  }
+};
+
 module.exports = router;

lib/routes/api/tts-cache.js

@@ -0,0 +1,29 @@
+const router = require('express').Router();
+const {
+  parseAccountSid
+} = require('./utils');
+
+router.delete('/', async(req, res) => {
+  const {purgeTtsCache} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  if (account_sid) {
+    await purgeTtsCache({account_sid});
+  } else {
+    await purgeTtsCache();
+  }
+  res.sendStatus(204);
+});
+
+router.get('/', async(req, res) => {
+  const {getTtsSize} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  let size = 0;
+  if (account_sid) {
+    size = await getTtsSize(`tts:${account_sid}:*`);
+  } else {
+    size = await getTtsSize();
+  }
+  res.status(200).json({size});
+});
+
+module.exports = router;

lib/routes/api/users.js

@@ -338,8 +338,8 @@ router.put('/:user_sid', async(req, res) => {
   //if (req.user.user_sid && req.user.user_sid !== user_sid) return res.sendStatus(403);
 
   if (!hasAdminAuth &&
-  !(hasAccountAuth && req.user.account_sid === user[0].account_sid) &&
-  !(hasServiceProviderAuth && req.user.service_provider_sid === user[0].service_provider_sid) &&
+  !(hasAccountAuth && user[0] && req.user.account_sid === user[0].account_sid) &&
+  !(hasServiceProviderAuth && user[0] && req.user.service_provider_sid === user[0].service_provider_sid) &&
   (req.user.user_sid && req.user.user_sid !== user_sid)) {
     return res.sendStatus(403);
   }

lib/routes/stripe/webhook.js

@@ -61,8 +61,7 @@ router.post('/', express.raw({type: 'application/json'}), async(req, res) => {
   }
 
   /* process event */
-  logger.info(`received webhook: ${evt.type}`);
-  if (evt.type.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
+  if (evt?.type?.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
   else {
     logger.debug(evt, 'unhandled stripe webook');
   }

lib/swagger/swagger.yaml

@@ -786,7 +786,7 @@ paths:
           required: true
           schema:
             type: string
-            example: mycorp.sip.jambonz.us
+            example: mycorp.sip.jambonz.cloud
       responses:
         200:
           description: indicates whether value is already in use
@@ -3046,6 +3046,12 @@ paths:
           enum:
             - inbound
             - outbound
+      - in: query
+        name: filter
+        required: false
+        schema:
+          type: string
+        description: Filter value can be caller ID, callee ID or call Sid
     get:
       tags:
         - Accounts
@@ -3245,6 +3251,18 @@ paths:
           enum:
             - inbound
             - outbound
+      - in: query
+        name: from
+        required: false
+        schema:
+          type: string
+        description: calling number to retrieve
+      - in: query
+        name: to
+        required: false
+        schema:
+          type: string
+        description: called number to retrieve
     get:
       tags:
         - Service Providers
@@ -4146,6 +4164,28 @@ paths:
                       type: string
                     length:
                       type: string
+  /Accounts/{AccountSid}/RegisteredSipUsers:
+    parameters:
+      - name: AccountSid
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+    get:
+      tags:
+        - Accounts
+      summary: retrieve online sip users for an account
+      operationId: listQueues
+      responses:
+        200:
+          description: retrieve online sip users for an account
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
   /Lcrs:
     post:
       tags:

lib/utils/email-utils.js

@@ -47,13 +47,15 @@ const sendEmailByCustomVendor = async(logger, from, to, subject, text) => {
 };
 
 const sendEmailByMailgun = async(logger, from, to, subject, text) => {
-  const mg = mailgun.client({
-    username: 'api',
-    key: process.env.MAILGUN_API_KEY
-  });
   if (!process.env.MAILGUN_API_KEY) throw new Error('MAILGUN_API_KEY env variable is not defined!');
   if (!process.env.MAILGUN_DOMAIN) throw new Error('MAILGUN_DOMAIN env variable is not defined!');
 
+  const mg = mailgun.client({
+    username: 'api',
+    key: process.env.MAILGUN_API_KEY,
+    ...(process.env.MAILGUN_URL && {url: process.env.MAILGUN_URL})
+  });
+
   try {
     const res = await mg.messages.create(process.env.MAILGUN_DOMAIN, {
       from,

lib/utils/encrypt-decrypt.js

@@ -23,7 +23,18 @@ const decrypt = (data) => {
   return decrpyted.toString();
 };
 
+const obscureKey = (key, key_spoiler_length = 6) => {
+  const key_spoiler_char = 'X';
+
+  if (!key || key.length <= key_spoiler_length) {
+    return key;
+  }
+
+  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
+};
+
 module.exports = {
   encrypt,
-  decrypt
+  decrypt,
+  obscureKey
 };

lib/utils/free_plans.json

@@ -2,7 +2,7 @@
   "trial": [
     {
       "category": "voice_call_session",
-      "quantity": 20
+      "quantity": 5
     },
     {
       "category": "device",

lib/utils/homer-utils.js

@@ -64,7 +64,7 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
   }
 };
 
-const getHomerPcap = async(logger, apiKey, callIds) => {
+const getHomerPcap = async(logger, apiKey, callIds, method) => {
   if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
     logger.debug('getHomerPcap: Homer integration not installed');
   }
@@ -73,18 +73,23 @@ const getHomerPcap = async(logger, apiKey, callIds) => {
     const stream = await postPcap('/api/v3/export/call/messages/pcap', {
       param: {
         transaction: {
-          call: true,
-          registration: true,
+          call: method === 'invite',
+          registration: method === 'register',
           rest: false
         },
         orlogic: true,
         search: {
-          '1_call': {
-            callid: callIds
-          },
-          '1_registration': {
-            callid: callIds
-          }
+          ...(method === 'invite' && {
+            '1_call': {
+              callid: callIds
+            }
+          })
+          ,
+          ...(method === 'register' && {
+            '1_registration': {
+              callid: callIds
+            }
+          })
         },
       },
       timestamp: {

lib/utils/jaeger-utils.js

@@ -9,7 +9,8 @@ const getJaegerTrace = async(logger, traceId) => {
   try {
     return await getJSON(`/api/v3/traces/${traceId}`);
   } catch (err) {
-    logger.error({err}, `getJaegerTrace: Error retrieving spans for traceId ${traceId}`);
+    const url = `${process.env.JAEGER_BASE_URL}/api/traces/${traceId}`;
+    logger.error({err, traceId}, `getJaegerTrace: Error retrieving spans from ${url}`);
   }
 };
 

lib/utils/jambonz-sample.text

@@ -0,0 +1 @@
+Hello From Jambonz. This file was created because Record all call bucket credential test.

lib/utils/speech-utils.js

@@ -203,6 +203,29 @@ const testWellSaidTts = async(logger, credentials) => {
   }
 };
 
+const testElevenlabs = async(logger, credentials) => {
+  const {api_key, model_id} = credentials;
+  try {
+    const post = bent('https://api.elevenlabs.io', 'POST', 'buffer', {
+      'xi-api-key': api_key,
+      'Accept': 'audio/mpeg',
+      'Content-Type': 'application/json'
+    });
+    const mp3 = await post('/v1/text-to-speech/21m00Tcm4TlvDq8ikWAM', {
+      text: 'Hello',
+      model_id,
+      voice_settings: {
+        stability: 0.5,
+        similarity_boost: 0.5
+      }
+    });
+    return mp3;
+  } catch (err) {
+    logger.info({err}, 'synthEvenlabs returned error');
+    throw err;
+  }
+};
+
 const testIbmTts = async(logger, getTtsVoices, credentials) => {
   const {tts_api_key, tts_region} = credentials;
   const voices = await getTtsVoices({vendor: 'ibm', credentials: {tts_api_key, tts_region}});
@@ -251,5 +274,6 @@ module.exports = {
   testDeepgramStt,
   testIbmTts,
   testIbmStt,
-  testSonioxStt
+  testSonioxStt,
+  testElevenlabs
 };

lib/utils/storage-utils.js

@@ -0,0 +1,134 @@
+const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3');
+const {Storage} = require('@google-cloud/storage');
+const fs = require('fs');
+const { BlobServiceClient } = require('@azure/storage-blob');
+
+// Azure
+
+async function testAzureStorage(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient('jambonz-sample.text');
+
+  await blockBlobClient.uploadFile(`${__dirname}/jambonz-sample.text`);
+}
+
+async function getAzureStorageObject(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient(opts.key);
+  const response = await blockBlobClient.download(0);
+  return response.readableStreamBody;
+}
+
+async function deleteAzureStorageObject(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient(opts.key);
+  await blockBlobClient.delete();
+}
+
+// Google
+
+function _initGoogleClient(opts) {
+  const serviceKey = JSON.parse(opts.service_key);
+  return new Storage({
+    projectId: serviceKey.project_id,
+    credentials: {
+      client_email: serviceKey.client_email,
+      private_key: serviceKey.private_key
+    },
+  });
+}
+
+async function testGoogleStorage(logger, opts) {
+  return new Promise((resolve, reject) => {
+    const storage = _initGoogleClient(opts);
+
+    const blob = storage.bucket(opts.name).file('jambonz-sample.text');
+
+    fs.createReadStream(`${__dirname}/jambonz-sample.text`)
+      .pipe(blob.createWriteStream())
+      .on('error', (err) => reject(err))
+      .on('finish', () => resolve());
+  });
+}
+
+async function getGoogleStorageObject(logger, opts) {
+  const storage = _initGoogleClient(opts);
+
+  const bucket = storage.bucket(opts.name);
+  const file = bucket.file(opts.key);
+  const [exists] = await file.exists();
+  if (exists) {
+    return file.createReadStream();
+  }
+}
+
+async function deleteGoogleStorageObject(logger, opts) {
+  const storage = _initGoogleClient(opts);
+
+  const bucket = storage.bucket(opts.name);
+  const file = bucket.file(opts.key);
+
+  await file.delete();
+}
+
+// S3
+
+function _initS3Client(opts) {
+  return new S3Client({
+    credentials: {
+      accessKeyId: opts.access_key_id,
+      secretAccessKey: opts.secret_access_key,
+    },
+    region: opts.region || 'us-east-1',
+    ...(opts.vendor === 's3_compatible' && { endpoint: opts.endpoint, forcePathStyle: true })
+  });
+}
+
+async function testS3Storage(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const input = {
+    'Body': 'Hello From Jambonz',
+    'Bucket': opts.name,
+    'Key': 'jambonz-sample.text'
+  };
+  const command = new PutObjectCommand(input);
+  await s3.send(command);
+}
+
+async function getS3Object(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const command = new GetObjectCommand(
+    {
+      Bucket: opts.name,
+      Key: opts.key
+    }
+  );
+  const res = await s3.send(command);
+  return res.Body;
+}
+
+async function deleteS3Object(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const command = new DeleteObjectCommand(
+    {
+      Bucket: opts.name,
+      Key: opts.key
+    }
+  );
+  await s3.send(command);
+}
+
+module.exports = {
+  testS3Storage,
+  getS3Object,
+  deleteS3Object,
+  testGoogleStorage,
+  getGoogleStorageObject,
+  deleteGoogleStorageObject,
+  testAzureStorage,
+  getAzureStorageObject,
+  deleteAzureStorageObject
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jambonz-api-server",
-  "version": "0.8.3",
+  "version": "0.8.4",
   "description": "",
   "main": "app.js",
   "scripts": {
@@ -19,15 +19,20 @@
     "url": "https://github.com/jambonz/jambonz-api-server.git"
   },
   "dependencies": {
-    "@aws-sdk/client-transcribe": "^3.290.0",
-    "@deepgram/sdk": "^1.10.2",
-    "@google-cloud/speech": "^5.1.0",
-    "@jambonz/db-helpers": "^0.7.9",
-    "@jambonz/realtimedb-helpers": "^0.7.2",
-    "@jambonz/speech-utils": "^0.0.12",
-    "@jambonz/time-series": "^0.2.5",
-    "@jambonz/verb-specifications": "^0.0.21",
-    "@soniox/soniox-node": "^1.1.0",
+    "@aws-sdk/client-s3": "^3.363.0",
+    "@aws-sdk/client-transcribe": "^3.363.0",
+    "@azure/storage-blob": "^12.15.0",
+    "@deepgram/sdk": "^1.21.0",
+    "@google-cloud/speech": "^5.2.0",
+    "@google-cloud/storage": "^6.12.0",
+    "@jambonz/db-helpers": "^0.9.0",
+    "@jambonz/lamejs": "^1.2.2",
+    "@jambonz/mw-registrar": "^0.2.5",
+    "@jambonz/realtimedb-helpers": "^0.8.7",
+    "@jambonz/speech-utils": "^0.0.15",
+    "@jambonz/time-series": "^0.2.8",
+    "@jambonz/verb-specifications": "^0.0.29",
+    "@soniox/soniox-node": "^1.1.1",
     "argon2": "^0.30.3",
     "bent": "^7.3.12",
     "cors": "^2.8.5",
@@ -38,7 +43,7 @@
     "helmet": "^5.1.0",
     "ibm-watson": "^7.1.2",
     "jsonwebtoken": "^9.0.0",
-    "mailgun.js": "^3.7.3",
+    "mailgun.js": "^9.1.2",
     "microsoft-cognitiveservices-speech-sdk": "^1.24.1",
     "mysql2": "^2.3.3",
     "nocache": "3.0.4",
@@ -49,6 +54,8 @@
     "stripe": "^8.222.0",
     "swagger-ui-express": "^4.4.0",
     "uuid": "^8.3.2",
+    "wav": "^1.0.2",
+    "ws": "^8.12.1",
     "yamljs": "^0.3.0"
   },
   "devDependencies": {

test/accounts.js

@@ -14,7 +14,7 @@ const {
   createPhoneNumber, 
   deleteObjectBySid} = require('./utils');
 const logger = require('../lib/logger');
-const { pushBack } = require('@jambonz/realtimedb-helpers')({
+const { addToSortedSet } = require('@jambonz/realtimedb-helpers')({
   host: process.env.JAMBONES_REDIS_HOST,
   port: process.env.JAMBONES_REDIS_PORT || 6379
 }, logger);
@@ -168,11 +168,33 @@ test('account tests', async(t) => {
         queue_event_hook: {
           url: 'http://example.com/q',
           method: 'post'
+        },
+        record_all_calls: true,
+        record_format: 'wav',
+        bucket_credential: {
+          vendor: 'aws_s3',
+          region: 'us-east-1',
+          name: 'recordings',
+          access_key_id: 'access_key_id',
+          secret_access_key: 'secret access key'
         }
+        
       }
     });
     t.ok(result.statusCode === 204, 'successfully updated account using account level token');
 
+    /* verify that bucket credential is updated*/
+    result = await request.get(`/Accounts/${sid}`, {
+      auth: {bearer: accountLevelToken},
+      json: true,
+    });
+
+    t.ok(result.bucket_credential.vendor === 'aws_s3', 'bucket_vendor was updated');
+    t.ok(result.bucket_credential.name === 'recordings', 'bucket_name was updated');
+    t.ok(result.bucket_credential.access_key_id === 'access_key_id', 'bucket_access_key_id was updated');
+    t.ok(result.record_all_calls === 1, 'record_all_calls was updated');
+    t.ok(result.record_format === 'wav', 'record_format was updated');
+
     /* verify that account level api key last_used was updated*/
     result = await request.get(`/Accounts/${sid}/ApiKeys`, {
       auth: {bearer: accountLevelToken},
@@ -268,8 +290,8 @@ test('account tests', async(t) => {
     t.ok(result.statusCode === 204, 'successfully deleted a call session limit for an account');
 
     /* query account queues */
-    await pushBack(`queue:${sid}:test`, 'url1');
-    await pushBack(`queue:${sid}:dummy`, 'url2');
+    await addToSortedSet(`queue:${sid}:test`, 'url1');
+    await addToSortedSet(`queue:${sid}:dummy`, 'url2');
 
     result = await request.get(`/Accounts/${sid}/Queues`, {
       auth: authAdmin,

test/applications.js

@@ -53,7 +53,7 @@ test('application tests', async(t) => {
         ]'
       }
     });
-    t.ok(result.statusCode === 400, 'Cant create application with invalid app_josn');
+    t.ok(result.statusCode === 400, 'Cant create application with invalid app_json');
     
     /* add an application */
     result = await request.post('/Applications', {
@@ -81,7 +81,15 @@ test('application tests', async(t) => {
               "seekOffset": 8000,\
               "actionHook": "/play/action"\
           }\
-        ]'
+        ]',
+        use_for_fallback_speech: 1,
+        fallback_speech_synthesis_vendor: 'google',
+        fallback_speech_synthesis_language: 'en-US',
+        fallback_speech_synthesis_voice: 'man',
+        fallback_speech_synthesis_label: 'label1',
+        fallback_speech_recognizer_vendor: 'google',
+        fallback_speech_recognizer_language: 'en-US',
+        fallback_speech_recognizer_label: 'label1'
       }
     });
     t.ok(result.statusCode === 201, 'successfully created application');
@@ -102,6 +110,14 @@ test('application tests', async(t) => {
     });
     t.ok(result.name === 'daveh' , 'successfully retrieved application by sid');
     t.ok(result.messaging_hook.url === 'http://example.com/sms' , 'successfully retrieved messaging_hook from application');
+    t.ok(result.use_for_fallback_speech === 1, 'successfully create use_for_fallback_speech');
+    t.ok(result.fallback_speech_synthesis_vendor === 'google', 'successfully create fallback_speech_synthesis_vendor');
+    t.ok(result.fallback_speech_synthesis_language === 'en-US', 'successfully create fallback_speech_synthesis_language');
+    t.ok(result.fallback_speech_synthesis_voice === 'man', 'successfully create fallback_speech_synthesis_voice');
+    t.ok(result.fallback_speech_synthesis_label === 'label1', 'successfully create fallback_speech_synthesis_label');
+    t.ok(result.fallback_speech_recognizer_vendor === 'google', 'successfully create fallback_speech_recognizer_vendor');
+    t.ok(result.fallback_speech_recognizer_language === 'en-US', 'successfully create fallback_speech_recognizer_language');
+    t.ok(result.fallback_speech_recognizer_label === 'label1', 'successfully create fallback_speech_recognizer_label');
     let app_json = JSON.parse(result.app_json);
     t.ok(app_json[0].verb === 'play', 'successfully retrieved app_json from application')
 
@@ -125,7 +141,16 @@ test('application tests', async(t) => {
               "X-Reason" : "maximum call duration exceeded"\
             }\
           }\
-        ]'
+        ]',
+        record_all_calls: true,
+        use_for_fallback_speech: 0,
+        fallback_speech_synthesis_vendor: 'microsoft',
+        fallback_speech_synthesis_language: 'en-US',
+        fallback_speech_synthesis_voice: 'woman',
+        fallback_speech_synthesis_label: 'label2',
+        fallback_speech_recognizer_vendor: 'microsoft',
+        fallback_speech_recognizer_language: 'en-US',
+        fallback_speech_recognizer_label: 'label2'
       }
     });
     t.ok(result.statusCode === 204, 'successfully updated application');
@@ -138,6 +163,15 @@ test('application tests', async(t) => {
     t.ok(result.messaging_hook.url === 'http://example2.com/mms' , 'successfully updated messaging_hook');
     app_json = JSON.parse(result.app_json);
     t.ok(app_json[0].verb === 'hangup', 'successfully updated app_json from application')
+    t.ok(result.record_all_calls === 1, 'successfully updated record_all_calls from application')
+    t.ok(result.use_for_fallback_speech === 0, 'successfully update use_for_fallback_speech');
+    t.ok(result.fallback_speech_synthesis_vendor === 'microsoft', 'successfully update fallback_speech_synthesis_vendor');
+    t.ok(result.fallback_speech_synthesis_language === 'en-US', 'successfully update fallback_speech_synthesis_language');
+    t.ok(result.fallback_speech_synthesis_voice === 'woman', 'successfully update fallback_speech_synthesis_voice');
+    t.ok(result.fallback_speech_synthesis_label === 'label2', 'successfully update fallback_speech_synthesis_label');
+    t.ok(result.fallback_speech_recognizer_vendor === 'microsoft', 'successfully update fallback_speech_recognizer_vendor');
+    t.ok(result.fallback_speech_recognizer_language === 'en-US', 'successfully update fallback_speech_recognizer_language');
+    t.ok(result.fallback_speech_recognizer_label === 'label2', 'successfully update fallback_speech_recognizer_label');
 
     /* remove applications app_json*/
     result = await request.put(`/Applications/${sid}`, {

test/call-test.js

@@ -31,8 +31,8 @@ test('Create Call Success With Synthesizer in Payload', async (t) => {
     auth: authUser,
     json: true,
     body: {
-      call_hook: "https://public-apps.jambonz.us/hello-world",
-      call_status_hook: "https://public-apps.jambonz.us/call-status",
+      call_hook: "https://public-apps.jambonz.cloud/hello-world",
+      call_status_hook: "https://public-apps.jambonz.cloud/call-status",
       from: "15083778299",
       to: {
         type: "phone",
@@ -73,8 +73,8 @@ test('Create Call Success Without Synthesizer in Payload', async (t) => {
     auth: authUser,
     json: true,
     body: {
-      call_hook: "https://public-apps.jambonz.us/hello-world",
-      call_status_hook: "https://public-apps.jambonz.us/call-status",
+      call_hook: "https://public-apps.jambonz.cloud/hello-world",
+      call_status_hook: "https://public-apps.jambonz.cloud/call-status",
       from: "15083778299",
       to: {
         type: "phone",
@@ -83,4 +83,74 @@ test('Create Call Success Without Synthesizer in Payload', async (t) => {
     }
   }).then(data => { t.ok(false, 'Create Call should not be success') })
     .catch(error => { t.ok(error.response.statusCode === 400, 'Call failed for no synthesizer') });
+});
+
+test("Create call with application sid and app_json", async(t) => {
+  const app = require('../app');
+
+  const service_provider_sid = await createServiceProvider(request, 'account3_has_synthesizer');
+  const account_sid = await createAccount(request, service_provider_sid, 'account3_has_synthesizer');
+
+  const token = jwt.sign({
+    account_sid,
+    scope: "account",
+    permissions: ["PROVISION_USERS", "PROVISION_SERVICES", "VIEW_ONLY"]
+  }, process.env.JWT_SECRET, { expiresIn: '1h' });
+  const authUser = { bearer: token };
+  const speech_sid = await createGoogleSpeechCredentials(request, account_sid, null, authUser, true, true);
+
+  // GIVEN
+/* add an application */
+
+const app_json = '[\
+  {\
+    "verb": "play",\
+    "url": "https://example.com/example.mp3",\
+    "timeoutSecs": 10,\
+    "seekOffset": 8000,\
+    "actionHook": "/play/action"\
+}\
+]';
+let result = await request.post('/Applications', {
+  resolveWithFullResponse: true,
+  auth: authUser,
+  json: true,
+  body: {
+    name: 'daveh',
+    account_sid,
+    call_hook: {
+      url: 'http://example.com'
+    },
+    call_status_hook: {
+      url: 'http://example.com/status',
+      method: 'POST'
+    },
+    messaging_hook: {
+      url: 'http://example.com/sms'
+    },
+    app_json
+  }
+});
+t.ok(result.statusCode === 201, 'successfully created application');
+const sid = result.body.sid;
+
+// WHEN
+result = await request.post(`/Accounts/${account_sid}/Calls`, {
+  resolveWithFullResponse: true,
+  auth: authUser,
+  json: true,
+  body: {
+    application_sid: sid,
+    from: "15083778299",
+    to: {
+      type: "phone",
+      number: "15089084809"
+    },
+  }
+});
+// THEN
+t.ok(result.statusCode === 201, 'successfully created Call without Synthesizer && application_sid');
+const fs_request = await getLastRequestFromFeatureServer('15083778299_createCall');
+const obj = JSON.parse(fs_request);
+t.ok(obj.body.app_json == app_json, 'app_json successfully added')
 });

test/clients.js

@@ -0,0 +1,141 @@
+const test = require('tape') ;
+const ADMIN_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734de';
+const authAdmin = {bearer: ADMIN_TOKEN};
+const request = require('request-promise-native').defaults({
+  baseUrl: 'http://127.0.0.1:3000/v1'
+});
+
+
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+test('client test', async(t) => {
+  const app = require('../app');
+  const {registrar} = app.locals;
+
+  try {
+    let result;
+    /* add a service provider */
+    result = await request.post('/ServiceProviders', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        name: 'client_sp',
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created client service provider');
+    const sp_sid = result.body.sid;
+
+    /* add an account */
+    result = await request.post('/Accounts', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        name: 'sample_account',
+        service_provider_sid: sp_sid,
+        sip_realm: 'drachtio.org',
+        registration_hook: {
+          url: 'http://example.com/reg',
+          method: 'get'
+        },
+        webhook_secret: 'foobar'
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created account');
+    const account_sid = result.body.sid;
+
+    /* add new entity */
+    result = await request.post('/Clients', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        account_sid,
+        username: 'client1',
+        password: 'sdf12412',
+        is_active: 1
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created Client');
+    const sid = result.body.sid;
+
+    /* register the client */
+    const r = await registrar.add(
+      "dhorton@drachtio.org",
+      {
+        contact: "10.10.1.1",
+        sbcAddress: "192.168.1.1",
+        protocol: "udp",
+      },
+      5
+    );
+    t.ok(r, 'successfully registered Client');
+
+    /* query all registered clients */
+    result = await request.get(`/Accounts/${account_sid}/RegisteredSipUsers`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 1 && result[0] === 'dhorton@drachtio.org', 
+      'successfully queried all registered clients');
+
+    /* query all entity */
+    result = await request.get('/Clients', {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 1 , 'successfully queried all Clients');
+
+    /* query one entity */
+    result = await request.get(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.account_sid === account_sid , 'successfully retrieved Client by sid');
+    t.ok(result.client_sid, 'successfully retrieved Client by sid');
+    t.ok(result.username ===  'client1', 'successfully retrieved Client by sid');
+    t.ok(result.is_active === 1 , 'successfully retrieved Client by sid');
+    t.ok(result.password === 'sXXXXXXX' , 'successfully retrieved Client by sid');
+
+    /* update the entity */
+    result = await request.put(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+      resolveWithFullResponse: true,
+      body: {
+        is_active: 0
+      }
+    });
+    t.ok(result.statusCode === 204, 'successfully updated Client');
+    /* query one entity */
+    result = await request.get(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.is_active === 0 , 'successfully updated Client');
+    t.ok(result.password === 'sXXXXXXX' , 'successfully retrieved Client by sid');
+
+    /* delete Client */
+    result = await request.delete(`/Clients/${sid}`, {
+      resolveWithFullResponse: true,
+      simple: false,
+      json: true,
+      auth: authAdmin
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted Clients');
+
+    /* query all entity */
+    result = await request.get('/Clients', {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 0 , 'successfully queried all Clients');
+    
+  } catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+})

test/index.js

@@ -23,4 +23,6 @@ require('./system-information');
 require('./lcr-carriers-set-entries');
 require('./lcr-routes');
 require('./lcrs');
+require('./tts-cache');
+require('./clients');
 require('./docker_stop');

test/oauth/gh-get-user.js

@@ -7,7 +7,7 @@ const test = async() => {
     headers: {
       Authorization: `Bearer ${process.env.GH_CODE}`,
       Accept: 'application/json',
-      'User-Agent': 'jambonz.us'
+      'User-Agent': 'jambonz.cloud'
     }
   }, (err, response, body) => {
     if (err) console.log(error);

test/recent-calls.js

@@ -75,8 +75,29 @@ test('recent calls tests', async(t) => {
       auth: authUser,
       json: true,
     });
+    console.log(JSON.stringify(result));
     t.ok(result.data.length === 5, 'retrieved 5 recent calls by account');
-    //console.log({data: result.data}, 'Account recent calls');
+
+    result = await request.get(`/Accounts/${account_sid}/RecentCalls?page=1&count=25&filter=1508`, {
+      auth: authUser,
+      json: true,
+    });
+    console.log(JSON.stringify(result));
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by account and from');
+
+    result = await request.get(`/Accounts/${account_sid}/RecentCalls?page=1&count=25&filter=15080`, {
+      auth: authUser,
+      json: true,
+    });
+    console.log(JSON.stringify(result));
+    t.ok(result.data.length === 0, 'retrieved 0 recent calls by account and non-matching from');
+
+    result = await request.get(`/Accounts/${account_sid}/RecentCalls?page=1&count=25&filter=1888`, {
+      auth: authUser,
+      json: true,
+    });
+    console.log(JSON.stringify(result));
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by account and to');
 
     /* query last 7 days by service provider */
     result = await request.get(`/ServiceProviders/${service_provider_sid}/RecentCalls?page=1&count=25`, {
@@ -84,6 +105,19 @@ test('recent calls tests', async(t) => {
       json: true,
     });
     t.ok(result.data.length === 5, 'retrieved 5 recent calls by service provider');
+
+    result = await request.get(`/ServiceProviders/${service_provider_sid}/RecentCalls?page=1&count=25&filter=1508`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by service provider and from');
+
+    result = await request.get(`/ServiceProviders/${service_provider_sid}/RecentCalls?page=1&count=25&filter=1888`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by service provider and to');
+
     //console.log({data: result.data}, 'SP recent calls');
 
     /* pull sip traces and pcap from homer */

test/sbcs.js

@@ -17,6 +17,37 @@ test('sbc_addresses tests', async(t) => {
     let result;
     const service_provider_sid = await createServiceProvider(request);
 
+    /* add service_provider user */
+    const sp_name = 'sbc_service_provider';
+    const sp_password = 'password';
+    result = await request.post(`/Users`, {
+      resolveWithFullResponse: true,
+      json: true,
+      auth: authAdmin,
+      body: {
+        name: sp_name,
+        email: 'sbc_sp@jambonz.com',
+        is_active: true,
+        force_change: false,
+        initial_password: sp_password,
+        service_provider_sid,
+      }
+    });
+    t.ok(result.statusCode === 201 && result.body.user_sid, 'SBC service_provider scope user created');
+    const sbc_sp_user_sid = result.body.user_sid;
+
+    result = await request.post('/login', {
+      resolveWithFullResponse: true,
+      json: true,
+      body: {
+        username: sp_name,
+        password: sp_password,  
+      }
+    });
+    t.ok(result.statusCode === 200 && result.body.token, 'successfully logged in as sbc user');
+    const authSbcSp = {bearer: result.body.token};
+
+
     /* add a service provider sbc */
     result = await request.post('/Sbcs', {
       resolveWithFullResponse: true,
@@ -38,6 +69,20 @@ test('sbc_addresses tests', async(t) => {
     //console.log(result.body)
     t.ok(result.body.length === 1 && result.body[0].ipv4 === '192.168.1.4', 'successfully retrieved service provider sbc');
 
+    result = await request.get('/Sbcs', {
+      resolveWithFullResponse: true,
+      auth: authSbcSp,
+      json: true
+    });
+    //console.log(result.body)
+    t.ok(result.body.length === 1 && result.body[0].ipv4 === '192.168.1.4', 'successfully retrieved service provider sbc');
+
+    await request.delete(`/Users/${sbc_sp_user_sid}`, {
+      resolveWithFullResponse: true,
+      json: true,
+      auth: authAdmin,
+    });
+
     await deleteObjectBySid(request, '/Sbcs', sid);
     await deleteObjectBySid(request, '/ServiceProviders', service_provider_sid);
 

test/speech-credentials.js

@@ -84,6 +84,7 @@ test('speech credentials tests', async(t) => {
       json: true,
       body: {
         vendor: 'google',
+        label: 'label1',
         service_key: jsonKey,
         use_for_tts: true,
         use_for_stt: true
@@ -111,6 +112,7 @@ test('speech credentials tests', async(t) => {
       json: true,
     });
     t.ok(result.vendor === 'google' , 'successfully retrieved speech credential by sid');
+    t.ok(result.label === 'label1' , 'label is successfully created');
     
     /* query all credentials */
     result = await request.get(`/Accounts/${account_sid}/SpeechCredentials`, {
@@ -445,6 +447,51 @@ test('speech credentials tests', async(t) => {
     });
     t.ok(result.statusCode === 204, 'successfully deleted speech credential');
 
+    /* add a credential for cobalt */
+    result = await request.post(`/Accounts/${account_sid}/SpeechCredentials`, {
+      resolveWithFullResponse: true,
+      auth: authUser,
+      json: true,
+      body: {
+        vendor: 'cobalt',
+        use_for_stt: true,
+        use_for_tts: false,
+        cobalt_server_uri: '32.32.32.32:2727',
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added speech credential for Cobalt');
+    const cobalt_sid = result.body.sid;
+
+    /* delete the credential */
+    result = await request.delete(`/Accounts/${account_sid}/SpeechCredentials/${cobalt_sid}`, {
+      auth: authUser,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted speech credential for Cobalt');
+
+    /* add a credential for elevenlabs */
+    result = await request.post(`/Accounts/${account_sid}/SpeechCredentials`, {
+      resolveWithFullResponse: true,
+      auth: authUser,
+      json: true,
+      body: {
+        vendor: 'elevenlabs',
+        use_for_stt: true,
+        use_for_tts: false,
+        api_key: 'asdasdasdasddsadasda',
+        model_id: 'eleven_multilingual_v2'
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added speech credential for Cobalt');
+    const elevenlabs_sid = result.body.sid;
+
+    /* delete the credential */
+    result = await request.delete(`/Accounts/${account_sid}/SpeechCredentials/${elevenlabs_sid}`, {
+      auth: authUser,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted speech credential for Cobalt');
+
     await deleteObjectBySid(request, '/Accounts', account_sid);
     await deleteObjectBySid(request, '/ServiceProviders', service_provider_sid);
     t.end();

test/tts-cache.js

@@ -0,0 +1,56 @@
+const test = require('tape') ;
+const jwt = require('jsonwebtoken');
+const ADMIN_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734de';
+const authAdmin = {bearer: ADMIN_TOKEN};
+const request = require('request-promise-native').defaults({
+  baseUrl: 'http://127.0.0.1:3000/v1'
+});
+const crypto = require('crypto');
+const logger = require('../lib/logger');
+
+const {
+  client,
+} = require('@jambonz/speech-utils')({
+  host: process.env.JAMBONES_REDIS_HOST,
+  port: process.env.JAMBONES_REDIS_PORT || 6379
+}, logger);
+
+function makeSynthKey({account_sid = '', vendor, language, voice, engine = '', text}) {
+  const hash = crypto.createHash('sha1');
+  hash.update(`${language}:${vendor}:${voice}:${engine}:${text}`);
+  return `tts${account_sid ? (':' + account_sid) : ''}:${hash.digest('hex')}`;
+}
+
+test('tts-cache', async(t) => {
+  const app = require('../app');
+  try {
+    // create caches
+    const minRecords = 8;
+    for (const i in Array(minRecords).fill(0)) {
+      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, text: i}), i);
+    }
+
+    let result = await request.get('/TtsCache', {
+      auth: authAdmin,
+      json: true,
+    });
+
+    t.ok(result.size === minRecords, 'get cache correctly');
+
+    result = await request.delete('/TtsCache', {
+      auth: authAdmin,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted application after removing phone number');
+
+    result = await request.get('/TtsCache', {
+      auth: authAdmin,
+      json: true,
+    });
+
+    t.ok(result.size === 0, 'deleted cache successfully');
+  } catch(err) {
+    console.error(err);
+    t.end(err);
+  }
+});

test/webapp_tests.js

@@ -192,7 +192,7 @@ test('webapp tests', async(t) => {
     t.ok(result.statusCode === 200 && result.body.available === true, 'indicates when email is available');
    
     /* check if a subdomain is available */
-    result = await request.get('/Availability?type=subdomain&value=mycompany.sip.jambonz.us', {
+    result = await request.get('/Availability?type=subdomain&value=mycompany.sip.jambonz.cloud', {
       resolveWithFullResponse: true,
       auth: authUser,
       json: true,