node image upgrade

* let realtimedb-help build redis configuraiton from env variables

* update speech-utils version

.github/workflows/docker-publish-dbcreate.yml

@@ -2,16 +2,10 @@ name: Docker
 
 on:
   push:
-    # Publish `main` as Docker `latest` image.
     branches:
       - main
-
-    # Publish `v1.2.3` tags as releases.
     tags:
-      - v*
-
-env:
-  IMAGE_NAME: db-create
+      - '*'
 
 jobs:
   push:
@@ -20,32 +14,42 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build . --file Dockerfile.db-create --tag $IMAGE_NAME
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Push image
+      - name: prepare tag 
+        id: prepare_tag
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+            IMAGE_ID=jambonz/db-create
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
 
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
 
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "main" ] && VERSION=latest
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
 
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.db-create
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.github/workflows/docker-publish.yml

@@ -2,6 +2,8 @@ name: Docker
 
 on:
   push:
+    branches:
+      - main
     tags:
       - '*'
 
@@ -18,7 +20,7 @@ jobs:
       - name: prepare tag 
         id: prepare_tag
         run: |
-            IMAGE_ID=$GITHUB_REPOSITORY
+            IMAGE_ID=jambonz/api-server
 
             # Strip git ref prefix from version
             VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
+FROM --platform=linux/amd64 node:18.18.2-alpine3.16 as base
 
 RUN apk --update --no-cache add --virtual .builds-deps build-base python3
 

README.md

@@ -33,6 +33,8 @@ Configuration is provided via environment variables:
 |K8S| service running as kubernetes service |no|
 |K8S_FEATURE_SERVER_SERVICE_NAME| feature server name(required for K8S) |no|
 |K8S_FEATURE_SERVER_SERVICE_PORT| feature server port(required for K8S) |no|
+|JAMBONZ_RECORD_WS_USERNAME| recording websocket username|no|
+|JAMBONZ_RECORD_WS_PASSWORD| recording websocket password|no|
 
 #### Database dependency
 A mysql database is used to store long-lived objects such as Accounts, Applications, etc. To create the database schema, use or review the scripts in the 'db' folder, particularly:

app.js

@@ -8,12 +8,18 @@ const rateLimit = require('express-rate-limit');
 const cors = require('cors');
 const passport = require('passport');
 const routes = require('./lib/routes');
+const Registrar = require('@jambonz/mw-registrar');
 
 assert.ok(process.env.JAMBONES_MYSQL_HOST &&
   process.env.JAMBONES_MYSQL_USER &&
   process.env.JAMBONES_MYSQL_PASSWORD &&
   process.env.JAMBONES_MYSQL_DATABASE, 'missing JAMBONES_MYSQL_XXX env vars');
-assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+if (process.env.JAMBONES_REDIS_SENTINELS) {
+  assert.ok(process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+    'missing JAMBONES_REDIS_SENTINEL_MASTER_NAME env var, JAMBONES_REDIS_SENTINEL_PASSWORD env var is optional');
+} else {
+  assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+}
 assert.ok(process.env.JAMBONES_TIME_SERIES_HOST, 'missing JAMBONES_TIME_SERIES_HOST env var');
 assert.ok(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET, 'missing ENCRYPTION_SECRET env var');
 assert.ok(process.env.JWT_SECRET, 'missing JWT_SECRET env var');
@@ -30,10 +36,11 @@ const {
   logger, process.env.JAMBONES_TIME_SERIES_HOST
 );
 const {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
@@ -42,11 +49,11 @@ const {
   incrKey
 } = require('./lib/helpers/realtimedb-helpers');
 const {
-  getTtsVoices
-} = require('@jambonz/speech-utils')({
-  host: process.env.JAMBONES_REDIS_HOST,
-  port: process.env.JAMBONES_REDIS_PORT || 6379
-}, logger);
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache,
+  synthAudio
+} = require('@jambonz/speech-utils')({redis_client: client}, logger);
 const {
   lookupAppBySid,
   lookupAccountBySid,
@@ -54,7 +61,8 @@ const {
   lookupAppByPhoneNumber,
   lookupCarrierBySid,
   lookupSipGatewayBySid,
-  lookupSmppGatewayBySid
+  lookupSmppGatewayBySid,
+  lookupClientByAccountAndUsername
 } = require('@jambonz/db-helpers')({
   host: process.env.JAMBONES_MYSQL_HOST,
   user: process.env.JAMBONES_MYSQL_USER,
@@ -66,17 +74,19 @@ const {
 const PORT = process.env.HTTP_PORT || 3000;
 const authStrategy = require('./lib/auth')(logger, retrieveKey);
 const {delayLoginMiddleware} = require('./lib/middleware');
+const Websocket = require('ws');
 
 passport.use(authStrategy);
 
 app.locals = app.locals || {};
 app.locals = {
   ...app.locals,
+  registrar: new Registrar(logger, client),
   logger,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
@@ -84,6 +94,9 @@ app.locals = {
   retrieveKey,
   deleteKey,
   getTtsVoices,
+  getTtsSize,
+  purgeTtsCache,
+  synthAudio,
   lookupAppBySid,
   lookupAccountBySid,
   lookupAccountByPhoneNumber,
@@ -91,6 +104,7 @@ app.locals = {
   lookupCarrierBySid,
   lookupSipGatewayBySid,
   lookupSmppGatewayBySid,
+  lookupClientByAccountAndUsername,
   queryCdrs,
   queryCdrsSP,
   queryAlerts,
@@ -114,6 +128,12 @@ const limiter = rateLimit({
   legacyHeaders: false, // Disable the `X-RateLimit-*` headers
 });
 
+// Setup websocket for recording audio
+const recordWsServer = require('./lib/record');
+const wsServer = new Websocket.Server({ noServer: true });
+wsServer.setMaxListeners(0);
+wsServer.on('connection', recordWsServer.bind(null, logger));
+
 if (process.env.JAMBONES_TRUST_PROXY) {
   const proxyCount = parseInt(process.env.JAMBONES_TRUST_PROXY);
   if (!isNaN(proxyCount) && proxyCount > 0) {
@@ -154,7 +174,52 @@ app.use((err, req, res, next) => {
   });
 });
 logger.info(`listening for HTTP traffic on port ${PORT}`);
-app.listen(PORT);
+const server = app.listen(PORT);
+
+
+const isValidWsKey = (hdr) => {
+  const username = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
+  const password = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
+  if (username && password) {
+    if (!hdr) {
+      // auth header is missing
+      return false;
+    }
+    const token = Buffer.from(`${username}:${password}`).toString('base64');
+    const arr = /^Basic (.*)$/.exec(hdr);
+    if (!Array.isArray(arr)) {
+      // malformed auth header
+      return false;
+    }
+    return arr[1] === token;
+  }
+  return true;
+};
+
+server.on('upgrade', (request, socket, head) => {
+  logger.debug({
+    url: request.url,
+    headers: request.headers,
+  }, 'received upgrade request');
+
+  /* verify the path starts with /transcribe */
+  if (!request.url.includes('/record/')) {
+    logger.info(`unhandled path: ${request.url}`);
+    return socket.write('HTTP/1.1 404 Not Found \r\n\r\n', () => socket.destroy());
+  }
+
+  /* verify the api key */
+  if (!isValidWsKey(request.headers['authorization'])) {
+    logger.info(`invalid auth header: ${request.headers['authorization'] || 'authorization header missing'}`);
+    return socket.write('HTTP/1.1 403 Forbidden \r\n\r\n', () => socket.destroy());
+  }
+
+  /* complete the upgrade */
+  wsServer.handleUpgrade(request, socket, head, (ws) => {
+    logger.info(`upgraded to websocket, url: ${request.url}`);
+    wsServer.emit('connection', ws, request.url);
+  });
+});
 
 // purge old calls from active call set every 10 mins
 async function purge() {

db/jambones-sql.sql

@@ -14,6 +14,8 @@ DROP TABLE IF EXISTS beta_invite_codes;
 
 DROP TABLE IF EXISTS call_routes;
 
+DROP TABLE IF EXISTS clients;
+
 DROP TABLE IF EXISTS dns_records;
 
 DROP TABLE IF EXISTS lcr;
@@ -52,6 +54,8 @@ DROP TABLE IF EXISTS signup_history;
 
 DROP TABLE IF EXISTS smpp_addresses;
 
+DROP TABLE IF EXISTS google_custom_voices;
+
 DROP TABLE IF EXISTS speech_credentials;
 
 DROP TABLE IF EXISTS system_information;
@@ -128,6 +132,19 @@ application_sid CHAR(36) NOT NULL,
 PRIMARY KEY (call_route_sid)
 ) COMMENT='a regex-based pattern match for call routing';
 
+CREATE TABLE clients
+(
+client_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+is_active BOOLEAN NOT NULL DEFAULT 1,
+username VARCHAR(64),
+password VARCHAR(1024),
+allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1,
+PRIMARY KEY (client_sid)
+);
+
 CREATE TABLE dns_records
 (
 dns_record_sid CHAR(36) NOT NULL UNIQUE ,
@@ -323,9 +340,20 @@ last_tested DATETIME,
 tts_tested_ok BOOLEAN,
 stt_tested_ok BOOLEAN,
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+label VARCHAR(64),
 PRIMARY KEY (speech_credential_sid)
 );
 
+CREATE TABLE google_custom_voices
+(
+google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+speech_credential_sid CHAR(36) NOT NULL,
+model VARCHAR(512) NOT NULL,
+reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+name VARCHAR(64) NOT NULL,
+PRIMARY KEY (google_custom_voice_sid)
+);
+
 CREATE TABLE system_information
 (
 domain_name VARCHAR(255),
@@ -412,7 +440,7 @@ PRIMARY KEY (smpp_gateway_sid)
 CREATE TABLE phone_numbers
 (
 phone_number_sid CHAR(36) UNIQUE ,
-number VARCHAR(132) NOT NULL UNIQUE ,
+number VARCHAR(132) NOT NULL,
 voip_carrier_sid CHAR(36),
 account_sid CHAR(36),
 application_sid CHAR(36),
@@ -425,11 +453,13 @@ CREATE TABLE sip_gateways
 sip_gateway_sid CHAR(36),
 ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
 netmask INTEGER NOT NULL DEFAULT 32,
-port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+port INTEGER COMMENT 'sip signaling port',
 inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+pad_crypto BOOLEAN NOT NULL DEFAULT 0,
+protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
 
@@ -466,9 +496,20 @@ app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
 speech_synthesis_voice VARCHAR(64),
+speech_synthesis_label VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
+speech_recognizer_label VARCHAR(64),
+use_for_fallback_speech BOOLEAN DEFAULT false,
+fallback_speech_synthesis_vendor VARCHAR(64),
+fallback_speech_synthesis_language VARCHAR(12),
+fallback_speech_synthesis_voice VARCHAR(64),
+fallback_speech_synthesis_label VARCHAR(64),
+fallback_speech_recognizer_vendor VARCHAR(64),
+fallback_speech_recognizer_language VARCHAR(64),
+fallback_speech_recognizer_label VARCHAR(64),
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
 ) COMMENT='A defined set of behaviors to be applied to phone calls ';
 
@@ -506,6 +547,9 @@ subspace_client_secret VARCHAR(255),
 subspace_sip_teleport_id VARCHAR(255),
 subspace_sip_teleport_destinations VARCHAR(255),
 siprec_hook_sid CHAR(36),
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
+record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
+bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
 PRIMARY KEY (account_sid)
 ) COMMENT='An enterprise that uses the platform for comm services';
 
@@ -526,6 +570,9 @@ ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERE
 
 ALTER TABLE call_routes ADD FOREIGN KEY application_sid_idxfk (application_sid) REFERENCES applications (application_sid);
 
+CREATE INDEX client_sid_idx ON clients (client_sid);
+ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid);
+
 CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
 ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
 
@@ -590,8 +637,6 @@ CREATE INDEX smpp_address_sid_idx ON smpp_addresses (smpp_address_sid);
 CREATE INDEX service_provider_sid_idx ON smpp_addresses (service_provider_sid);
 ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-CREATE UNIQUE INDEX speech_credentials_idx_1 ON speech_credentials (vendor,account_sid);
-
 CREATE INDEX speech_credential_sid_idx ON speech_credentials (speech_credential_sid);
 CREATE INDEX service_provider_sid_idx ON speech_credentials (service_provider_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
@@ -599,6 +644,10 @@ ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (ser
 CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid);
+CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid);
+ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE;
+
 CREATE INDEX user_sid_idx ON users (user_sid);
 CREATE INDEX email_idx ON users (email);
 CREATE INDEX phone_idx ON users (phone);
@@ -628,6 +677,8 @@ CREATE INDEX smpp_gateway_sid_idx ON smpp_gateways (smpp_gateway_sid);
 CREATE INDEX voip_carrier_sid_idx ON smpp_gateways (voip_carrier_sid);
 ALTER TABLE smpp_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
+CREATE UNIQUE INDEX phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid);
+
 CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
 CREATE INDEX number_idx ON phone_numbers (number);
 CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);

db/seed-integration-test.sql

@@ -22,7 +22,7 @@ values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9
 
 -- create one service provider and one account
 insert into service_providers (service_provider_sid, name, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.us');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.cloud');
 
 insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
 values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'wh_secret_cJqgtMDPzDhhnjmaJH6Mtk');
@@ -38,9 +38,9 @@ values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fs', '38700987-c7a4-4685-a5bb-af378f9
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES

db/seed-production-database-open-source.sql

@@ -24,10 +24,9 @@ values ('09e92f3c-9d73-4303-b63f-3668574862ce', '1cf2f4f4-64c4-4249-9a3e-5bb4cb5
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
-
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES
 ('7087fe50-8acb-4f3b-b820-97b573723aab', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'hello world', 'd31568d0-b193-4a05-8ff6-778369bc6efe', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US'),
@@ -68,9 +67,6 @@ VALUES
 ('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
 ('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
 ('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
-('48b108e3-1ce7-4f18-a4cb-e41e63688bdf', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.193', 30, 5060, 1, 0),
-('d9131a69-fe44-4c2a-ba82-4adc81f628dd', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.194', 30, 5060, 1, 0),
-('34a6a311-4bd6-49ca-aa77-edd3cb92c6e1', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.195', 30, 5060, 1, 0),
 ('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
 ('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),

db/upgrade-jambonz-db.js

@@ -89,6 +89,7 @@ const sql = {
     'ALTER TABLE `users` ADD COLUMN `is_active` BOOLEAN NOT NULL default true',
   ],
   8003: [
+    'SET FOREIGN_KEY_CHECKS=0',
     'ALTER TABLE `voip_carriers` ADD COLUMN `register_status` VARCHAR(4096)',
     'ALTER TABLE `sbc_addresses` ADD COLUMN `last_updated` DATETIME',
     'ALTER TABLE `sbc_addresses` ADD COLUMN `tls_port` INTEGER',
@@ -99,6 +100,97 @@ const sql = {
     sip_domain_name VARCHAR(255),
     monitoring_domain_name VARCHAR(255)
     )`,
+    'DROP TABLE IF EXISTS `lcr_routes`',
+    'DROP TABLE IF EXISTS `lcr_carrier_set_entry`',
+    `CREATE TABLE lcr_routes
+    (
+    lcr_route_sid CHAR(36),
+    lcr_sid CHAR(36) NOT NULL,
+    regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
+    description VARCHAR(1024),
+    priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
+    PRIMARY KEY (lcr_route_sid)
+    )`,
+    `CREATE TABLE lcr
+    (
+    lcr_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+    service_provider_sid CHAR(36),
+    account_sid CHAR(36),
+    PRIMARY KEY (lcr_sid)
+    )`,
+    `CREATE TABLE lcr_carrier_set_entry
+    (
+    lcr_carrier_set_entry_sid CHAR(36),
+    workload INTEGER NOT NULL DEFAULT 1 COMMENT 'represents a proportion of traffic to send through the associated carrier; can be used for load balancing traffic across carriers with a common priority for a destination',
+    lcr_route_sid CHAR(36) NOT NULL,
+    voip_carrier_sid CHAR(36) NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 0 COMMENT 'lower priority carriers are attempted first',
+    PRIMARY KEY (lcr_carrier_set_entry_sid)
+    )`,
+    'CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid)',
+    'ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid)',
+    'CREATE INDEX lcr_sid_idx ON lcr (lcr_sid)',
+    'ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid)',
+    'CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid)',
+    'CREATE INDEX account_sid_idx ON lcr (account_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY lcr_route_sid_idxfk (lcr_route_sid) REFERENCES lcr_routes (lcr_route_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_3 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid)',
+    'SET FOREIGN_KEY_CHECKS=1',
+  ],
+  8004: [
+    'alter table accounts add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table accounts add column bucket_credential VARCHAR(8192)',
+    'alter table accounts add column record_format VARCHAR(16) NOT NULL DEFAULT \'mp3\'',
+    'alter table applications add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table phone_numbers DROP INDEX number',
+    'create unique index phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid)',
+    `CREATE TABLE clients
+    (
+    client_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    username VARCHAR(64),
+    password VARCHAR(1024),
+    PRIMARY KEY (client_sid)
+    )`,
+    'CREATE INDEX client_sid_idx ON clients (client_sid)',
+    'ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid)',
+    'ALTER TABLE sip_gateways ADD COLUMN protocol ENUM(\'udp\',\'tcp\',\'tls\', \'tls/srtp\') DEFAULT \'udp\''
+  ],
+  8005: [
+    'DROP INDEX speech_credentials_idx_1 ON speech_credentials',
+    'ALTER TABLE speech_credentials ADD COLUMN label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN use_for_fallback_speech BOOLEAN DEFAULT false',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_language VARCHAR(12)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_voice VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_language VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE sip_gateways ADD COLUMN pad_crypto BOOLEAN NOT NULL DEFAULT 0',
+    'ALTER TABLE sip_gateways MODIFY port INTEGER',
+    `CREATE TABLE google_custom_voices
+    (
+    google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+    speech_credential_sid CHAR(36) NOT NULL,
+    model VARCHAR(512) NOT NULL,
+    reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+    name VARCHAR(64) NOT NULL,
+    PRIMARY KEY (google_custom_voice_sid)
+    )
+    `,
+    'CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid)',
+    'CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid)',
+    'ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE',
+    'ALTER TABLE clients ADD COLUMN allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1',
+    'ALTER TABLE clients ADD COLUMN allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1',
+    'ALTER TABLE clients ADD COLUMN allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1'
   ]
 };
 
@@ -129,6 +221,8 @@ const doIt = async() => {
         if (val < 7007) upgrades.push(...sql['7007']);
         if (val < 8000) upgrades.push(...sql['8000']);
         if (val < 8003) upgrades.push(...sql['8003']);
+        if (val < 8004) upgrades.push(...sql['8004']);
+        if (val < 8005) upgrades.push(...sql['8005']);
 
         // perform all upgrades
         logger.info({upgrades}, 'applying schema upgrades..');

db/webapp-tests.sql

@@ -2,7 +2,7 @@ SET FOREIGN_KEY_CHECKS=0;
 
 -- create one service provider
 insert into service_providers (service_provider_sid, name, description, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.us', 'jambonz.us service provider', 'sip.yakeeda.com');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.cloud', 'jambonz.cloud service provider', 'sip.yakeeda.com');
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
 
@@ -19,8 +19,8 @@ insert into sip_gateways (sip_gateway_sid, voip_carrier_sid, ipv4, port, inbound
 values ('46b727eb-c7dc-44fa-b063-96e48d408e4a', '5145b436-2f38-4029-8d4c-fd8c67831c7a', '3.3.3.3', 5060, 1, 1, 1);
 
 -- create the test application and test phone number
-insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.us/testCall', 'POST');
-insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.us/callStatus', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.cloud/testCall', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.cloud/callStatus', 'POST');
 insert into applications (application_sid, name, service_provider_sid, call_hook_sid, call_status_hook_sid, 
 speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 values ('7a489343-02ed-471e-8df0-fc5e1b98ce8f', 'Test application', '2708b1b3-2736-40ea-b502-c53d8396247f', 

lib/helpers/realtimedb-helpers.js

@@ -1,10 +1,11 @@
 const logger = require('../logger');
 
 const {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
@@ -12,16 +13,14 @@ const {
   deleteKey,
   incrKey,
   client: redisClient,
-} = require('@jambonz/realtimedb-helpers')({
-  host: process.env.JAMBONES_REDIS_HOST || 'localhost',
-  port: process.env.JAMBONES_REDIS_PORT || 6379
-}, logger);
+} = require('@jambonz/realtimedb-helpers')({}, logger);
 
 module.exports = {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
-  listQueues,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,

lib/logger.js

@@ -1,11 +1,7 @@
-const opts = Object.assign({
-  timestamp: () => {
-    return `, "time": "${new Date().toISOString()}"`;
-  }
-}, {
+const opts = {
   level: process.env.JAMBONES_LOGLEVEL || 'info'
-});
-
-const logger = require('pino')(opts);
+};
+const pino = require('pino');
+const logger = pino(opts, pino.destination(1, {sync: false}));
 
 module.exports = logger;

lib/models/account.js

@@ -4,7 +4,7 @@ const {getMysqlConnection} = require('../db');
 const {promisePool} = require('../db');
 const { v4: uuid } = require('uuid');
 
-const {encrypt} = require('../utils/encrypt-decrypt');
+const {encrypt, decrypt} = require('../utils/encrypt-decrypt');
 
 const retrieveSql = `SELECT * from accounts acc
 LEFT JOIN webhooks AS rh
@@ -34,7 +34,7 @@ AND effective_end_date IS NULL
 AND pending=0`;
 
 const updatePaymentInfoSql = `UPDATE account_subscriptions 
-SET last4 = ?, exp_month = ?, exp_year = ?, card_type = ? 
+SET last4 = ?, stripe_payment_method_id=?, exp_month = ?, exp_year = ?, card_type = ? 
 WHERE account_sid = ? 
 AND effective_end_date IS NULL`;
 
@@ -55,6 +55,13 @@ WHERE account_sid = ?
 AND effective_end_date IS NULL 
 AND pending = 0`;
 
+const extractBucketCredential = (obj) => {
+  const {bucket_credential} = obj;
+  if (bucket_credential) {
+    obj.bucket_credential = JSON.parse(decrypt(bucket_credential));
+  }
+};
+
 function transmogrifyResults(results) {
   return results.map((row) => {
     const obj = row.acc;
@@ -75,6 +82,8 @@ function transmogrifyResults(results) {
     else obj.queue_event_hook = null;
     delete obj.queue_event_hook_sid;
 
+    extractBucketCredential(obj);
+
     return obj;
   });
 }
@@ -197,10 +206,10 @@ class Account extends Model {
   }
 
   static async updatePaymentInfo(logger, account_sid, pm) {
-    const {card} = pm;
+    const {id, card} = pm;
     const last4_encrypted = encrypt(card.last4);
     await promisePool.execute(updatePaymentInfoSql,
-      [last4_encrypted, card.exp_month, card.exp_year, card.brand, account_sid]);
+      [last4_encrypted, id, card.exp_month, card.exp_year, card.brand, account_sid]);
   }
 
   static async provisionPendingSubscription(logger, account_sid, products, payment_method, subscription_id) {
@@ -238,7 +247,6 @@ class Account extends Model {
     }));
     return account_subscription_sid;
   }
-
 }
 
 Account.table = 'accounts';
@@ -319,7 +327,15 @@ Account.fields = [
     type: 'string',
   },
   {
-    name: 'lcr_sid',
+    name: 'record_all_calls',
+    type: 'number'
+  },
+  {
+    name: 'record_format',
+    type: 'string'
+  },
+  {
+    name: 'bucket_credential',
     type: 'string'
   }
 ];

lib/models/application.js

@@ -120,6 +120,10 @@ Application.fields = [
   {
     name: 'messaging_hook_sid',
     type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number',
   }
 ];
 

lib/models/client.js

@@ -0,0 +1,70 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Client extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(service_provider_sid) {
+    const sql = `SELECT c.client_sid, c.account_sid, c.is_active, c.username, c.hashed_password
+      FROM ${this.table} AS c LEFT JOIN accounts AS acc ON c.account_sid = acc.account_sid
+      LEFT JOIN service_providers AS sp ON sp.service_provider_sid = accs.service_provider_sid
+      WHERE sp.service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, service_provider_sid);
+    return rows;
+  }
+
+  static async retrieveByAccountSidAndUserName(account_sid, username) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ? AND username = ?`;
+    const [rows] = await promisePool.query(sql, [account_sid, username]);
+    return rows;
+  }
+}
+
+Client.table = 'clients';
+Client.fields = [
+  {
+    name: 'client_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+  {
+    name: 'username',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'password',
+    type: 'string'
+  },
+  {
+    name: 'allow_direct_app_calling',
+    type: 'number'
+  },
+  {
+    name: 'allow_direct_queue_calling',
+    type: 'number'
+  },
+  {
+    name: 'allow_direct_user_calling',
+    type: 'number'
+  }
+];
+
+module.exports = Client;

lib/models/google-custom-voice.js

@@ -0,0 +1,61 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class GoogleCustomVoice extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllBySpeechCredentialSid(speech_credential_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE speech_credential_sid = ?`;
+    const [rows] = await promisePool.query(sql, speech_credential_sid);
+    return rows;
+  }
+
+  static async deleteAllBySpeechCredentialSid(speech_credential_sid) {
+    const sql = `DELETE FROM ${this.table} WHERE speech_credential_sid = ?`;
+    const [rows] = await promisePool.query(sql, speech_credential_sid);
+    return rows;
+  }
+
+  static async retrieveAllByLabel(service_provider_sid, account_sid, label) {
+    let sql;
+    if (account_sid) {
+      sql = `SELECT gcv.* FROM ${this.table} gcv 
+LEFT JOIN speech_credentials sc ON gcv.speech_credential_sid = sc.speech_credential_sid 
+WHERE sc.account_sid = ? OR (sc.account_sid is NULL && sc.service_provider_sid = ?) 
+${label ? 'AND label = ?' : 'AND label is NULL'}`;
+    } else {
+      sql = `SELECT gcv.* FROM ${this.table} gcv 
+LEFT JOIN speech_credentials sc ON gcv.speech_credential_sid = sc.speech_credential_sid 
+WHERE sc.service_provider_sid = ? ${label ? 'AND label = ?' : 'AND label is NULL'}`;
+    }
+    const [rows] = await promisePool.query(sql, [...(account_sid ?
+      [account_sid, service_provider_sid] : [service_provider_sid]), label]);
+    return rows;
+  }
+}
+GoogleCustomVoice.table = 'google_custom_voices';
+GoogleCustomVoice.fields = [
+  {
+    name: 'google_custom_voice_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'model',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'reported_usage',
+    type: 'number'
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  }
+];
+
+module.exports = GoogleCustomVoice;

lib/models/phone-number.js

@@ -1,6 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
-const sql = 'SELECT * from phone_numbers WHERE account_sid = ?';
+const sqlRetrieveAll = 'SELECT * from phone_numbers WHERE account_sid = ? ORDER BY number';
+const sqlRetrieveOne = 'SELECT * from phone_numbers WHERE phone_number_sid = ? AND account_sid = ? ORDER BY number';
 const sqlSP = `SELECT * 
 FROM phone_numbers 
 WHERE account_sid IN 
@@ -8,7 +9,7 @@ WHERE account_sid IN
   SELECT account_sid 
   FROM accounts 
   WHERE service_provider_sid = ?
-)`;
+) ORDER BY number`;
 
 class PhoneNumber extends Model {
   constructor() {
@@ -16,8 +17,8 @@ class PhoneNumber extends Model {
   }
 
   static async retrieveAll(account_sid) {
-    if (!account_sid) return super.retrieveAll();
-    const [rows] = await promisePool.query(sql, account_sid);
+    if (!account_sid) return await super.retrieveAll();
+    const [rows] = await promisePool.query(sqlRetrieveAll, account_sid);
     return rows;
   }
   static async retrieveAllForSP(service_provider_sid) {
@@ -30,7 +31,7 @@ class PhoneNumber extends Model {
    */
   static async retrieve(sid, account_sid) {
     if (!account_sid) return super.retrieve(sid);
-    const [rows] = await promisePool.query(`${sql} AND phone_number_sid = ?`, [account_sid, sid]);
+    const [rows] = await promisePool.query(sqlRetrieveOne, [sid, account_sid]);
     return rows;
   }
 }

lib/models/sip-gateway.js

@@ -51,6 +51,10 @@ SipGateway.fields = [
     name: 'is_active',
     type: 'number'
   },
+  {
+    name: 'pad_crypto',
+    type: 'number'
+  },
   {
     name: 'account_sid',
     type: 'string'
@@ -58,6 +62,10 @@ SipGateway.fields = [
   {
     name: 'application_sid',
     type: 'string'
+  },
+  {
+    name: 'protocol',
+    type: 'string'
   }
 ];
 

lib/models/speech-credential.js

@@ -1,7 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
 const retrieveSql = 'SELECT * from speech_credentials WHERE account_sid = ?';
-const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ?';
+const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ? and account_sid is null';
 
 class SpeechCredential extends Model {
   constructor() {
@@ -20,6 +20,18 @@ class SpeechCredential extends Model {
     return rows;
   }
 
+  static async getSpeechCredentialsByVendorAndLabel(service_provider_sid, account_sid, vendor, label) {
+    let sql;
+    if (account_sid) {
+      sql = `SELECT * FROM speech_credentials WHERE account_sid = ? AND vendor = ? ${label ? 'AND label = ?' : ''}`;
+    } else {
+      sql = `SELECT * FROM speech_credentials WHERE service_provider_sid = ? AND vendor = ?
+        ${label ? 'AND label = ?' : ''}`;
+    }
+    const [rows] = await promisePool.query(sql, [account_sid ? account_sid : service_provider_sid, vendor, label]);
+    return rows;
+  }
+
   static async disableStt(account_sid) {
     await promisePool.execute('UPDATE speech_credentials SET use_for_stt = 0 WHERE account_sid = ?', [account_sid]);
   }
@@ -86,6 +98,10 @@ SpeechCredential.fields = [
   {
     name: 'last_tested',
     type: 'date'
+  },
+  {
+    name: 'label',
+    type: 'string'
   }
 ];
 

lib/record/azure-storage.js

@@ -0,0 +1,41 @@
+const { Writable } = require('stream');
+const { BlobServiceClient } = require('@azure/storage-blob');
+const { v4: uuidv4 } = require('uuid');
+
+class AzureStorageUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+    this.blockBlobClient = blobServiceClient.getContainerClient(opts.bucketName).getBlockBlobClient(opts.Key);
+    this.metadata = opts.metadata;
+    this.blocks = [];
+  }
+
+  async _write(chunk, encoding, callback) {
+    const blockID = uuidv4().replace(/-/g, '');
+    this.blocks.push(blockID);
+    try {
+      await this.blockBlobClient.stageBlock(blockID, chunk, chunk.length);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this.blockBlobClient.commitBlockList(this.blocks);
+      // remove all null/undefined props
+      const filteredObj = Object.entries(this.metadata).reduce((acc, [key, val]) => {
+        if (val !== undefined && val !== null) acc[key] = val;
+        return acc;
+      }, {});
+      await this.blockBlobClient.setMetadata(filteredObj);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = AzureStorageUploadStream;

lib/record/encoder.js

@@ -0,0 +1,61 @@
+const { Transform } = require('stream');
+const lamejs = require('@jambonz/lamejs');
+
+class PCMToMP3Encoder extends Transform {
+  constructor(options, logger) {
+    super(options);
+
+    const channels = options.channels || 1;
+    const sampleRate = options.sampleRate || 8000;
+    const bitRate = options.bitRate || 128;
+
+    this.encoder = new lamejs.Mp3Encoder(channels, sampleRate, bitRate);
+    this.channels = channels;
+    this.logger = logger;
+  }
+
+  _transform(chunk, encoding, callback) {
+    try {
+      // Convert chunk buffer into Int16Array for lamejs
+      const samples = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
+
+      // Split input samples into left and right channel arrays if stereo
+      let leftChannel, rightChannel;
+      if (this.channels === 2) {
+        leftChannel = new Int16Array(samples.length / 2);
+        rightChannel = new Int16Array(samples.length / 2);
+
+        for (let i = 0; i < samples.length; i += 2) {
+          leftChannel[i / 2] = samples[i];
+          rightChannel[i / 2] = samples[i + 1];
+        }
+      } else {
+        leftChannel = samples;
+      }
+
+      // Encode the input data
+      const mp3Data = this.encoder.encodeBuffer(leftChannel, rightChannel);
+
+      if (mp3Data.length > 0) {
+        this.push(Buffer.from(mp3Data));
+      }
+      callback();
+    } catch (err) {
+      this.logger.error(
+        { err },
+        'Error while mp3 transform');
+    }
+  }
+
+  _flush(callback) {
+    // Finalize encoding and flush the internal buffers
+    const mp3Data = this.encoder.flush();
+
+    if (mp3Data.length > 0) {
+      this.push(Buffer.from(mp3Data));
+    }
+    callback();
+  }
+}
+
+module.exports = PCMToMP3Encoder;

lib/record/google-storage.js

@@ -0,0 +1,41 @@
+const { Storage } = require('@google-cloud/storage');
+const { Writable } = require('stream');
+
+class GoogleStorageUploadStream extends Writable {
+
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.metadata = opts.metadata;
+
+    const storage = new Storage(opts.bucketCredential);
+    this.gcsFile = storage.bucket(opts.bucketName).file(opts.Key);
+    this.writeStream = this.gcsFile.createWriteStream();
+
+    this.writeStream.on('error', (err) => this.logger.error(err));
+    this.writeStream.on('finish', () => {
+      this.logger.info('google storage Upload completed.');
+      this._addMetadata();
+    });
+  }
+
+  _write(chunk, encoding, callback) {
+    this.writeStream.write(chunk, encoding, callback);
+  }
+
+  _final(callback) {
+    this.writeStream.end();
+    this.writeStream.once('finish', callback);
+  }
+
+  async _addMetadata() {
+    try {
+      await this.gcsFile.setMetadata({metadata: this.metadata});
+      this.logger.info('Google storage Upload and metadata setting completed.');
+    } catch (err) {
+      this.logger.error(err, 'Google storage  An error occurred while setting metadata');
+    }
+  }
+}
+
+module.exports = GoogleStorageUploadStream;

lib/record/index.js

@@ -0,0 +1,6 @@
+
+async function record(logger, socket) {
+  return require('./upload')(logger, socket);
+}
+
+module.exports = record;

lib/record/s3-multipart-upload-stream.js

@@ -0,0 +1,103 @@
+const { Writable } = require('stream');
+const {
+  S3Client,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+} = require('@aws-sdk/client-s3');
+
+class S3MultipartUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.bucketName = opts.bucketName;
+    this.objectKey = opts.objectKey;
+    this.uploadId = null;
+    this.partNumber = 1;
+    this.multipartETags = [];
+    this.buffer = Buffer.alloc(0);
+    this.minPartSize = 5 * 1024 * 1024; // 5 MB
+    this.s3 = new S3Client(opts.bucketCredential);
+    this.metadata = opts.metadata;
+  }
+
+  async _initMultipartUpload() {
+    const command = new CreateMultipartUploadCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      Metadata: this.metadata
+    });
+    const response = await this.s3.send(command);
+    return response.UploadId;
+  }
+
+  async _uploadBuffer() {
+    const uploadPartCommand = new UploadPartCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      PartNumber: this.partNumber,
+      UploadId: this.uploadId,
+      Body: this.buffer,
+    });
+
+    const uploadPartResponse = await this.s3.send(uploadPartCommand);
+    this.multipartETags.push({
+      ETag: uploadPartResponse.ETag,
+      PartNumber: this.partNumber,
+    });
+    this.partNumber += 1;
+  }
+
+  async _write(chunk, encoding, callback) {
+    try {
+      if (!this.uploadId) {
+        this.uploadId = await this._initMultipartUpload();
+      }
+
+      this.buffer = Buffer.concat([this.buffer, chunk]);
+
+      if (this.buffer.length >= this.minPartSize) {
+        await this._uploadBuffer();
+        this.buffer = Buffer.alloc(0);
+      }
+
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _finalize(err) {
+    try {
+      if (this.buffer.length > 0) {
+        await this._uploadBuffer();
+      }
+
+      const completeMultipartUploadCommand = new CompleteMultipartUploadCommand({
+        Bucket: this.bucketName,
+        Key: this.objectKey,
+        MultipartUpload: {
+          Parts: this.multipartETags.sort((a, b) => a.PartNumber - b.PartNumber),
+        },
+        UploadId: this.uploadId,
+      });
+
+      await this.s3.send(completeMultipartUploadCommand);
+      this.logger.info('Finished upload to S3');
+    } catch (error) {
+      this.logger.error('Error completing multipart upload:', error);
+      throw error;
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this._finalize();
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = S3MultipartUploadStream;

lib/record/upload.js

@@ -0,0 +1,99 @@
+const Account = require('../models/account');
+const Websocket = require('ws');
+const PCMToMP3Encoder = require('./encoder');
+const wav = require('wav');
+const { getUploader } = require('./utils');
+
+async function upload(logger, socket) {
+  socket._recvInitialMetadata = false;
+  socket.on('message', async function(data, isBinary) {
+    try {
+      if (!isBinary && !socket._recvInitialMetadata) {
+        socket._recvInitialMetadata = true;
+        logger.debug(`initial metadata: ${data}`);
+        const obj = JSON.parse(data.toString());
+        logger.info({ obj }, 'received JSON message from jambonz');
+        const { sampleRate, accountSid, callSid, direction, from, to,
+          callId, applicationSid, originatingSipIp, originatingSipTrunkName } = obj;
+        const account = await Account.retrieve(accountSid);
+        if (account && account.length && account[0].bucket_credential) {
+          const obj = account[0].bucket_credential;
+          // add tags to metadata
+          const metadata = {
+            accountSid,
+            callSid,
+            direction,
+            from,
+            to,
+            callId,
+            applicationSid,
+            originatingSipIp,
+            originatingSipTrunkName,
+            sampleRate: `${sampleRate}`
+          };
+          if (obj.tags && obj.tags.length) {
+            obj.tags.forEach((tag) => {
+              metadata[tag.Key] = tag.Value;
+            });
+          }
+          // create S3 path
+          const day = new Date();
+          let key = `${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
+          key += `/${day.getDate().toString().padStart(2, '0')}/${callSid}.${account[0].record_format}`;
+
+          // Uploader
+          const uploadStream = getUploader(key, metadata, obj, logger);
+          if (!uploadStream) {
+            logger.info('There is no available record uploader, close the socket.');
+            socket.close();
+          }
+
+          /**encoder */
+          let encoder;
+          if (account[0].record_format === 'wav') {
+            encoder = new wav.Writer({ channels: 2, sampleRate, bitDepth: 16 });
+          } else {
+            // default is mp3
+            encoder = new PCMToMP3Encoder({
+              channels: 2,
+              sampleRate: sampleRate,
+              bitrate: 128
+            }, logger);
+          }
+          const handleError = (err, streamType) => {
+            logger.error(
+              { err },
+              `Error while streaming for vendor: ${obj.vendor}, pipe: ${streamType}: ${err.message}`
+            );
+          };
+
+          /* start streaming data */
+          const duplex = Websocket.createWebSocketStream(socket);
+          duplex
+            .on('error', (err) => handleError(err, 'duplex'))
+            .pipe(encoder)
+            .on('error', (err) => handleError(err, 'encoder'))
+            .pipe(uploadStream)
+            .on('error', (err) => handleError(err, 'uploadStream'));
+
+        } else {
+          logger.info(`account ${accountSid} does not have any bucket credential, close the socket`);
+          socket.close();
+        }
+      }
+    } catch (err) {
+      logger.error({ err, data }, 'error parsing message during connection');
+    }
+  });
+  socket.on('error', function(err) {
+    logger.error({ err }, 'record upload: error');
+  });
+  socket.on('close', (data) => {
+    logger.info({ data }, 'record upload: close');
+  });
+  socket.on('end', function(err) {
+    logger.error({ err }, 'record upload: socket closed from jambonz');
+  });
+}
+
+module.exports = upload;

lib/record/utils.js

@@ -0,0 +1,58 @@
+const AzureStorageUploadStream = require('./azure-storage');
+const GoogleStorageUploadStream = require('./google-storage');
+const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
+
+const getUploader = (key, metadata, bucket_credential, logger) => {
+  const uploaderOpts = {
+    bucketName: bucket_credential.name,
+    objectKey: key,
+    metadata
+  };
+  try {
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+        uploaderOpts.bucketCredential = {
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: bucket_credential.region || 'us-east-1'
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 's3_compatible':
+        uploaderOpts.bucketCredential = {
+          endpoint: bucket_credential.endpoint,
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: 'us-east-1',
+          forcePathStyle: true
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 'google':
+        const serviceKey = JSON.parse(bucket_credential.service_key);
+        uploaderOpts.bucketCredential = {
+          projectId: serviceKey.project_id,
+          credentials: {
+            client_email: serviceKey.client_email,
+            private_key: serviceKey.private_key
+          }
+        };
+        return new GoogleStorageUploadStream(logger, uploaderOpts);
+      case 'azure':
+        uploaderOpts.connection_string = bucket_credential.connection_string;
+        return new AzureStorageUploadStream(logger, uploaderOpts);
+      default:
+        logger.error(`unknown bucket vendor: ${bucket_credential.vendor}`);
+        break;
+    }
+  } catch (err) {
+    logger.error(`Error creating uploader, vendor: ${bucket_credential.vendor}, reason: ${err.message}`);
+  }
+  return null;
+};
+
+module.exports = {
+  getUploader
+};

lib/routes/api/accounts.js

@@ -1,4 +1,5 @@
 const router = require('express').Router();
+const assert = require('assert');
 const request = require('request');
 const {DbErrorBadRequest, DbErrorForbidden, DbErrorUnprocessableRequest} = require('../../utils/errors');
 const Account = require('../../models/account');
@@ -22,6 +23,8 @@ const {
 } = require('./utils');
 const short = require('short-uuid');
 const VoipCarrier = require('../../models/voip-carrier');
+const { encrypt } = require('../../utils/encrypt-decrypt');
+const { testS3Storage, testGoogleStorage, testAzureStorage } = require('../../utils/storage-utils');
 const translator = short();
 
 let idx = 0;
@@ -38,20 +41,14 @@ const getFsUrl = async(logger, retrieveSet, setName) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/createCall`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/createCall`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
-};
-
 const validateRequest = async(req, account_sid) => {
   try {
     if (req.user.hasScope('admin')) {
@@ -89,6 +86,7 @@ router.use('/:sid/Charges', hasAccountPermissions, require('./charges'));
 router.use('/:sid/SipRealms', hasAccountPermissions, require('./sip-realm'));
 router.use('/:sid/PredefinedCarriers', hasAccountPermissions, require('./add-from-predefined-carrier'));
 router.use('/:sid/Limits', hasAccountPermissions, require('./limits'));
+router.use('/:sid/TtsCache', hasAccountPermissions, require('./tts-cache'));
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
@@ -148,6 +146,49 @@ router.post('/:sid/VoipCarriers', async(req, res) => {
   }
 });
 
+router.get('/:sid/RegisteredSipUsers', async(req, res) => {
+  const {logger, registrar} = req.app.locals;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const result = await Account.retrieve(account_sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${account_sid}`);
+    }
+    const users = await registrar.getRegisteredUsersForRealm(result[0].sip_realm);
+    res.status(200).json(users.map((u) => `${u}@${result[0].sip_realm}`));
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid/RegisteredSipUsers/:client', async(req, res) => {
+  const {logger, registrar, lookupClientByAccountAndUsername} = req.app.locals;
+  const client = req.params.client;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const result = await Account.retrieve(account_sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${account_sid}`);
+    }
+    const user = await registrar.query(`${client}@${result[0].sip_realm}`);
+    const [clientDb] = await lookupClientByAccountAndUsername(account_sid, client);
+    res.status(200).json({
+      name: client,
+      contact: user ? user.contact : null,
+      expiryTime: user ? user.expiryTime : 0,
+      protocol: user ? user.protocol : null,
+      allow_direct_app_calling: clientDb ? clientDb.allow_direct_app_calling : 0,
+      allow_direct_queue_calling: clientDb ? clientDb.allow_direct_queue_calling : 0,
+      allow_direct_user_calling: clientDb ? clientDb.allow_direct_user_calling : 0,
+      registered_status: user ? 'active' : 'inactive'
+    });
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 function coerceNumbers(callInfo) {
   if (Array.isArray(callInfo)) {
     return callInfo.map((ci) => {
@@ -178,11 +219,13 @@ function validateUpdateCall(opts) {
     'child_call_hook',
     'call_status',
     'listen_status',
+    'transcribe_status',
     'conf_hold_status',
     'conf_mute_status',
     'mute_status',
     'sip_request',
-    'record'
+    'record',
+    'tag'
   ]
     .reduce((acc, prop) => (opts[prop] ? ++acc : acc), 0);
 
@@ -218,8 +261,8 @@ function validateUpdateCall(opts) {
     throw new DbErrorBadRequest('invalid conf_mute_status');
   }
   if (opts.sip_request &&
-    (!opts.sip_request.method && !opts.sip_request.content_type ||  !opts.sip_request.content_type)) {
-    throw new DbErrorBadRequest('sip_request requires content_type and content properties');
+    (!opts.sip_request.method || !opts.sip_request.content_type ||  !opts.sip_request.content)) {
+    throw new DbErrorBadRequest('sip_request requires method, content_type and content properties');
   }
   if (opts.record && !opts.record.action) {
     throw new DbErrorBadRequest('record requires action property');
@@ -227,6 +270,9 @@ function validateUpdateCall(opts) {
   if ('startCallRecording' === opts.record?.action && !opts.record.siprecServerURL) {
     throw new DbErrorBadRequest('record requires siprecServerURL property when starting recording');
   }
+  if (opts.tag && (typeof opts.tag !== 'object' || Array.isArray(opts.tag) || opts.tag === null)) {
+    throw new DbErrorBadRequest('invalid tag data');
+  }
 }
 
 function validateTo(to) {
@@ -263,6 +309,7 @@ async function validateCreateCall(logger, sid, req) {
       const application = await lookupAppBySid(obj.application_sid);
       Object.assign(obj, {
         call_hook: application.call_hook,
+        app_json: application.app_json,
         call_status_hook: application.call_status_hook,
         speech_synthesis_vendor: application.speech_synthesis_vendor,
         speech_synthesis_language: application.speech_synthesis_language,
@@ -449,7 +496,6 @@ router.get('/:sid', async(req, res) => {
   try {
     const account_sid = parseAccountSid(req);
     await validateRequest(req, account_sid);
-
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
     const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
@@ -535,6 +581,58 @@ router.delete('/:sid/SubspaceTeleport', async(req, res) => {
   }
 });
 
+function encryptBucketCredential(obj) {
+  if (!obj.bucket_credential) return;
+  const {
+    vendor,
+    region,
+    name,
+    access_key_id,
+    secret_access_key,
+    tags,
+    service_key,
+    connection_string,
+    endpoint
+  } = obj.bucket_credential;
+
+  switch (vendor) {
+    case 'aws_s3':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(region, 'invalid aws bucket region: region is required');
+      const awsData = JSON.stringify({vendor, region, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(awsData);
+      break;
+    case 's3_compatible':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(endpoint, 'invalid endpoint uri: endpoint is required');
+      const s3Data = JSON.stringify({vendor, endpoint, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(s3Data);
+      break;
+    case 'google':
+      assert(service_key, 'invalid google cloud storage credential: service_key is required');
+      const googleData = JSON.stringify({vendor, name, service_key, tags});
+      obj.bucket_credential = encrypt(googleData);
+      break;
+    case 'azure':
+      assert(name, 'invalid azure container name: name is required');
+      assert(connection_string, 'invalid azure cloud storage credential: connection_string is required');
+      const azureData = JSON.stringify({vendor, name, connection_string, tags});
+      obj.bucket_credential = encrypt(azureData);
+      break;
+    case 'none':
+      obj.bucket_credential = null;
+      break;
+    default:
+      throw new DbErrorBadRequest(`unknown storage vendor: ${vendor}`);
+  }
+}
+
 /**
  * update
  */
@@ -581,6 +679,8 @@ router.put('/:sid', async(req, res) => {
       delete obj.registration_hook;
       delete obj.queue_event_hook;
 
+      encryptBucketCredential(obj);
+
       const rowsAffected = await Account.update(sid, obj);
       if (rowsAffected === 0) {
         return res.status(404).end();
@@ -673,6 +773,44 @@ account_subscriptions WHERE account_sid = ?)
   }
 });
 
+/* Test Bucket credential Keys */
+router.post('/:sid/BucketCredentialTest', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const {vendor, name, region, access_key_id, secret_access_key, service_key, connection_string, endpoint} = req.body;
+    const ret = {
+      status: 'not tested'
+    };
+
+    switch (vendor) {
+      case 'aws_s3':
+        await testS3Storage(logger, {vendor, name, region, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 's3_compatible':
+        await testS3Storage(logger, {vendor, name, endpoint, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 'google':
+        await testGoogleStorage(logger, {vendor, name, service_key});
+        ret.status = 'ok';
+        break;
+      case 'azure':
+        await testAzureStorage(logger, {vendor, name, connection_string});
+        ret.status = 'ok';
+        break;
+      default:
+        throw new DbErrorBadRequest(`Does not support test for ${vendor}`);
+    }
+    return res.status(200).json(ret);
+  }
+  catch (err) {
+    return res.status(200).json({status: 'failed', reason: err.message});
+  }
+});
+
 /**
  * retrieve account level api keys
  */
@@ -696,7 +834,7 @@ router.get('/:sid/ApiKeys', async(req, res) => {
  */
 router.post('/:sid/Calls', async(req, res) => {
   const {retrieveSet, logger} = req.app.locals;
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
+  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
   const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
 
   if (!serviceUrl) {
@@ -857,7 +995,7 @@ router.post('/:sid/Messages', async(req, res) => {
     const account_sid = parseAccountSid(req);
     await validateRequest(req, account_sid);
 
-    const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
+    const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
     const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
     if (!serviceUrl) res.json({msg: 'no available feature servers at this time'}).status(480);
     await validateCreateMessage(logger, account_sid, req);
@@ -894,12 +1032,12 @@ router.post('/:sid/Messages', async(req, res) => {
  * retrieve info for a group of queues under an account
  */
 router.get('/:sid/Queues', async(req, res) => {
-  const {logger, listQueues} = req.app.locals;
+  const {logger, listSortedSets} = req.app.locals;
   const { search } = req.query || {};
   try {
     const accountSid = parseAccountSid(req);
     await validateRequest(req, accountSid);
-    const queues = search ? await listQueues(accountSid, search) : await listQueues(accountSid);
+    const queues = search ? await listSortedSets(accountSid, search) : await listSortedSets(accountSid);
     logger.debug(`retrieved ${queues.length} queues for account sid ${accountSid}`);
     res.status(200).json(queues);
     updateLastUsed(logger, accountSid, req).catch((err) => {});
@@ -908,4 +1046,5 @@ router.get('/:sid/Queues', async(req, res) => {
   }
 });
 
+
 module.exports = router;

lib/routes/api/clients.js

@@ -0,0 +1,88 @@
+const router = require('express').Router();
+const decorate = require('./decorate');
+const sysError = require('../error');
+const Client = require('../../models/client');
+const Account = require('../../models/account');
+const { DbErrorBadRequest, DbErrorForbidden } = require('../../utils/errors');
+const { encrypt, decrypt, obscureKey } = require('../../utils/encrypt-decrypt');
+
+const commonCheck = async(req) => {
+  if (req.user.hasAccountAuth) {
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth && req.body.account_sid) {
+    const accounts = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
+    if (accounts.length === 0) {
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  }
+
+  if (req.body.password) {
+    req.body.password = encrypt(req.body.password);
+  }
+};
+
+const validateAdd = async(req) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length && clients[0].client_sid !== sid) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, Client, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Client.retrieveAll() : req.user.hasAccountAuth ?
+        await Client.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Client.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+    const ret = results.map((c) => {
+      c.password = obscureKey(decrypt(c.password), 1);
+      return c;
+    });
+    res.status(200).json(ret);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Client.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const client = results[0];
+    client.password = obscureKey(decrypt(client.password), 1);
+    if (req.user.hasAccountAuth && client.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth) {
+      const accounts = await Account.retrieve(client.account_sid, req.user.service_provider_sid);
+      if (!accounts.length) {
+        return res.sendStatus(404);
+      }
+    }
+    return res.status(200).json(client);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/forgot-password.js

@@ -6,6 +6,7 @@ const {validateEmail, emailSimpleText} = require('../../utils/email-utils');
 const {promisePool} = require('../../db');
 const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
+const assert = require('assert');
 const sql = `SELECT * from users user 
 LEFT JOIN accounts AS acc 
 ON acc.account_sid = user.account_sid 
@@ -26,7 +27,8 @@ function createOauthEmailText(provider) {
 }
 
 function createResetEmailText(link) {
-  const baseUrl = 'http://localhost:3001';
+  assert(process.env.JAMBONZ_BASE_URL, 'process.env.JAMBONZ_BASE_URL is missing');
+  const baseUrl = process.env.JAMBONZ_BASE_URL;
 
   return `Hi there!
 

lib/routes/api/google-custom-voices.js

@@ -0,0 +1,77 @@
+const router = require('express').Router();
+const GoogleCustomVoice = require('../../models/google-custom-voice');
+const SpeechCredential = require('../../models/speech-credential');
+const decorate = require('./decorate');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateCredentialPermission = async(req) => {
+  const credential = await SpeechCredential.retrieve(req.body.speech_credential_sid);
+  if (!credential || credential.length === 0) {
+    throw new DbErrorBadRequest('Invalid speech_credential_sid');
+  }
+  const cred = credential[0];
+
+  if (req.user.hasServiceProviderAuth && cred.service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+  if (req.user.hasAccountAuth && cred.account_sid !== req.user.account_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+};
+
+const validateAdd = async(req) => {
+  if (!req.body.speech_credential_sid) {
+    throw new DbErrorBadRequest('missing speech_credential_sid');
+  }
+
+  await validateCredentialPermission(req);
+};
+
+const validateUpdate = async(req) => {
+  if (req.body.speech_credential_sid) {
+    await validateCredentialPermission(req);
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, GoogleCustomVoice, ['add', 'retrieve', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const account_sid = req.user.account_sid || req.query.account_sid;
+  const service_provider_sid = req.user.service_provider_sid || req.query.service_provider_sid;
+  const speech_credential_sid = req.query.speech_credential_sid;
+  const label = req.query.label;
+  try {
+    let results = [];
+    if (speech_credential_sid) {
+      const [cred] = await SpeechCredential.retrieve(speech_credential_sid);
+      if (!cred) {
+        return res.sendStatus(404);
+      }
+      if (account_sid && cred.account_sid && cred.account_sid !== account_sid) {
+        throw new DbErrorForbidden('Insufficient privileges');
+      }
+      if (service_provider_sid && cred.service_provider_sid && cred.service_provider_sid !== service_provider_sid) {
+        throw new DbErrorForbidden('Insufficient privileges');
+      }
+      results = await GoogleCustomVoice.retrieveAllBySpeechCredentialSid(speech_credential_sid);
+    } else {
+      if (!account_sid && !service_provider_sid) {
+        throw new DbErrorBadRequest('missing account_sid or service_provider_sid in query parameters');
+      }
+      results = await GoogleCustomVoice.retrieveAllByLabel(service_provider_sid, account_sid, label);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/index.js

@@ -17,6 +17,7 @@ const isAdminScope = (req, res, next) => {
 
 api.use('/BetaInviteCodes', isAdminScope, require('./beta-invite-codes'));
 api.use('/SystemInformation', isAdminScope, require('./system-information'));
+api.use('/TtsCache', isAdminScope, require('./tts-cache'));
 api.use('/ServiceProviders', require('./service-providers'));
 api.use('/VoipCarriers', require('./voip-carriers'));
 api.use('/Webhooks', require('./webhooks'));
@@ -50,6 +51,9 @@ api.use('/PasswordSettings', require('./password-settings'));
 api.use('/Lcrs', require('./lcrs'));
 api.use('/LcrRoutes', require('./lcr-routes'));
 api.use('/LcrCarrierSetEntries', require('./lcr-carrier-set-entries'));
+api.use('/Clients', require('./clients'));
+// Google Custom Voices
+api.use('/GoogleCustomVoices', require('./google-custom-voices'));
 
 // messaging
 api.use('/Smpps', require('./smpps'));   // our smpp server info

lib/routes/api/lcrs.js

@@ -100,6 +100,93 @@ const preconditions = {
 
 decorate(router, Lcr, ['add', 'update', 'delete'], preconditions);
 
+const validateLcrBatchAdd = async(lcr_sid, body, lookupCarrierBySid) => {
+  for (const lcr_route of body) {
+    lcr_route.lcr_sid = lcr_sid;
+    if (!lcr_route.lcr_carrier_set_entries || lcr_route.lcr_carrier_set_entries.length === 0) {
+      throw new DbErrorBadRequest('Lcr Route batch process require lcr_carrier_set_entries');
+    }
+    for (const entry of lcr_route.lcr_carrier_set_entries) {
+      // check voip_carrier_sid is exist
+      if (!entry.voip_carrier_sid) {
+        throw new DbErrorBadRequest('One of lcr_carrier_set_entries is missing voip_carrier_sid');
+      }
+      const carrier = await lookupCarrierBySid(entry.voip_carrier_sid);
+      if (!carrier) {
+        throw new DbErrorBadRequest('unknown voip_carrier_sid');
+      }
+    }
+  }
+};
+
+
+const addNewLcrRoute = async(lcr_route) => {
+  const lcr_sid = lcr_route.lcr_sid;
+  const lcr_carrier_set_entries = lcr_route.lcr_carrier_set_entries;
+  delete lcr_route.lcr_carrier_set_entries;
+  const lcr_route_sid = await LcrRoutes.make(lcr_route);
+  for (const entry of lcr_carrier_set_entries) {
+    entry.lcr_route_sid = lcr_route_sid;
+    const lcr_carrier_set_entry_sid = await LcrCarrierSetEntry.make(entry);
+    if (lcr_route.priority === 9999) {
+      // this is default lcr set entry
+      const [lcr] = await Lcr.retrieve(lcr_sid);
+      if (lcr) {
+        lcr.default_carrier_set_entry_sid = lcr_carrier_set_entry_sid;
+        delete lcr.lcr_sid;
+        await Lcr.update(lcr_sid, lcr);
+      }
+    }
+  }
+};
+
+router.post('/:sid/Routes', async(req, res) => {
+  const results = await Lcr.retrieve(req.params.sid);
+  if (results.length === 0) return res.sendStatus(404);
+  const {logger, lookupCarrierBySid} = req.app.locals;
+  try {
+    const body = req.body;
+    await validateLcrBatchAdd(req.params.sid, body, lookupCarrierBySid);
+    for (const lcr_route of body) {
+      await addNewLcrRoute(lcr_route, lookupCarrierBySid);
+    }
+    res.sendStatus(204);
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.put('/:sid/Routes', async(req, res) => {
+  const results = await Lcr.retrieve(req.params.sid);
+  if (results.length === 0) return res.sendStatus(404);
+  const {logger, lookupCarrierBySid} = req.app.locals;
+  try {
+    const body = req.body;
+    await validateLcrBatchAdd(req.params.sid, body, lookupCarrierBySid);
+    for (const lcr_route of body) {
+      if (lcr_route.lcr_route_sid) {
+        const lcr_route_sid = lcr_route.lcr_route_sid;
+        delete lcr_route.lcr_route_sid;
+        const lcr_carrier_set_entries = lcr_route.lcr_carrier_set_entries;
+        delete lcr_route.lcr_carrier_set_entries;
+        await LcrRoutes.update(lcr_route_sid, lcr_route);
+        for (const entry of lcr_carrier_set_entries) {
+          const lcr_carrier_set_entry_sid = entry.lcr_carrier_set_entry_sid;
+          delete entry.lcr_carrier_set_entry_sid;
+          await LcrCarrierSetEntry.update(lcr_carrier_set_entry_sid, entry);
+        }
+      } else {
+        // Route is not available yet, let create it now
+        await addNewLcrRoute(lcr_route, lookupCarrierBySid);
+      }
+    }
+    res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {

lib/routes/api/login.js

@@ -13,8 +13,6 @@ WHERE up.permission_sid = p.permission_sid
 AND up.user_sid = ? 
 `;
 const retrieveSql = 'SELECT * from users where name = ?';
-const tokenSql = 'SELECT token from api_keys where account_sid IS NULL AND service_provider_sid IS NULL';
-
 
 router.post('/', async(req, res) => {
   const {logger, incrKey, retrieveKey} = req.app.locals;
@@ -54,11 +52,6 @@ router.post('/', async(req, res) => {
       return res.sendStatus(403);
     }
     const force_change = !!r[0].force_change;
-    const [t] = await promisePool.query(tokenSql);
-    if (t.length === 0) {
-      logger.error('Database has no admin token provisioned...run reset_admin_password');
-      return res.sendStatus(500);
-    }
 
     const [p] = await promisePool.query(retrievePemissionsSql, r[0].user_sid);
     const permissions = p.map((x) => x.name);

lib/routes/api/phone-numbers.js

@@ -94,9 +94,9 @@ decorate(router, PhoneNumber, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = req.user.hasAdminAuth ?
-      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null) :
-      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid);
+    const results = req.user.hasServiceProviderAuth ?
+      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid) :
+      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -120,6 +120,9 @@ router.get('/:sid', async(req, res) => {
         throw new DbErrorBadRequest('insufficient privileges');
       }
     }
+    if (req.user.hasAccountAuth && results.length > 1) {
+      return res.status(200).json(results.filter((r) => r.phone_number_sid === sid)[0]);
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/recent-calls.js

@@ -3,6 +3,15 @@ const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
 const {getJaegerTrace} = require('../../utils/jaeger-utils');
+const Account = require('../../models/account');
+const {
+  getS3Object,
+  getGoogleStorageObject,
+  getAzureStorageObject,
+  deleteS3Object,
+  deleteGoogleStorageObject,
+  deleteAzureStorageObject
+} = require('../../utils/storage-utils');
 
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
@@ -20,7 +29,7 @@ router.get('/', async(req, res) => {
     logger.debug({opts: req.query}, 'GET /RecentCalls');
     const account_sid = parseAccountSid(req.originalUrl);
     const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
-    const {page, count, trunk, direction, days, answered, start, end} = req.query || {};
+    const {page, count, trunk, direction, days, answered, start, end, filter} = req.query || {};
     if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
     if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
 
@@ -35,6 +44,7 @@ router.get('/', async(req, res) => {
         answered,
         start: days ? undefined : start,
         end: days ? undefined : end,
+        filter
       });
       res.status(200).json(data);
     }
@@ -49,6 +59,7 @@ router.get('/', async(req, res) => {
         answered,
         start: days ? undefined : start,
         end: days ? undefined : end,
+        filter
       });
       res.status(200).json(data);
     }
@@ -74,12 +85,12 @@ router.get('/:call_id', async(req, res) => {
   }
 });
 
-router.get('/:call_id/pcap', async(req, res) => {
+router.get('/:call_id/:method/pcap', async(req, res) => {
   const {logger} = req.app.locals;
   try {
     const token = await getHomerApiKey(logger);
     if (!token) return res.sendStatus(400, {msg: 'getHomerApiKey: Failed to get Homer API token; check server config'});
-    const stream = await getHomerPcap(logger, token, [req.params.call_id]);
+    const stream = await getHomerPcap(logger, token, [req.params.call_id], req.params.method);
     if (!stream) {
       logger.info(`getHomerApiKey: unable to get sip traces from Homer for ${req.params.call_id}`);
       return res.sendStatus(404);
@@ -111,4 +122,84 @@ router.get('/trace/:trace_id', async(req, res) => {
   }
 });
 
+router.get('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+    const getOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+    let stream;
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        stream = await getS3Object(logger, getOptions);
+        break;
+      case 'google':
+        stream = await getGoogleStorageObject(logger, getOptions);
+        break;
+      case 'azure':
+        stream = await getAzureStorageObject(logger, getOptions);
+        break;
+      default:
+        logger.error(`There is no handler for fetching record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.set({
+      'Content-Type': `audio/${format || 'mp3'}`
+    });
+    if (stream) {
+      stream.pipe(res);
+    } else {
+      return res.sendStatus(404);
+    }
+  }  catch (err) {
+    logger.error({err}, ` error retrieving recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
+router.delete('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+
+    const deleteOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        await deleteS3Object(logger, deleteOptions);
+        break;
+      case 'google':
+        await deleteGoogleStorageObject(logger, deleteOptions);
+        break;
+      case 'azure':
+        await deleteAzureStorageObject(logger, deleteOptions);
+        break;
+      default:
+        logger.error(`There is no handler for deleting record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.sendStatus(204);
+  }  catch (err) {
+    logger.error({err}, ` error deleting recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
 module.exports = router;

lib/routes/api/register.js

@@ -16,8 +16,9 @@ const insertUserSql = `INSERT into users
 (user_sid, account_sid, name, email, provider, provider_userid, email_validated) 
 values (?, ?, ?, ?, ?, ?, 1)`;
 const insertUserLocalSql = `INSERT into users 
-(user_sid, account_sid, name, email, email_activation_code, email_validated, provider, hashed_password) 
-values (?, ?, ?, ?, ?, 0, 'local', ?)`;
+(user_sid, account_sid, name, email, email_activation_code, email_validated, provider,
+hashed_password, service_provider_sid) 
+values (?, ?, ?, ?, ?, 0, 'local', ?, ?)`;
 const insertAccountSql = `INSERT into accounts 
 (account_sid, service_provider_sid, name, is_active, webhook_secret, trial_end_date) 
 values (?, ?, ?, ?, ?, CURDATE() + INTERVAL 21 DAY)`;
@@ -36,7 +37,7 @@ const insertSignupHistorySql = `INSERT into signup_history
 values (?, ?)`;
 
 const addLocalUser = async(logger, user_sid, account_sid,
-  name, email, email_activation_code, passwordHash) => {
+  name, email, email_activation_code, passwordHash, service_provider_sid) => {
   const [r] = await promisePool.execute(insertUserLocalSql,
     [
       user_sid,
@@ -44,7 +45,8 @@ const addLocalUser = async(logger, user_sid, account_sid,
       name,
       email,
       email_activation_code,
-      passwordHash
+      passwordHash,
+      service_provider_sid
     ]);
   debug({r}, 'Result from adding user');
 };
@@ -145,7 +147,7 @@ router.post('/', async(req, res) => {
       const user = await doGithubAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from github');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         email_validated: user.email_validated,
         avatar_url: user.avatar_url,
@@ -157,7 +159,7 @@ router.post('/', async(req, res) => {
       const user = await doGoogleAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from google');
       Object.assign(userProfile, {
-        name: user.name || user.email,
+        name: user.email || user.email,
         email: user.email,
         email_validated: user.verified_email,
         picture: user.picture,
@@ -170,7 +172,7 @@ router.post('/', async(req, res) => {
       logger.info({user}, 'retrieved user details for local provider');
       debug({user}, 'retrieved user details for local provider');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         provider: 'local',
         email_activation_code: user.email_activation_code
@@ -280,7 +282,8 @@ router.post('/', async(req, res) => {
         const passwordHash = await generateHashedPassword(req.body.password);
         debug(`hashed password: ${passwordHash}`);
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
-          userProfile.name, userProfile.email, userProfile.email_activation_code, passwordHash);
+          userProfile.name, userProfile.email, userProfile.email_activation_code,
+          passwordHash, req.body.service_provider_sid);
         debug('added local user');
       }
       else {
@@ -293,17 +296,25 @@ router.post('/', async(req, res) => {
       const callStatusSid = uuid();
       const helloWordSid = uuid();
       const dialTimeSid = uuid();
+      const echoSid = uuid();
 
-      /* 3 webhooks */
-      await promisePool.execute(insertWebookSql, [callStatusSid, 'https://public-apps.jambonz.us/call-status', 'POST']);
-      await promisePool.execute(insertWebookSql, [helloWordSid, 'https://public-apps.jambonz.us/hello-world', 'POST']);
-      await promisePool.execute(insertWebookSql, [dialTimeSid, 'https://public-apps.jambonz.us/dial-time', 'POST']);
+      /* 4 webhooks */
+      await promisePool.execute(insertWebookSql,
+        [callStatusSid, 'https://public-apps.jambonz.cloud/call-status', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [helloWordSid, 'https://public-apps.jambonz.cloud/hello-world', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [dialTimeSid, 'https://public-apps.jambonz.cloud/dial-time', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [echoSid, 'https://public-apps.jambonz.cloud/echo', 'POST']);
 
       /* 2 applications */
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'hello world',
         helloWordSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'dial time clock',
         dialTimeSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
+      await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'simple echo test',
+        echoSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
 
       Object.assign(userProfile, {
         pristine: true,
@@ -327,7 +338,7 @@ router.post('/', async(req, res) => {
 
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
           userProfile.name, userProfile.email, userProfile.email_activation_code,
-          passwordHash);
+          passwordHash, req.body.service_provider_sid);
 
         /* note: we deactivate the old user once the new email is validated */
       }
@@ -349,6 +360,8 @@ router.post('/', async(req, res) => {
     const token = jwt.sign({
       user_sid: userProfile.user_sid,
       account_sid: userProfile.account_sid,
+      service_provider_sid: req.body.service_provider_sid,
+      scope: 'account',
       email: userProfile.email,
       name: userProfile.name
     }, process.env.JWT_SECRET, { expiresIn });

lib/routes/api/sbcs.js

@@ -28,20 +28,13 @@ router.get('/', async(req, res) => {
 
     if (req.user.hasAccountAuth) {
       const [r] = await promisePool.query('SELECT * from accounts WHERE account_sid = ?', req.user.account_sid);
-      if (0 === r.length) throw new Error('invalid account_sid');
+      if (0 === r.length) throw new DbErrorBadRequest('invalid account_sid');
 
       service_provider_sid = r[0].service_provider_sid;
     }
 
     if (req.user.hasServiceProviderAuth) {
-      const [r] = await promisePool.query(
-        'SELECT * from service_providers where service_provider_sid = ?',
-        service_provider_sid);
-      if (0 === r.length) throw new Error('invalid account_sid');
-
-      service_provider_sid = r[0].service_provider_sid;
-
-      if (!service_provider_sid) throw new DbErrorBadRequest('missing service_provider_sid in query');
+      service_provider_sid = req.user.service_provider_sid;
     }
 
     /** generally, we have a global set of SBCs that all accounts use.

lib/routes/api/sms-inbound.js

@@ -14,20 +14,14 @@ const getFsUrl = async(logger, retrieveSet, setName, provider) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/messaging/${provider}`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/messaging/${provider}`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
-};
-
 const doSendResponse = async(res, respondFn, body) => {
   if (typeof respondFn === 'number') res.sendStatus(respondFn);
   else if (typeof respondFn !== 'function') res.sendStatus(200);
@@ -44,7 +38,7 @@ router.post('/:provider', async(req, res) => {
     lookupAppByPhoneNumber,
     logger
   } = req.app.locals;
-  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:active-fs`;
+  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:fs-service-url`;
   logger.debug({path: req.path, body: req.body}, 'incomingSMS from carrier');
 
   // search for provider module

lib/routes/api/speech-credentials.js

@@ -5,6 +5,7 @@ const SpeechCredential = require('../../models/speech-credential');
 const sysError = require('../error');
 const {decrypt, encrypt} = require('../../utils/encrypt-decrypt');
 const {parseAccountSid, parseServiceProviderSid, parseSpeechCredentialSid} = require('./utils');
+const {decryptCredential, testWhisper} = require('../../utils/speech-utils');
 const {DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const {
   testGoogleTts,
@@ -19,8 +20,11 @@ const {
   testDeepgramStt,
   testSonioxStt,
   testIbmTts,
-  testIbmStt
+  testIbmStt,
+  testElevenlabs,
+  testAssemblyStt
 } = require('../../utils/speech-utils');
+const bent = require('bent');
 const {promisePool} = require('../../db');
 
 const validateAdd = async(req) => {
@@ -99,17 +103,6 @@ const validateTest = async(req, speech_credentials) => {
   }
 };
 
-const obscureKey = (key) => {
-  const key_spoiler_length = 6;
-  const key_spoiler_char = 'X';
-
-  if (key.length <= key_spoiler_length) {
-    return key;
-  }
-
-  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
-};
-
 const encryptCredential = (obj) => {
   const {
     vendor,
@@ -125,8 +118,10 @@ const encryptCredential = (obj) => {
     nuance_stt_uri,
     use_custom_tts,
     custom_tts_endpoint,
+    custom_tts_endpoint_url,
     use_custom_stt,
     custom_stt_endpoint,
+    custom_stt_endpoint_url,
     tts_api_key,
     tts_region,
     stt_api_key,
@@ -135,7 +130,9 @@ const encryptCredential = (obj) => {
     instance_id,
     custom_stt_url,
     custom_tts_url,
-    auth_token = ''
+    auth_token = '',
+    cobalt_server_uri,
+    model_id
   } = obj;
 
   switch (vendor) {
@@ -158,15 +155,19 @@ const encryptCredential = (obj) => {
       return encrypt(awsData);
 
     case 'microsoft':
-      assert(region, 'invalid azure speech credential: region is required');
-      assert(api_key, 'invalid azure speech credential: api_key is required');
+      if (!custom_tts_endpoint_url && !custom_stt_endpoint_url) {
+        assert(region, 'invalid azure speech credential: region is required');
+        assert(api_key, 'invalid azure speech credential: api_key is required');
+      }
       const azureData = JSON.stringify({
-        region,
-        api_key,
+        ...(region && {region}),
+        ...(api_key && {api_key}),
         use_custom_tts,
         custom_tts_endpoint,
+        custom_tts_endpoint_url,
         use_custom_stt,
-        custom_stt_endpoint
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
       });
       return encrypt(azureData);
 
@@ -201,6 +202,28 @@ const encryptCredential = (obj) => {
       const sonioxData = JSON.stringify({api_key});
       return encrypt(sonioxData);
 
+    case 'cobalt':
+      assert(cobalt_server_uri, 'invalid cobalt speech credential: cobalt_server_uri is required');
+      const cobaltData = JSON.stringify({cobalt_server_uri});
+      return encrypt(cobaltData);
+
+    case 'elevenlabs':
+      assert(api_key, 'invalid elevenLabs speech credential: api_key is required');
+      assert(model_id, 'invalid elevenLabs speech credential: model_id is required');
+      const elevenlabsData = JSON.stringify({api_key, model_id});
+      return encrypt(elevenlabsData);
+
+    case 'assemblyai':
+      assert(api_key, 'invalid assemblyai speech credential: api_key is required');
+      const assemblyaiData = JSON.stringify({api_key});
+      return encrypt(assemblyaiData);
+
+    case 'whisper':
+      assert(api_key, 'invalid whisper speech credential: api_key is required');
+      assert(model_id, 'invalid whisper speech credential: model_id is required');
+      const whisperData = JSON.stringify({api_key, model_id});
+      return encrypt(whisperData);
+
     default:
       if (vendor.startsWith('custom:')) {
         const customData = JSON.stringify({auth_token, custom_stt_url, custom_tts_url});
@@ -218,6 +241,7 @@ router.post('/', async(req, res) => {
       use_for_stt,
       use_for_tts,
       vendor,
+      label
     } = req.body;
     const account_sid = req.user.account_sid || req.body.account_sid;
     const service_provider_sid = req.user.service_provider_sid ||
@@ -232,11 +256,21 @@ router.post('/', async(req, res) => {
       }
     }
 
+    // Check if vendor and label is already used for account or SP
+    if (label) {
+      const existingSpeech = await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+        service_provider_sid, account_sid, vendor, label);
+      if (existingSpeech.length > 0) {
+        throw new DbErrorUnprocessableRequest(`Label ${label} is already in use for another speech credential`);
+      }
+    }
+
     const encrypted_credential = encryptCredential(req.body);
     const uuid = await SpeechCredential.make({
       account_sid,
       service_provider_sid,
       vendor,
+      label,
       use_for_tts,
       use_for_stt,
       credential: encrypted_credential
@@ -273,66 +307,7 @@ router.get('/', async(req, res) => {
     res.status(200).json(creds.map((c) => {
       const {credential, ...obj} = c;
 
-      if ('google' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        const key_header = '-----BEGIN PRIVATE KEY-----\n';
-        const obscured = {
-          ...o,
-          private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
-        };
-        obj.service_key = obscured;
-      }
-      else if ('aws' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.access_key_id = o.access_key_id;
-        obj.secret_access_key = obscureKey(o.secret_access_key);
-        obj.aws_region = o.aws_region;
-        logger.info({obj, o}, 'retrieving aws speech credential');
-      }
-      else if ('microsoft' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-        obj.region = o.region;
-        obj.use_custom_tts = o.use_custom_tts;
-        obj.custom_tts_endpoint = o.custom_tts_endpoint;
-        obj.use_custom_stt = o.use_custom_stt;
-        obj.custom_stt_endpoint = o.custom_stt_endpoint;
-        logger.info({obj, o}, 'retrieving azure speech credential');
-      }
-      else if ('wellsaid' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if ('nuance' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.client_id = o.client_id;
-        obj.secret = o.secret ? obscureKey(o.secret) : null;
-      }
-      else if ('deepgram' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if ('ibm' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.tts_api_key = obscureKey(o.tts_api_key);
-        obj.tts_region = o.tts_region;
-        obj.stt_api_key = obscureKey(o.stt_api_key);
-        obj.stt_region = o.stt_region;
-        obj.instance_id = o.instance_id;
-      } else if ('nvidia' == obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.riva_server_uri = o.riva_server_uri;
-      }
-      else if ('soniox' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if (obj.vendor.startsWith('custom:')) {
-        const o = JSON.parse(decrypt(credential));
-        obj.auth_token = obscureKey(o.auth_token);
-        obj.custom_stt_url = o.custom_stt_url;
-        obj.custom_tts_url = o.custom_tts_url;
-      }
+      decryptCredential(obj, credential, logger);
 
       if (req.user.hasAccountAuth && obj.account_sid === null) {
         delete obj.api_key;
@@ -362,66 +337,7 @@ router.get('/:sid', async(req, res) => {
     await validateRetrieveUpdateDelete(req, cred);
 
     const {credential, ...obj} = cred[0];
-    if ('google' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      const key_header = '-----BEGIN PRIVATE KEY-----\n';
-      const obscured = {
-        ...o,
-        private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
-      };
-      obj.service_key = JSON.stringify(obscured);
-    }
-    else if ('aws' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.access_key_id = o.access_key_id;
-      obj.secret_access_key = obscureKey(o.secret_access_key);
-      obj.aws_region = o.aws_region;
-    }
-    else if ('microsoft' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-      obj.region = o.region;
-      obj.use_custom_tts = o.use_custom_tts;
-      obj.custom_tts_endpoint = o.custom_tts_endpoint;
-      obj.use_custom_stt = o.use_custom_stt;
-      obj.custom_stt_endpoint = o.custom_stt_endpoint;
-    }
-    else if ('wellsaid' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if ('nuance' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.client_id = o.client_id;
-      obj.secret = o.secret ? obscureKey(o.secret) : null;
-      obj.nuance_tts_uri = o.nuance_tts_uri;
-      obj.nuance_stt_uri = o.nuance_stt_uri;
-    }
-    else if ('deepgram' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if ('ibm' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.tts_api_key = obscureKey(o.tts_api_key);
-      obj.tts_region = o.tts_region;
-      obj.stt_api_key = obscureKey(o.stt_api_key);
-      obj.stt_region = o.stt_region;
-      obj.instance_id = o.instance_id;
-    } else if ('nvidia' == obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.riva_server_uri = o.riva_server_uri;
-    }
-    else if ('soniox' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if (obj.vendor.startsWith('custom:')) {
-      const o = JSON.parse(decrypt(credential));
-      obj.auth_token = obscureKey(o.auth_token);
-      obj.custom_stt_url = o.custom_stt_url;
-      obj.custom_tts_url = o.custom_tts_url;
-    }
+    decryptCredential(obj, credential, logger);
 
     if (req.user.hasAccountAuth && obj.account_sid === null) {
       delete obj.api_key;
@@ -488,8 +404,14 @@ router.put('/:sid', async(req, res) => {
         const {
           use_custom_tts,
           custom_tts_endpoint,
+          custom_tts_endpoint_url,
           use_custom_stt,
-          custom_stt_endpoint
+          custom_stt_endpoint,
+          custom_stt_endpoint_url,
+          custom_stt_url,
+          custom_tts_url,
+          cobalt_server_uri,
+          model_id
         } = req.body;
 
         const newCred = {
@@ -499,13 +421,19 @@ router.put('/:sid', async(req, res) => {
           aws_region,
           use_custom_tts,
           custom_tts_endpoint,
+          custom_tts_endpoint_url,
           use_custom_stt,
           custom_stt_endpoint,
+          custom_stt_endpoint_url,
           stt_region,
           tts_region,
           riva_server_uri,
           nuance_stt_uri,
-          nuance_tts_uri
+          nuance_tts_uri,
+          custom_stt_url,
+          custom_tts_url,
+          cobalt_server_uri,
+          model_id
         };
         logger.info({o, newCred}, 'updating speech credential with this new credential');
         obj.credential = encryptCredential(newCred);
@@ -534,7 +462,7 @@ router.put('/:sid', async(req, res) => {
  * Test a credential
  */
 router.get('/:sid/test', async(req, res) => {
-  const logger = req.app.locals.logger;
+  const {logger, synthAudio} = req.app.locals;
   try {
     const sid = parseSpeechCredentialSid(req);
     const creds = await SpeechCredential.retrieve(sid);
@@ -618,8 +546,10 @@ router.get('/:sid/test', async(req, res) => {
         region,
         use_custom_tts,
         custom_tts_endpoint,
+        custom_tts_endpoint_url,
         use_custom_stt,
-        custom_stt_endpoint
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
       } = credential;
       if (cred.use_for_tts) {
         try {
@@ -628,8 +558,10 @@ router.get('/:sid/test', async(req, res) => {
             region,
             use_custom_tts,
             custom_tts_endpoint,
+            custom_tts_endpoint_url,
             use_custom_stt,
-            custom_stt_endpoint
+            custom_stt_endpoint,
+            custom_stt_endpoint_url
           });
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
@@ -758,6 +690,41 @@ router.get('/:sid/test', async(req, res) => {
           SpeechCredential.sttTestResult(sid, false);
         }
       }
+    } else if (cred.vendor === 'elevenlabs') {
+      const {api_key, model_id} = credential;
+      if (cred.use_for_tts) {
+        try {
+          await testElevenlabs(logger, {api_key, model_id});
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'assemblyai') {
+      const {api_key} = credential;
+      if (cred.use_for_stt) {
+        try {
+          await testAssemblyStt(logger, {api_key});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'whisper') {
+      if (cred.use_for_tts) {
+        try {
+          await testWhisper(logger, synthAudio, credential);
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
     }
 
     res.status(200).json(results);
@@ -767,4 +734,93 @@ router.get('/:sid/test', async(req, res) => {
   }
 });
 
+/**
+ * Fetch speech voices and languages
+ */
+
+router.post('/voices', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {vendor, label} = req.body;
+  const account_sid = req.user.account_sid || req.body.account_sid;
+  const service_provider_sid = req.user.service_provider_sid ||
+    req.body.service_provider_sid || parseServiceProviderSid(req);
+  try {
+    res.status(200).json(await getTtsVoices(vendor, label, service_provider_sid, account_sid));
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.post('/languages', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {vendor, label} = req.body;
+  const account_sid = req.user.account_sid || req.body.account_sid;
+  const service_provider_sid = req.user.service_provider_sid ||
+    req.body.service_provider_sid || parseServiceProviderSid(req);
+  try {
+    res.status(200).json(await getTtsLanguages(vendor, label, service_provider_sid, account_sid));
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+const getTtsVoices = async(vendor, label, service_provider_sid, account_sid) => {
+  const credentials = await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+    service_provider_sid, account_sid, vendor, label);
+  const tmp = credentials && credentials.length > 0 ? credentials[0] : null;
+  const cred = tmp ? JSON.parse(decrypt(tmp.credential)) : null;
+  if (vendor === 'elevenlabs') {
+    const get = bent('https://api.elevenlabs.io', 'GET', 'json', {
+      ...(cred && {
+        'xi-api-key' : cred.api_key
+      })
+    });
+    const resp = await get('/v1/voices');
+    return resp ? resp.voices.map((v) => {
+      let name = `${v.name}${v.category !== 'premade' ? ` (${v.category})` : ''} -
+${v.labels.accent ? ` ${v.labels.accent},` : ''}
+${v.labels.description ? ` ${v.labels.description},` : ''}
+${v.labels.age ? ` ${v.labels.age},` : ''}
+${v.labels.gender ? ` ${v.labels.gender},` : ''}
+${v.labels['use case'] ? ` ${v.labels['use case']},` : ''}
+`;
+      const lastIndex = name.lastIndexOf(',');
+      if (lastIndex !== -1) {
+        name = name.substring(0, lastIndex);
+      }
+      return {
+        value: v.voice_id,
+        name
+      };
+    }).sort((a, b) => a.name.localeCompare(b.name)) : [];
+  }
+  return [];
+};
+
+const getTtsLanguages = async(vendor, label, service_provider_sid, account_sid) => {
+  const credentials = await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+    service_provider_sid, account_sid, vendor, label);
+  const tmp = credentials && credentials.length > 0 ? credentials[0] : null;
+  const cred = tmp ? JSON.parse(decrypt(tmp.credential)) : null;
+  if (vendor === 'elevenlabs') {
+    if (!cred) {
+      return [];
+    }
+    const get = bent('https://api.elevenlabs.io', 'GET', 'json', {
+      'xi-api-key' : cred.api_key
+    });
+    const resp = await get('/v1/models');
+    if (!resp || resp.length === 0) {
+      return [];
+    }
+    const model = resp.find((m) => m.model_id === cred.model_id);
+    return model ? model.languages.map((l) => {
+      return {
+        value: l.language_id,
+        name: l.name
+      };
+    }).sort((a, b) => a.name.localeCompare(b.name)) : [];
+  }
+};
+
 module.exports = router;

lib/routes/api/system-information.js

@@ -8,7 +8,7 @@ router.post('/', async(req, res) => {
 
 router.get('/', async(req, res) => {
   const [sysInfo] = await SystemInformation.retrieveAll();
-  res.status(200).json(sysInfo);
+  res.status(200).json(sysInfo || {});
 });
 
 module.exports = router;

lib/routes/api/tts-cache.js

@@ -0,0 +1,111 @@
+const router = require('express').Router();
+const {
+  parseAccountSid
+} = require('./utils');
+const SpeechCredential = require('../../models/speech-credential');
+const fs = require('fs');
+const { v4: uuidv4 } = require('uuid');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const Account = require('../../models/account');
+const sysError = require('../error');
+const { getSpeechCredential, decryptCredential } = require('../../utils/speech-utils');
+
+router.delete('/', async(req, res) => {
+  const {purgeTtsCache} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  if (account_sid) {
+    await purgeTtsCache({account_sid});
+  } else {
+    await purgeTtsCache();
+  }
+  res.sendStatus(204);
+});
+
+router.get('/', async(req, res) => {
+  const {getTtsSize} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  let size = 0;
+  if (account_sid) {
+    size = await getTtsSize(`tts:${account_sid}:*`);
+  } else {
+    size = await getTtsSize();
+  }
+  res.status(200).json({size});
+});
+
+router.post('/Synthesize', async(req, res) => {
+  const {logger, synthAudio} = req.app.locals;
+  try {
+    const accountSid = parseAccountSid(req);
+    const body = req.body;
+    if (!body.speech_credential_sid || !body.text || !body.language || !body.voice) {
+      throw new DbErrorBadRequest('speech_credential_sid, text, language, voice are all required');
+    }
+
+    const result = await Account.retrieve(accountSid);
+    if (!result || result.length === 0 || !result[0].is_active) {
+      throw new DbErrorBadRequest(`Account not found for sid ${accountSid}`);
+    }
+    const credentials = await SpeechCredential.retrieve(body.speech_credential_sid);
+    if (!credentials || credentials.length === 0) {
+      throw new
+      DbErrorBadRequest(`There is no available speech credential for ${body.speech_credential_sid}`);
+    }
+    const {credential, ...obj} = credentials[0];
+
+    decryptCredential(obj, credential, logger, false);
+    const cred = getSpeechCredential(obj, logger);
+
+    const { text, language, engine = 'standard' } = body;
+    const salt = uuidv4();
+    /* parse Nuance voices into name and model */
+    let voice = body.voice;
+    let model;
+    if (cred.vendor === 'nuance' && voice) {
+      const arr = /([A-Za-z-]*)\s+-\s+(enhanced|standard)/.exec(voice);
+      if (arr) {
+        voice = arr[1];
+        model = arr[2];
+      }
+    }
+    const stats = {
+      histogram: () => {},
+      increment: () => {},
+    };
+    const { filePath } = await synthAudio(stats, {
+      account_sid: accountSid,
+      text,
+      vendor: cred.vendor,
+      language,
+      voice,
+      engine,
+      model,
+      salt,
+      credentials: cred,
+      disableTtsCache: false
+    });
+
+    const stat = fs.statSync(filePath);
+    res.writeHead(200, {
+      'Content-Type': 'audio/mpeg',
+      'Content-Length': stat.size,
+    });
+
+    const readStream = fs.createReadStream(filePath);
+    // We replaced all the event handlers with a simple call to readStream.pipe()
+    readStream.pipe(res);
+
+    readStream.on('end', () => {
+      // Delete the file after it's been read
+      fs.unlink(filePath, (err) => {
+        if (err) throw err;
+        logger.info(`${filePath} was deleted`);
+      });
+    });
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/users.js

@@ -338,8 +338,8 @@ router.put('/:user_sid', async(req, res) => {
   //if (req.user.user_sid && req.user.user_sid !== user_sid) return res.sendStatus(403);
 
   if (!hasAdminAuth &&
-  !(hasAccountAuth && req.user.account_sid === user[0].account_sid) &&
-  !(hasServiceProviderAuth && req.user.service_provider_sid === user[0].service_provider_sid) &&
+  !(hasAccountAuth && user[0] && req.user.account_sid === user[0].account_sid) &&
+  !(hasServiceProviderAuth && user[0] && req.user.service_provider_sid === user[0].service_provider_sid) &&
   (req.user.user_sid && req.user.user_sid !== user_sid)) {
     return res.sendStatus(403);
   }

lib/routes/stripe/webhook.js

@@ -61,8 +61,7 @@ router.post('/', express.raw({type: 'application/json'}), async(req, res) => {
   }
 
   /* process event */
-  logger.info(`received webhook: ${evt.type}`);
-  if (evt.type.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
+  if (evt?.type?.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
   else {
     logger.debug(evt, 'unhandled stripe webook');
   }

lib/swagger/swagger.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.0
 info:
-  title: jambonz REST API
-  description: jambonz REST API
+  title: Jambonz REST API
+  description: Jambonz REST API specification
   contact:
     email: daveh@drachtio.org
   license:
@@ -44,6 +44,8 @@ tags:
     description: Least Cost Routing Routes operations
   - name: LcrCarrierSetEntries
     description: Least Cost Routing Carrier Set Entries operation
+  - name: GoogleCustomVoices
+    description: Google Custom voices operation
 paths:
   /BetaInviteCodes:
     post:
@@ -786,7 +788,7 @@ paths:
           required: true
           schema:
             type: string
-            example: mycorp.sip.jambonz.us
+            example: mycorp.sip.jambonz.cloud
       responses:
         200:
           description: indicates whether value is already in use
@@ -3046,6 +3048,12 @@ paths:
           enum:
             - inbound
             - outbound
+      - in: query
+        name: filter
+        required: false
+        schema:
+          type: string
+        description: Filter value can be caller ID, callee ID or call Sid
     get:
       tags:
         - Accounts
@@ -3245,6 +3253,18 @@ paths:
           enum:
             - inbound
             - outbound
+      - in: query
+        name: from
+        required: false
+        schema:
+          type: string
+        description: calling number to retrieve
+      - in: query
+        name: to
+        required: false
+        schema:
+          type: string
+        description: called number to retrieve
     get:
       tags:
         - Service Providers
@@ -3779,7 +3799,7 @@ paths:
   /Accounts/{AccountSid}/Calls:
     post:
       tags:
-       - Accounts
+      - Accounts
       summary: create a call
       operationId: createCall
       parameters:
@@ -3838,6 +3858,10 @@ paths:
                   type: object
                   description: The customer SIP headers to associate with the call
                   example: {"X-Custom-Header": "Hello"}
+                sipRequestWithinDialogHook:
+                  type: string
+                  description: The sip indialog hook to receive session messages
+                  example: '/customHook'
       responses:
         201:
           description: call successfully created
@@ -4146,6 +4170,108 @@ paths:
                       type: string
                     length:
                       type: string
+  /Accounts/{AccountSid}/RegisteredSipUsers:
+    parameters:
+      - name: AccountSid
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+    get:
+      tags:
+        - Accounts
+      summary: retrieve online sip users for an account
+      operationId: listQueues
+      responses:
+        200:
+          description: retrieve online sip users for an account
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+  /Accounts/{AccountSid}/RegisteredSipUsers/{Client}:
+    parameters:
+      - name: Client
+        in: path
+        required: true
+        style: simple
+        explode: false
+        schema:
+          type: string
+    get:
+      tags:
+        - Accounts
+      summary: retrieve registered client registration
+      operationId: getRegisteredClient
+      responses:
+        200:
+          description: registered client found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RegisteredClient'
+  /Accounts/{AccountSid}/TtsCache/Synthesize:
+    post:
+      tags:
+        - Accounts
+      summary: get TTS from provider
+      operationId: Synthesize
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                speech_credential_sid:
+                  type: string
+                  description: Speech credential Sid
+                  example: 553b4b6b-8918-4394-a46d-1e3c5a3c717b
+                text:
+                  type: string
+                  description: the text to convert to audio
+                  example: Hello How are you
+                language:
+                  type: string
+                  description: language is used in text
+                  example: en-US
+                voice:
+                  type: string
+                  description: voice ID
+                  example: en-US-Standard-C
+              required:
+                - speech_credential_sid
+                - text
+                - language
+                - voice
+      responses:
+        200:
+          description: Audio is created
+          content:
+            audio/mpeg:
+              schema:
+                type: string
+                format: binary
+        400:
+          description: bad request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+        422:
+          description: unprocessable entity
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
   /Lcrs:
     post:
       tags:
@@ -4293,6 +4419,69 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/GeneralError'
+  /Lcrs/{LcrSid}/Routes:
+    parameters:
+      - name: LcrSid
+        in: path
+        required: true
+        style: simple
+        explode: false
+        schema:
+          type: string
+    post:
+      tags:
+        - Lcrs
+      summary: Create least cost routing routes and carrier set entries
+      operationId: createLeastCostRoutingRoutesAndCarrierEntries
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/LcrRoutes'
+      responses:
+        204:
+          description: least cost routing routes and carrier set entries created
+        400:
+          description: bad request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+        404:
+          description: least cost routing not found
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+    put:
+      tags:
+        - Lcrs
+      summary: update least cost routing routes and carrier set entries
+      operationId: updateLeastCostRoutingRoutesAndCarrierEntries
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/LcrRoutes'
+      responses:
+        204:
+          description: least cost routing ruoutes and carrier entries updated
+        400:
+          description: bad request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+        404:
+          description: least cost routing not found
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
   /LcrRoutes:
     post:
       tags:
@@ -4585,6 +4774,173 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/GeneralError'
+  /GoogleCustomVoices:
+    post:
+      tags:
+        - GoogleCustomVoices
+      summary: create a Google custom voice
+      operationId: createGoogleCustomVoice
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: '#/components/schemas/GoogleCustomVoice'
+              required:
+                - speech_credential_sid
+                - name
+                - reported_usage
+                - model
+      responses:
+        201:
+          description: Least Cost Routing Carrier Set Entry successfully created
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessfulAdd'
+        400:
+          description: bad request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+        422:
+          description: unprocessable entity
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+    get:
+      tags:
+        - GoogleCustomVoices
+      parameters:
+        - in: query
+          name: service_provider_sid
+          required: false
+          schema:
+            type: string
+            description: return only the google voice custom operated belong to this service provider
+        - in: query
+          name: account_sid
+          required: false
+          schema:
+            type: string
+            description: return only the google voice custom operated belong to this account_sid
+            
+        - in: query
+          name: speech_credential_sid
+          required: false
+          schema:
+            type: string
+            description: return only the google voice custom operated belong to this speech credential
+      summary: list google custom voices
+      operationId: listGoogleCustomVoices
+      responses:
+        200:
+          description: list oflist google custom voices
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/GoogleCustomVoice'
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+  /GoogleCustomVoices/{GoogleCustomVoiceSid}:
+    parameters:
+      - name: GoogleCustomVoiceSid
+        in: path
+        required: true
+        style: simple
+        explode: false
+        schema:
+          type: string
+    delete:
+      tags:
+        - GoogleCustomVoices
+      summary: delete a google custom voice
+      operationId: deleteGoogleCustomVoice
+      responses:
+        204:
+          description: google custom voice successfully deleted
+        404:
+          description: google custom voice not found
+        422:
+          description: unprocessable entity
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+                example:
+                  msg: a service provider with active accounts can not be deleted
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+    get:
+      tags:
+        - GoogleCustomVoices
+      summary: retrieve google custom voice
+      operationId: getGoogleCustomVoice
+      responses:
+        200:
+          description: google custom voice found
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GoogleCustomVoice'
+        404:
+          description: google custom voice not found
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+    put:
+      tags:
+        - GoogleCustomVoices
+      summary: update google custom voice
+      operationId: updateGoogleCustomVoice
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/GoogleCustomVoice'
+      responses:
+        204:
+          description: google custom voice updated
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GoogleCustomVoice'
+        400:
+          description: bad request
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
+        404:
+          description: least cost routing carrier set entry not found
+        500:
+          description: system error
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/GeneralError'
 components:
   securitySchemes:
     bearerAuth:
@@ -5543,6 +5899,73 @@ components:
       - lcr_route_sid
       - voip_carrier_sid
       - priority
+    LcrRouteAndCarrierEntries:
+      allOf:
+        - $ref: '#/components/schemas/LcrRoute'
+        - type: object
+          properties:
+            lcr_carrier_set_entries:
+              type: array
+              items:
+                $ref: '#/components/schemas/LcrCarrierSetEntry'
+    LcrRoutes:
+      type: array
+      items:
+        $ref: '#/components/schemas/LcrRouteAndCarrierEntries'
+    GoogleCustomVoice:
+      type: object
+      properties:
+        speech_credential_sid:
+          type: string
+          example: 3fa85f64-5717-4562-b3fc-2c963f66afa6
+        name:
+          type: string
+          example: Sally
+        reported_usage:
+          type: string
+          example: REALTIME
+        model:
+          type: string
+          example: projects/12412312/locations/global/models/2134124123-2dbf-43be-9593-12314123
+      required:
+      - speech_credential_sid
+      - name
+      - reported_usage
+      - model
+    RegisteredClient:
+      type: object
+      properties:
+        name:
+          type: string
+          example: xhoaluu
+        contact:
+          type: string
+          example: sip:0dluqjt6@od41sl9jfc9m.invalid;transport=ws
+        expiryTime:
+          type: number
+          example: 1698981449173
+        protocol:
+          type: string
+          example: wss
+        allow_direct_app_calling:
+          type: number
+          example: 1
+        allow_direct_queue_calling:
+          type: number
+          example: 1
+        allow_direct_user_calling:
+          type: number
+          example: 1
+        registered_status:
+          type: string
+          enum:
+            - active
+            - inactive
+      required:
+      - speech_credential_sid
+      - name
+      - reported_usage
+      - model
 
 security:
   - bearerAuth: []

lib/utils/dns-utils.js

@@ -89,7 +89,12 @@ const createDnsRecords = async(logger, domain, name, value, ttl = 3600) => {
       const str = await res.text();
       return JSON.parse(str);
     }
-    logger.error({res}, 'Error creating records');
+    let body;
+    try {
+      body = await res.json();
+    } catch (err) {
+    }
+    logger.error({headers: res.headers, body}, `Error creating records, status ${res.statusCode}`);
   } catch (err) {
     logger.error({err}, 'Error retrieving domains');
   }

lib/utils/email-utils.js

@@ -47,13 +47,15 @@ const sendEmailByCustomVendor = async(logger, from, to, subject, text) => {
 };
 
 const sendEmailByMailgun = async(logger, from, to, subject, text) => {
-  const mg = mailgun.client({
-    username: 'api',
-    key: process.env.MAILGUN_API_KEY
-  });
   if (!process.env.MAILGUN_API_KEY) throw new Error('MAILGUN_API_KEY env variable is not defined!');
   if (!process.env.MAILGUN_DOMAIN) throw new Error('MAILGUN_DOMAIN env variable is not defined!');
 
+  const mg = mailgun.client({
+    username: 'api',
+    key: process.env.MAILGUN_API_KEY,
+    ...(process.env.MAILGUN_URL && {url: process.env.MAILGUN_URL})
+  });
+
   try {
     const res = await mg.messages.create(process.env.MAILGUN_DOMAIN, {
       from,

lib/utils/encrypt-decrypt.js

@@ -23,7 +23,18 @@ const decrypt = (data) => {
   return decrpyted.toString();
 };
 
+const obscureKey = (key, key_spoiler_length = 6) => {
+  const key_spoiler_char = 'X';
+
+  if (!key || key.length <= key_spoiler_length) {
+    return key;
+  }
+
+  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
+};
+
 module.exports = {
   encrypt,
-  decrypt
+  decrypt,
+  obscureKey
 };

lib/utils/free_plans.json

@@ -2,7 +2,7 @@
   "trial": [
     {
       "category": "voice_call_session",
-      "quantity": 20
+      "quantity": 5
     },
     {
       "category": "device",

lib/utils/homer-utils.js

@@ -64,7 +64,7 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
   }
 };
 
-const getHomerPcap = async(logger, apiKey, callIds) => {
+const getHomerPcap = async(logger, apiKey, callIds, method) => {
   if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
     logger.debug('getHomerPcap: Homer integration not installed');
   }
@@ -73,18 +73,23 @@ const getHomerPcap = async(logger, apiKey, callIds) => {
     const stream = await postPcap('/api/v3/export/call/messages/pcap', {
       param: {
         transaction: {
-          call: true,
-          registration: true,
+          call: method === 'invite',
+          registration: method === 'register',
           rest: false
         },
         orlogic: true,
         search: {
-          '1_call': {
-            callid: callIds
-          },
-          '1_registration': {
-            callid: callIds
-          }
+          ...(method === 'invite' && {
+            '1_call': {
+              callid: callIds
+            }
+          })
+          ,
+          ...(method === 'register' && {
+            '1_registration': {
+              callid: callIds
+            }
+          })
         },
       },
       timestamp: {

lib/utils/jaeger-utils.js

@@ -9,7 +9,8 @@ const getJaegerTrace = async(logger, traceId) => {
   try {
     return await getJSON(`/api/v3/traces/${traceId}`);
   } catch (err) {
-    logger.error({err}, `getJaegerTrace: Error retrieving spans for traceId ${traceId}`);
+    const url = `${process.env.JAEGER_BASE_URL}/api/traces/${traceId}`;
+    logger.error({err, traceId}, `getJaegerTrace: Error retrieving spans from ${url}`);
   }
 };
 

lib/utils/jambonz-sample.text

@@ -0,0 +1 @@
+Hello From Jambonz. This file was created because Record all call bucket credential test.

lib/utils/speech-utils.js

@@ -5,6 +5,8 @@ const sdk = require('microsoft-cognitiveservices-speech-sdk');
 const { SpeechClient } = require('@soniox/soniox-node');
 const bent = require('bent');
 const fs = require('fs');
+const { AssemblyAI } = require('assemblyai');
+const {decrypt, obscureKey} = require('./encrypt-decrypt');
 
 
 const testSonioxStt = async(logger, credentials) => {
@@ -203,6 +205,46 @@ const testWellSaidTts = async(logger, credentials) => {
   }
 };
 
+const testElevenlabs = async(logger, credentials) => {
+  const {api_key, model_id} = credentials;
+  try {
+    const post = bent('https://api.elevenlabs.io', 'POST', 'buffer', {
+      'xi-api-key': api_key,
+      'Accept': 'audio/mpeg',
+      'Content-Type': 'application/json'
+    });
+    const mp3 = await post('/v1/text-to-speech/21m00Tcm4TlvDq8ikWAM', {
+      text: 'Hello',
+      model_id,
+      voice_settings: {
+        stability: 0.5,
+        similarity_boost: 0.5
+      }
+    });
+    return mp3;
+  } catch (err) {
+    logger.info({err}, 'synthEvenlabs returned error');
+    throw err;
+  }
+};
+
+const testWhisper = async(logger, synthAudio, credentials) => {
+  try {
+    await synthAudio({increment: () => {}, histogram: () => {}},
+      {
+        vendor: 'whisper',
+        credentials,
+        language: 'en-US',
+        voice: 'alloy',
+        text: 'Hi there and welcome to jambones!'
+      }
+    );
+  } catch (err) {
+    logger.info({err}, 'synthEvenlabs returned error');
+    throw err;
+  }
+};
+
 const testIbmTts = async(logger, getTtsVoices, credentials) => {
   const {tts_api_key, tts_region} = credentials;
   const voices = await getTtsVoices({vendor: 'ibm', credentials: {tts_api_key, tts_region}});
@@ -237,6 +279,137 @@ const testWellSaidStt = async(logger, credentials) => {
   return true;
 };
 
+const testAssemblyStt = async(logger, credentials) => {
+  const {api_key} = credentials;
+
+  const assemblyai = new AssemblyAI({
+    apiKey: api_key
+  });
+
+  const audioUrl = `${__dirname}/../../data/test_audio.wav`;
+
+  return new Promise((resolve, reject) => {
+    assemblyai.transcripts
+      .create({ audio_url: audioUrl })
+      .then((transcript) => {
+        logger.debug({transcript}, 'got transcription from AssemblyAi');
+        if (transcript.status === 'error') {
+          return reject({message: transcript.error});
+        }
+        return resolve(transcript.text);
+      })
+      .catch((err) => {
+        logger.info({err}, 'failed to get assemblyAI transcription');
+        reject(err);
+      });
+  });
+};
+
+const getSpeechCredential = (credential, logger) => {
+  const {vendor} = credential;
+  logger.info(
+    `Speech vendor: ${credential.vendor} ${credential.label ? `, label: ${credential.label}` : ''} selected`);
+  if ('google' === vendor) {
+    try {
+      const cred = JSON.parse(credential.service_key.replace(/\n/g, '\\n'));
+      return {
+        ...credential,
+        credentials: cred
+      };
+    } catch (err) {
+      logger.info({err}, `malformed google service_key provisioned for account ${credential.speech_credential_sid}`);
+    }
+  }
+  else if (['aws', 'polly'].includes(vendor)) {
+    return {
+      ...credential,
+      accessKeyId: credential.access_key_id,
+      secretAccessKey: credential.secret_access_key,
+      region: credential.aws_region || 'us-east-1'
+    };
+  }
+  return credential;
+};
+
+function decryptCredential(obj, credential, logger, isObscureKey = true) {
+  if ('google' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    const key_header = '-----BEGIN PRIVATE KEY-----\n';
+    const obscured = {
+      ...o,
+      private_key: `${key_header}${isObscureKey ?
+        obscureKey(o.private_key.slice(key_header.length, o.private_key.length)) :
+        o.private_key.slice(key_header.length, o.private_key.length)}`
+    };
+    obj.service_key = JSON.stringify(obscured);
+  }
+  else if ('aws' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.access_key_id = o.access_key_id;
+    obj.secret_access_key = isObscureKey ? obscureKey(o.secret_access_key) : o.secret_access_key;
+    obj.aws_region = o.aws_region;
+    logger.info({obj, o}, 'retrieving aws speech credential');
+  }
+  else if ('microsoft' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = isObscureKey ? obscureKey(o.api_key) : o.api_key;
+    obj.region = o.region;
+    obj.use_custom_tts = o.use_custom_tts;
+    obj.custom_tts_endpoint = o.custom_tts_endpoint;
+    obj.custom_tts_endpoint_url = o.custom_tts_endpoint_url;
+    obj.use_custom_stt = o.use_custom_stt;
+    obj.custom_stt_endpoint = o.custom_stt_endpoint;
+    obj.custom_stt_endpoint_url = o.custom_stt_endpoint_url;
+    logger.info({obj, o}, 'retrieving azure speech credential');
+  }
+  else if ('wellsaid' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = isObscureKey ? obscureKey(o.api_key) : o.api_key;
+  }
+  else if ('nuance' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.client_id = o.client_id;
+    obj.secret = o.secret ? (isObscureKey ? obscureKey(o.secret) : o.secret) : null;
+  }
+  else if ('deepgram' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = isObscureKey ? obscureKey(o.api_key) : o.api_key;
+  }
+  else if ('ibm' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.tts_api_key = isObscureKey ? obscureKey(o.tts_api_key) : o.tts_api_key;
+    obj.tts_region = o.tts_region;
+    obj.stt_api_key = isObscureKey ? obscureKey(o.stt_api_key) : o.stt_api_key;
+    obj.stt_region = o.stt_region;
+    obj.instance_id = o.instance_id;
+  } else if ('nvidia' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.riva_server_uri = o.riva_server_uri;
+  } else if ('cobalt' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.cobalt_server_uri = o.cobalt_server_uri;
+  } else if ('soniox' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = isObscureKey ? obscureKey(o.api_key) : o.api_key;
+  } else if ('elevenlabs' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = isObscureKey ? obscureKey(o.api_key) : o.api_key;
+    obj.model_id = o.model_id;
+  } else if (obj.vendor.startsWith('custom:')) {
+    const o = JSON.parse(decrypt(credential));
+    obj.auth_token = isObscureKey ? obscureKey(o.auth_token) : o.auth_token;
+    obj.custom_stt_url = o.custom_stt_url;
+    obj.custom_tts_url = o.custom_tts_url;
+  } else if ('assemblyai' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = isObscureKey ? obscureKey(o.api_key) : o.api_key;
+  } else if ('whisper' === obj.vendor) {
+    const o = JSON.parse(decrypt(credential));
+    obj.api_key = isObscureKey ? obscureKey(o.api_key) : o.api_key;
+    obj.model_id = o.model_id;
+  }
+}
+
 module.exports = {
   testGoogleTts,
   testGoogleStt,
@@ -251,5 +424,10 @@ module.exports = {
   testDeepgramStt,
   testIbmTts,
   testIbmStt,
-  testSonioxStt
+  testSonioxStt,
+  testElevenlabs,
+  testAssemblyStt,
+  getSpeechCredential,
+  decryptCredential,
+  testWhisper
 };

lib/utils/storage-utils.js

@@ -0,0 +1,134 @@
+const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3');
+const {Storage} = require('@google-cloud/storage');
+const fs = require('fs');
+const { BlobServiceClient } = require('@azure/storage-blob');
+
+// Azure
+
+async function testAzureStorage(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient('jambonz-sample.text');
+
+  await blockBlobClient.uploadFile(`${__dirname}/jambonz-sample.text`);
+}
+
+async function getAzureStorageObject(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient(opts.key);
+  const response = await blockBlobClient.download(0);
+  return response.readableStreamBody;
+}
+
+async function deleteAzureStorageObject(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient(opts.key);
+  await blockBlobClient.delete();
+}
+
+// Google
+
+function _initGoogleClient(opts) {
+  const serviceKey = JSON.parse(opts.service_key);
+  return new Storage({
+    projectId: serviceKey.project_id,
+    credentials: {
+      client_email: serviceKey.client_email,
+      private_key: serviceKey.private_key
+    },
+  });
+}
+
+async function testGoogleStorage(logger, opts) {
+  return new Promise((resolve, reject) => {
+    const storage = _initGoogleClient(opts);
+
+    const blob = storage.bucket(opts.name).file('jambonz-sample.text');
+
+    fs.createReadStream(`${__dirname}/jambonz-sample.text`)
+      .pipe(blob.createWriteStream())
+      .on('error', (err) => reject(err))
+      .on('finish', () => resolve());
+  });
+}
+
+async function getGoogleStorageObject(logger, opts) {
+  const storage = _initGoogleClient(opts);
+
+  const bucket = storage.bucket(opts.name);
+  const file = bucket.file(opts.key);
+  const [exists] = await file.exists();
+  if (exists) {
+    return file.createReadStream();
+  }
+}
+
+async function deleteGoogleStorageObject(logger, opts) {
+  const storage = _initGoogleClient(opts);
+
+  const bucket = storage.bucket(opts.name);
+  const file = bucket.file(opts.key);
+
+  await file.delete();
+}
+
+// S3
+
+function _initS3Client(opts) {
+  return new S3Client({
+    credentials: {
+      accessKeyId: opts.access_key_id,
+      secretAccessKey: opts.secret_access_key,
+    },
+    region: opts.region || 'us-east-1',
+    ...(opts.vendor === 's3_compatible' && { endpoint: opts.endpoint, forcePathStyle: true })
+  });
+}
+
+async function testS3Storage(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const input = {
+    'Body': 'Hello From Jambonz',
+    'Bucket': opts.name,
+    'Key': 'jambonz-sample.text'
+  };
+  const command = new PutObjectCommand(input);
+  await s3.send(command);
+}
+
+async function getS3Object(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const command = new GetObjectCommand(
+    {
+      Bucket: opts.name,
+      Key: opts.key
+    }
+  );
+  const res = await s3.send(command);
+  return res.Body;
+}
+
+async function deleteS3Object(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const command = new DeleteObjectCommand(
+    {
+      Bucket: opts.name,
+      Key: opts.key
+    }
+  );
+  await s3.send(command);
+}
+
+module.exports = {
+  testS3Storage,
+  getS3Object,
+  deleteS3Object,
+  testGoogleStorage,
+  getGoogleStorageObject,
+  deleteGoogleStorageObject,
+  testAzureStorage,
+  getAzureStorageObject,
+  deleteAzureStorageObject
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "jambonz-api-server",
-  "version": "v0.8.2",
+  "version": "0.8.5",
   "description": "",
   "main": "app.js",
   "scripts": {
@@ -19,16 +19,22 @@
     "url": "https://github.com/jambonz/jambonz-api-server.git"
   },
   "dependencies": {
-    "@aws-sdk/client-transcribe": "^3.290.0",
-    "@deepgram/sdk": "^1.10.2",
-    "@google-cloud/speech": "^5.1.0",
-    "@jambonz/db-helpers": "^0.7.9",
-    "@jambonz/realtimedb-helpers": "^0.7.2",
-    "@jambonz/speech-utils": "^0.0.12",
-    "@jambonz/time-series": "^0.2.5",
-    "@jambonz/verb-specifications": "^0.0.21",
-    "@soniox/soniox-node": "^1.1.0",
+    "@aws-sdk/client-s3": "^3.363.0",
+    "@aws-sdk/client-transcribe": "^3.363.0",
+    "@azure/storage-blob": "^12.15.0",
+    "@deepgram/sdk": "^1.21.0",
+    "@google-cloud/speech": "^5.2.0",
+    "@google-cloud/storage": "^6.12.0",
+    "@jambonz/db-helpers": "^0.9.1",
+    "@jambonz/lamejs": "^1.2.2",
+    "@jambonz/mw-registrar": "^0.2.5",
+    "@jambonz/realtimedb-helpers": "^0.8.7",
+    "@jambonz/speech-utils": "^0.0.26",
+    "@jambonz/time-series": "^0.2.8",
+    "@jambonz/verb-specifications": "^0.0.45",
+    "@soniox/soniox-node": "^1.1.1",
     "argon2": "^0.30.3",
+    "assemblyai": "^3.0.1",
     "bent": "^7.3.12",
     "cors": "^2.8.5",
     "debug": "^4.3.4",
@@ -38,8 +44,8 @@
     "helmet": "^5.1.0",
     "ibm-watson": "^7.1.2",
     "jsonwebtoken": "^9.0.0",
-    "mailgun.js": "^3.7.3",
-    "microsoft-cognitiveservices-speech-sdk": "^1.24.1",
+    "mailgun.js": "^9.1.2",
+    "microsoft-cognitiveservices-speech-sdk": "1.31.0",
     "mysql2": "^2.3.3",
     "nocache": "3.0.4",
     "passport": "^0.6.0",
@@ -49,6 +55,8 @@
     "stripe": "^8.222.0",
     "swagger-ui-express": "^4.4.0",
     "uuid": "^8.3.2",
+    "wav": "^1.0.2",
+    "ws": "^8.12.1",
     "yamljs": "^0.3.0"
   },
   "devDependencies": {

test/accounts.js

@@ -14,7 +14,7 @@ const {
   createPhoneNumber, 
   deleteObjectBySid} = require('./utils');
 const logger = require('../lib/logger');
-const { pushBack } = require('@jambonz/realtimedb-helpers')({
+const { addToSortedSet } = require('@jambonz/realtimedb-helpers')({
   host: process.env.JAMBONES_REDIS_HOST,
   port: process.env.JAMBONES_REDIS_PORT || 6379
 }, logger);
@@ -168,11 +168,33 @@ test('account tests', async(t) => {
         queue_event_hook: {
           url: 'http://example.com/q',
           method: 'post'
+        },
+        record_all_calls: true,
+        record_format: 'wav',
+        bucket_credential: {
+          vendor: 'aws_s3',
+          region: 'us-east-1',
+          name: 'recordings',
+          access_key_id: 'access_key_id',
+          secret_access_key: 'secret access key'
         }
+        
       }
     });
     t.ok(result.statusCode === 204, 'successfully updated account using account level token');
 
+    /* verify that bucket credential is updated*/
+    result = await request.get(`/Accounts/${sid}`, {
+      auth: {bearer: accountLevelToken},
+      json: true,
+    });
+
+    t.ok(result.bucket_credential.vendor === 'aws_s3', 'bucket_vendor was updated');
+    t.ok(result.bucket_credential.name === 'recordings', 'bucket_name was updated');
+    t.ok(result.bucket_credential.access_key_id === 'access_key_id', 'bucket_access_key_id was updated');
+    t.ok(result.record_all_calls === 1, 'record_all_calls was updated');
+    t.ok(result.record_format === 'wav', 'record_format was updated');
+
     /* verify that account level api key last_used was updated*/
     result = await request.get(`/Accounts/${sid}/ApiKeys`, {
       auth: {bearer: accountLevelToken},
@@ -268,8 +290,8 @@ test('account tests', async(t) => {
     t.ok(result.statusCode === 204, 'successfully deleted a call session limit for an account');
 
     /* query account queues */
-    await pushBack(`queue:${sid}:test`, 'url1');
-    await pushBack(`queue:${sid}:dummy`, 'url2');
+    await addToSortedSet(`queue:${sid}:test`, 'url1');
+    await addToSortedSet(`queue:${sid}:dummy`, 'url2');
 
     result = await request.get(`/Accounts/${sid}/Queues`, {
       auth: authAdmin,

test/applications.js

@@ -53,7 +53,7 @@ test('application tests', async(t) => {
         ]'
       }
     });
-    t.ok(result.statusCode === 400, 'Cant create application with invalid app_josn');
+    t.ok(result.statusCode === 400, 'Cant create application with invalid app_json');
     
     /* add an application */
     result = await request.post('/Applications', {
@@ -81,7 +81,15 @@ test('application tests', async(t) => {
               "seekOffset": 8000,\
               "actionHook": "/play/action"\
           }\
-        ]'
+        ]',
+        use_for_fallback_speech: 1,
+        fallback_speech_synthesis_vendor: 'google',
+        fallback_speech_synthesis_language: 'en-US',
+        fallback_speech_synthesis_voice: 'man',
+        fallback_speech_synthesis_label: 'label1',
+        fallback_speech_recognizer_vendor: 'google',
+        fallback_speech_recognizer_language: 'en-US',
+        fallback_speech_recognizer_label: 'label1'
       }
     });
     t.ok(result.statusCode === 201, 'successfully created application');
@@ -102,6 +110,14 @@ test('application tests', async(t) => {
     });
     t.ok(result.name === 'daveh' , 'successfully retrieved application by sid');
     t.ok(result.messaging_hook.url === 'http://example.com/sms' , 'successfully retrieved messaging_hook from application');
+    t.ok(result.use_for_fallback_speech === 1, 'successfully create use_for_fallback_speech');
+    t.ok(result.fallback_speech_synthesis_vendor === 'google', 'successfully create fallback_speech_synthesis_vendor');
+    t.ok(result.fallback_speech_synthesis_language === 'en-US', 'successfully create fallback_speech_synthesis_language');
+    t.ok(result.fallback_speech_synthesis_voice === 'man', 'successfully create fallback_speech_synthesis_voice');
+    t.ok(result.fallback_speech_synthesis_label === 'label1', 'successfully create fallback_speech_synthesis_label');
+    t.ok(result.fallback_speech_recognizer_vendor === 'google', 'successfully create fallback_speech_recognizer_vendor');
+    t.ok(result.fallback_speech_recognizer_language === 'en-US', 'successfully create fallback_speech_recognizer_language');
+    t.ok(result.fallback_speech_recognizer_label === 'label1', 'successfully create fallback_speech_recognizer_label');
     let app_json = JSON.parse(result.app_json);
     t.ok(app_json[0].verb === 'play', 'successfully retrieved app_json from application')
 
@@ -125,7 +141,16 @@ test('application tests', async(t) => {
               "X-Reason" : "maximum call duration exceeded"\
             }\
           }\
-        ]'
+        ]',
+        record_all_calls: true,
+        use_for_fallback_speech: 0,
+        fallback_speech_synthesis_vendor: 'microsoft',
+        fallback_speech_synthesis_language: 'en-US',
+        fallback_speech_synthesis_voice: 'woman',
+        fallback_speech_synthesis_label: 'label2',
+        fallback_speech_recognizer_vendor: 'microsoft',
+        fallback_speech_recognizer_language: 'en-US',
+        fallback_speech_recognizer_label: 'label2'
       }
     });
     t.ok(result.statusCode === 204, 'successfully updated application');
@@ -138,6 +163,15 @@ test('application tests', async(t) => {
     t.ok(result.messaging_hook.url === 'http://example2.com/mms' , 'successfully updated messaging_hook');
     app_json = JSON.parse(result.app_json);
     t.ok(app_json[0].verb === 'hangup', 'successfully updated app_json from application')
+    t.ok(result.record_all_calls === 1, 'successfully updated record_all_calls from application')
+    t.ok(result.use_for_fallback_speech === 0, 'successfully update use_for_fallback_speech');
+    t.ok(result.fallback_speech_synthesis_vendor === 'microsoft', 'successfully update fallback_speech_synthesis_vendor');
+    t.ok(result.fallback_speech_synthesis_language === 'en-US', 'successfully update fallback_speech_synthesis_language');
+    t.ok(result.fallback_speech_synthesis_voice === 'woman', 'successfully update fallback_speech_synthesis_voice');
+    t.ok(result.fallback_speech_synthesis_label === 'label2', 'successfully update fallback_speech_synthesis_label');
+    t.ok(result.fallback_speech_recognizer_vendor === 'microsoft', 'successfully update fallback_speech_recognizer_vendor');
+    t.ok(result.fallback_speech_recognizer_language === 'en-US', 'successfully update fallback_speech_recognizer_language');
+    t.ok(result.fallback_speech_recognizer_label === 'label2', 'successfully update fallback_speech_recognizer_label');
 
     /* remove applications app_json*/
     result = await request.put(`/Applications/${sid}`, {

test/call-test.js

@@ -31,8 +31,8 @@ test('Create Call Success With Synthesizer in Payload', async (t) => {
     auth: authUser,
     json: true,
     body: {
-      call_hook: "https://public-apps.jambonz.us/hello-world",
-      call_status_hook: "https://public-apps.jambonz.us/call-status",
+      call_hook: "https://public-apps.jambonz.cloud/hello-world",
+      call_status_hook: "https://public-apps.jambonz.cloud/call-status",
       from: "15083778299",
       to: {
         type: "phone",
@@ -73,8 +73,8 @@ test('Create Call Success Without Synthesizer in Payload', async (t) => {
     auth: authUser,
     json: true,
     body: {
-      call_hook: "https://public-apps.jambonz.us/hello-world",
-      call_status_hook: "https://public-apps.jambonz.us/call-status",
+      call_hook: "https://public-apps.jambonz.cloud/hello-world",
+      call_status_hook: "https://public-apps.jambonz.cloud/call-status",
       from: "15083778299",
       to: {
         type: "phone",
@@ -83,4 +83,74 @@ test('Create Call Success Without Synthesizer in Payload', async (t) => {
     }
   }).then(data => { t.ok(false, 'Create Call should not be success') })
     .catch(error => { t.ok(error.response.statusCode === 400, 'Call failed for no synthesizer') });
+});
+
+test("Create call with application sid and app_json", async(t) => {
+  const app = require('../app');
+
+  const service_provider_sid = await createServiceProvider(request, 'account3_has_synthesizer');
+  const account_sid = await createAccount(request, service_provider_sid, 'account3_has_synthesizer');
+
+  const token = jwt.sign({
+    account_sid,
+    scope: "account",
+    permissions: ["PROVISION_USERS", "PROVISION_SERVICES", "VIEW_ONLY"]
+  }, process.env.JWT_SECRET, { expiresIn: '1h' });
+  const authUser = { bearer: token };
+  const speech_sid = await createGoogleSpeechCredentials(request, account_sid, null, authUser, true, true);
+
+  // GIVEN
+/* add an application */
+
+const app_json = '[\
+  {\
+    "verb": "play",\
+    "url": "https://example.com/example.mp3",\
+    "timeoutSecs": 10,\
+    "seekOffset": 8000,\
+    "actionHook": "/play/action"\
+}\
+]';
+let result = await request.post('/Applications', {
+  resolveWithFullResponse: true,
+  auth: authUser,
+  json: true,
+  body: {
+    name: 'daveh',
+    account_sid,
+    call_hook: {
+      url: 'http://example.com'
+    },
+    call_status_hook: {
+      url: 'http://example.com/status',
+      method: 'POST'
+    },
+    messaging_hook: {
+      url: 'http://example.com/sms'
+    },
+    app_json
+  }
+});
+t.ok(result.statusCode === 201, 'successfully created application');
+const sid = result.body.sid;
+
+// WHEN
+result = await request.post(`/Accounts/${account_sid}/Calls`, {
+  resolveWithFullResponse: true,
+  auth: authUser,
+  json: true,
+  body: {
+    application_sid: sid,
+    from: "15083778299",
+    to: {
+      type: "phone",
+      number: "15089084809"
+    },
+  }
+});
+// THEN
+t.ok(result.statusCode === 201, 'successfully created Call without Synthesizer && application_sid');
+const fs_request = await getLastRequestFromFeatureServer('15083778299_createCall');
+const obj = JSON.parse(fs_request);
+t.ok(obj.body.app_json == app_json, 'app_json successfully added')
 });

test/clients.js

@@ -0,0 +1,145 @@
+const test = require('tape') ;
+const ADMIN_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734de';
+const authAdmin = {bearer: ADMIN_TOKEN};
+const request = require('request-promise-native').defaults({
+  baseUrl: 'http://127.0.0.1:3000/v1'
+});
+
+
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+test('client test', async(t) => {
+  const app = require('../app');
+  const {registrar} = app.locals;
+
+  try {
+    let result;
+    /* add a service provider */
+    result = await request.post('/ServiceProviders', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        name: 'client_sp',
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created client service provider');
+    const sp_sid = result.body.sid;
+
+    /* add an account */
+    result = await request.post('/Accounts', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        name: 'sample_account',
+        service_provider_sid: sp_sid,
+        sip_realm: 'drachtio.org',
+        registration_hook: {
+          url: 'http://example.com/reg',
+          method: 'get'
+        },
+        webhook_secret: 'foobar'
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created account');
+    const account_sid = result.body.sid;
+
+    /* add new entity */
+    result = await request.post('/Clients', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        account_sid,
+        username: 'client1',
+        password: 'sdf12412',
+        is_active: 1,
+        allow_direct_app_calling: 0,
+        allow_direct_queue_calling: 1,
+        allow_direct_user_calling: 1,
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created Client');
+    const sid = result.body.sid;
+
+    /* register the client */
+    const r = await registrar.add(
+      "dhorton@drachtio.org",
+      {
+        contact: "10.10.1.1",
+        sbcAddress: "192.168.1.1",
+        protocol: "udp",
+      },
+      5
+    );
+    t.ok(r, 'successfully registered Client');
+
+    /* query all registered clients */
+    result = await request.get(`/Accounts/${account_sid}/RegisteredSipUsers`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 1 && result[0] === 'dhorton@drachtio.org', 
+      'successfully queried all registered clients');
+
+    /* query all entity */
+    result = await request.get('/Clients', {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 1 , 'successfully queried all Clients');
+
+    /* query one entity */
+    result = await request.get(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.account_sid === account_sid , 'successfully retrieved Client by sid');
+    t.ok(result.client_sid, 'successfully retrieved Client by sid');
+    t.ok(result.username ===  'client1', 'successfully retrieved Client by sid');
+    t.ok(result.is_active === 1 , 'successfully retrieved Client by sid');
+    t.ok(result.password === 'sXXXXXXX' , 'successfully retrieved Client by sid');
+    t.ok(result.allow_direct_app_calling === 0 , 'successfully retrieved Client by sid');
+
+    /* update the entity */
+    result = await request.put(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+      resolveWithFullResponse: true,
+      body: {
+        is_active: 0
+      }
+    });
+    t.ok(result.statusCode === 204, 'successfully updated Client');
+    /* query one entity */
+    result = await request.get(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.is_active === 0 , 'successfully updated Client');
+    t.ok(result.password === 'sXXXXXXX' , 'successfully retrieved Client by sid');
+
+    /* delete Client */
+    result = await request.delete(`/Clients/${sid}`, {
+      resolveWithFullResponse: true,
+      simple: false,
+      json: true,
+      auth: authAdmin
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted Clients');
+
+    /* query all entity */
+    result = await request.get('/Clients', {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 0 , 'successfully queried all Clients');
+    
+  } catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+})

test/index.js

@@ -23,4 +23,6 @@ require('./system-information');
 require('./lcr-carriers-set-entries');
 require('./lcr-routes');
 require('./lcrs');
+require('./tts-cache');
+require('./clients');
 require('./docker_stop');

test/oauth/gh-get-user.js

@@ -7,7 +7,7 @@ const test = async() => {
     headers: {
       Authorization: `Bearer ${process.env.GH_CODE}`,
       Accept: 'application/json',
-      'User-Agent': 'jambonz.us'
+      'User-Agent': 'jambonz.cloud'
     }
   }, (err, response, body) => {
     if (err) console.log(error);

test/recent-calls.js

@@ -75,8 +75,29 @@ test('recent calls tests', async(t) => {
       auth: authUser,
       json: true,
     });
+    console.log(JSON.stringify(result));
     t.ok(result.data.length === 5, 'retrieved 5 recent calls by account');
-    //console.log({data: result.data}, 'Account recent calls');
+
+    result = await request.get(`/Accounts/${account_sid}/RecentCalls?page=1&count=25&filter=1508`, {
+      auth: authUser,
+      json: true,
+    });
+    console.log(JSON.stringify(result));
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by account and from');
+
+    result = await request.get(`/Accounts/${account_sid}/RecentCalls?page=1&count=25&filter=15080`, {
+      auth: authUser,
+      json: true,
+    });
+    console.log(JSON.stringify(result));
+    t.ok(result.data.length === 0, 'retrieved 0 recent calls by account and non-matching from');
+
+    result = await request.get(`/Accounts/${account_sid}/RecentCalls?page=1&count=25&filter=1888`, {
+      auth: authUser,
+      json: true,
+    });
+    console.log(JSON.stringify(result));
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by account and to');
 
     /* query last 7 days by service provider */
     result = await request.get(`/ServiceProviders/${service_provider_sid}/RecentCalls?page=1&count=25`, {
@@ -84,6 +105,19 @@ test('recent calls tests', async(t) => {
       json: true,
     });
     t.ok(result.data.length === 5, 'retrieved 5 recent calls by service provider');
+
+    result = await request.get(`/ServiceProviders/${service_provider_sid}/RecentCalls?page=1&count=25&filter=1508`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by service provider and from');
+
+    result = await request.get(`/ServiceProviders/${service_provider_sid}/RecentCalls?page=1&count=25&filter=1888`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.data.length === 5, 'retrieved 5 recent calls by service provider and to');
+
     //console.log({data: result.data}, 'SP recent calls');
 
     /* pull sip traces and pcap from homer */

test/sbcs.js

@@ -17,6 +17,37 @@ test('sbc_addresses tests', async(t) => {
     let result;
     const service_provider_sid = await createServiceProvider(request);
 
+    /* add service_provider user */
+    const sp_name = 'sbc_service_provider';
+    const sp_password = 'password';
+    result = await request.post(`/Users`, {
+      resolveWithFullResponse: true,
+      json: true,
+      auth: authAdmin,
+      body: {
+        name: sp_name,
+        email: 'sbc_sp@jambonz.com',
+        is_active: true,
+        force_change: false,
+        initial_password: sp_password,
+        service_provider_sid,
+      }
+    });
+    t.ok(result.statusCode === 201 && result.body.user_sid, 'SBC service_provider scope user created');
+    const sbc_sp_user_sid = result.body.user_sid;
+
+    result = await request.post('/login', {
+      resolveWithFullResponse: true,
+      json: true,
+      body: {
+        username: sp_name,
+        password: sp_password,  
+      }
+    });
+    t.ok(result.statusCode === 200 && result.body.token, 'successfully logged in as sbc user');
+    const authSbcSp = {bearer: result.body.token};
+
+
     /* add a service provider sbc */
     result = await request.post('/Sbcs', {
       resolveWithFullResponse: true,
@@ -38,6 +69,20 @@ test('sbc_addresses tests', async(t) => {
     //console.log(result.body)
     t.ok(result.body.length === 1 && result.body[0].ipv4 === '192.168.1.4', 'successfully retrieved service provider sbc');
 
+    result = await request.get('/Sbcs', {
+      resolveWithFullResponse: true,
+      auth: authSbcSp,
+      json: true
+    });
+    //console.log(result.body)
+    t.ok(result.body.length === 1 && result.body[0].ipv4 === '192.168.1.4', 'successfully retrieved service provider sbc');
+
+    await request.delete(`/Users/${sbc_sp_user_sid}`, {
+      resolveWithFullResponse: true,
+      json: true,
+      auth: authAdmin,
+    });
+
     await deleteObjectBySid(request, '/Sbcs', sid);
     await deleteObjectBySid(request, '/ServiceProviders', service_provider_sid);
 

test/sip-gateways.js

@@ -27,7 +27,8 @@ test('sip gateway tests', async(t) => {
         ipv4: '192.168.1.1',
         netmask: 32,
         inbound: true,
-        outbound: true
+        outbound: true,
+        protocol: 'tcp'
       }
     });
     t.ok(result.statusCode === 201, 'successfully created sip gateway ');
@@ -48,6 +49,7 @@ test('sip gateway tests', async(t) => {
     });
     //console.log(`result: ${JSON.stringify(result)}`);
     t.ok(result.ipv4 === '192.168.1.1' , 'successfully retrieved voip carrier by sid');
+    t.ok(result.protocol === 'tcp' , 'successfully retrieved voip carrier by sid');
 
 
     /* update sip gateway */
@@ -58,7 +60,8 @@ test('sip gateway tests', async(t) => {
       body: {
         port: 5061,
         netmask:24,
-        outbound: false
+        outbound: false,
+        protocol: 'udp'
       }
     });
     t.ok(result.statusCode === 204, 'successfully updated voip carrier');

test/speech-credentials.js

@@ -84,6 +84,7 @@ test('speech credentials tests', async(t) => {
       json: true,
       body: {
         vendor: 'google',
+        label: 'label1',
         service_key: jsonKey,
         use_for_tts: true,
         use_for_stt: true
@@ -111,6 +112,7 @@ test('speech credentials tests', async(t) => {
       json: true,
     });
     t.ok(result.vendor === 'google' , 'successfully retrieved speech credential by sid');
+    t.ok(result.label === 'label1' , 'label is successfully created');
     
     /* query all credentials */
     result = await request.get(`/Accounts/${account_sid}/SpeechCredentials`, {
@@ -445,6 +447,117 @@ test('speech credentials tests', async(t) => {
     });
     t.ok(result.statusCode === 204, 'successfully deleted speech credential');
 
+    /* add a credential for cobalt */
+    result = await request.post(`/Accounts/${account_sid}/SpeechCredentials`, {
+      resolveWithFullResponse: true,
+      auth: authUser,
+      json: true,
+      body: {
+        vendor: 'cobalt',
+        use_for_stt: true,
+        use_for_tts: false,
+        cobalt_server_uri: '32.32.32.32:2727',
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added speech credential for Cobalt');
+    const cobalt_sid = result.body.sid;
+
+    /* delete the credential */
+    result = await request.delete(`/Accounts/${account_sid}/SpeechCredentials/${cobalt_sid}`, {
+      auth: authUser,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted speech credential for Cobalt');
+
+    /* add a credential for elevenlabs */
+    result = await request.post(`/Accounts/${account_sid}/SpeechCredentials`, {
+      resolveWithFullResponse: true,
+      auth: authUser,
+      json: true,
+      body: {
+        vendor: 'elevenlabs',
+        use_for_stt: true,
+        use_for_tts: false,
+        api_key: 'asdasdasdasddsadasda',
+        model_id: 'eleven_multilingual_v2'
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added speech credential for Cobalt');
+    const elevenlabs_sid = result.body.sid;
+
+    /* delete the credential */
+    result = await request.delete(`/Accounts/${account_sid}/SpeechCredentials/${elevenlabs_sid}`, {
+      auth: authUser,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted speech credential for Cobalt');
+
+
+    /* add a credential for custom voices google */
+    result = await request.post(`/Accounts/${account_sid}/SpeechCredentials`, {
+      resolveWithFullResponse: true,
+      auth: authUser,
+      json: true,
+      body: {
+        vendor: 'google',
+        use_for_stt: true,
+        use_for_tts: false,
+        service_key: jsonKey
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added speech credential for custom voice google');
+    const customvoice_google_speech_credential_sid = result.body.sid;
+
+    result = await request.post(`/GoogleCustomVoices`, {
+      resolveWithFullResponse: true,
+      auth: authUser,
+      json: true,
+      body: {
+        speech_credential_sid: customvoice_google_speech_credential_sid,
+        name: "Sally",
+        reported_usage: 'REALTIME',
+        model: "path/to/modelId"
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added custom voice google');
+    const customvoice_google_sid = result.body.sid;
+
+    /* delete the credential */
+    result = await request.delete(`/GoogleCustomVoices/${customvoice_google_sid}`, {
+      auth: authUser,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted custom voice google');
+
+
+    /* delete the credential */
+    result = await request.delete(`/Accounts/${account_sid}/SpeechCredentials/${customvoice_google_speech_credential_sid}`, {
+      auth: authUser,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted speech credential for custom voice google');
+
+    /* add a credential for assemblyAI */
+    result = await request.post(`/Accounts/${account_sid}/SpeechCredentials`, {
+      resolveWithFullResponse: true,
+      auth: authUser,
+      json: true,
+      body: {
+        vendor: 'assemblyai',
+        use_for_stt: true,
+        api_key: "APIKEY"
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added speech credential for assemblyai');
+    const assemblyAiSid = result.body.sid;
+
+    /* delete the credential */
+    result = await request.delete(`/Accounts/${account_sid}/SpeechCredentials/${assemblyAiSid}`, {
+      auth: authUser,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted speech credential');
+
     await deleteObjectBySid(request, '/Accounts', account_sid);
     await deleteObjectBySid(request, '/ServiceProviders', service_provider_sid);
     t.end();

test/tts-cache.js

@@ -0,0 +1,56 @@
+const test = require('tape') ;
+const jwt = require('jsonwebtoken');
+const ADMIN_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734de';
+const authAdmin = {bearer: ADMIN_TOKEN};
+const request = require('request-promise-native').defaults({
+  baseUrl: 'http://127.0.0.1:3000/v1'
+});
+const crypto = require('crypto');
+const logger = require('../lib/logger');
+
+const {
+  client,
+} = require('@jambonz/speech-utils')({
+  host: process.env.JAMBONES_REDIS_HOST,
+  port: process.env.JAMBONES_REDIS_PORT || 6379
+}, logger);
+
+function makeSynthKey({account_sid = '', vendor, language, voice, engine = '', text}) {
+  const hash = crypto.createHash('sha1');
+  hash.update(`${language}:${vendor}:${voice}:${engine}:${text}`);
+  return `tts${account_sid ? (':' + account_sid) : ''}:${hash.digest('hex')}`;
+}
+
+test('tts-cache', async(t) => {
+  const app = require('../app');
+  try {
+    // create caches
+    const minRecords = 8;
+    for (const i in Array(minRecords).fill(0)) {
+      await client.set(makeSynthKey({vendor: i, language: i, voice: i, engine: i, text: i}), i);
+    }
+
+    let result = await request.get('/TtsCache', {
+      auth: authAdmin,
+      json: true,
+    });
+
+    t.ok(result.size === minRecords, 'get cache correctly');
+
+    result = await request.delete('/TtsCache', {
+      auth: authAdmin,
+      resolveWithFullResponse: true,
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted application after removing phone number');
+
+    result = await request.get('/TtsCache', {
+      auth: authAdmin,
+      json: true,
+    });
+
+    t.ok(result.size === 0, 'deleted cache successfully');
+  } catch(err) {
+    console.error(err);
+    t.end(err);
+  }
+});

test/webapp_tests.js

@@ -192,7 +192,7 @@ test('webapp tests', async(t) => {
     t.ok(result.statusCode === 200 && result.body.available === true, 'indicates when email is available');
    
     /* check if a subdomain is available */
-    result = await request.get('/Availability?type=subdomain&value=mycompany.sip.jambonz.us', {
+    result = await request.get('/Availability?type=subdomain&value=mycompany.sip.jambonz.cloud', {
       resolveWithFullResponse: true,
       auth: authUser,
       json: true,