support playht3.0 languages (#357)

* support playht3.0 languages

* update speech utils version

.github/workflows/ci.yml

@@ -7,11 +7,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          docker-compose --version
       - uses: actions/setup-node@v3
         with:
           node-version: lts/*
       - run: npm install
       - run: npm run jslint
+      - name: Install Docker Compose
+        run: |
+          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+          sudo chmod +x /usr/local/bin/docker-compose
+          docker-compose --version
       - run: npm test
 
  

.github/workflows/docker-publish-dbcreate.yml

@@ -2,16 +2,10 @@ name: Docker
 
 on:
   push:
-    # Publish `main` as Docker `latest` image.
     branches:
       - main
-
-    # Publish `v1.2.3` tags as releases.
     tags:
-      - v*
-
-env:
-  IMAGE_NAME: db-create
+      - '*'
 
 jobs:
   push:
@@ -20,32 +14,42 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build . --file Dockerfile.db-create --tag $IMAGE_NAME
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Push image
+      - name: prepare tag 
+        id: prepare_tag
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+            IMAGE_ID=jambonz/db-create
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
 
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
 
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "main" ] && VERSION=latest
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
 
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.db-create
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.github/workflows/docker-publish.yml

@@ -2,16 +2,10 @@ name: Docker
 
 on:
   push:
-    # Publish `main` as Docker `latest` image.
     branches:
       - main
-
-    # Publish `v1.2.3` tags as releases.
     tags:
-      - v*
-
-env:
-  IMAGE_NAME: api-server
+      - '*'
 
 jobs:
   push:
@@ -20,32 +14,41 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v3
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build . --file Dockerfile --tag $IMAGE_NAME
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Push image
+      - name: prepare tag 
+        id: prepare_tag
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+            IMAGE_ID=jambonz/api-server
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
 
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
 
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "main" ] && VERSION=latest
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
 
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18.14.1-alpine3.16 as base
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
 
 RUN apk --update --no-cache add --virtual .builds-deps build-base python3
 

Dockerfile.db-create

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 node:18.14.1-alpine3.16 as base
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
 
 RUN apk --update --no-cache add --virtual .builds-deps build-base python3
 

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 Drachtio Communications Services, LLC
+Copyright (c) 2018-2024 FirstFive8, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,29 +1,46 @@
 # jambonz-api-server ![Build Status](https://github.com/jambonz/jambonz-api-server/workflows/CI/badge.svg)
 
-Jambones REST API server.
+Jambones REST API server of the jambones platform.
 
 ## Configuration
 
-This process requires the following environment variables to be set.
+Configuration is provided via environment variables:
 
-```
-JAMBONES_MYSQL_HOST
-JAMBONES_MYSQL_USER
-JAMBONES_MYSQL_PASSWORD
-JAMBONES_MYSQL_DATABASE
-JAMBONES_MYSQL_CONNECTION_LIMIT   # defaults to 10
-JAMBONES_REDIS_HOST
-JAMBONES_REDIS_PORT
-JAMBONES_LOGLEVEL                 # defaults to info
-JAMBONES_API_VERSION              # defaults to v1
-HTTP_PORT                         # defaults to 3000
-```
+| variable | meaning | required?|
+|----------|----------|---------|
+|JWT_SECRET| secret for signing JWT token |yes|
+|JWT_EXPIRES_IN| expiration time for JWT token(in minutes) |no|
+|ENCRYPTION_SECRET| secret for credential encryption(JWT_SECRET is deprecated) |yes|
+|HTTP_PORT| tcp port to listen on for API requests from jambonz-api-server |no|
+|JAMBONES_LOGLEVEL| log level for application, 'info' or 'debug' |no|
+|JAMBONES_MYSQL_HOST| mysql host |yes|
+|JAMBONES_MYSQL_USER| mysql username |yes|
+|JAMBONES_MYSQL_PASSWORD|  mysql password |yes|
+|JAMBONES_MYSQL_DATABASE| mysql data |yes|
+|JAMBONES_MYSQL_PORT| mysql port |no|
+|JAMBONES_MYSQL_CONNECTION_LIMIT| mysql connection limit |no|
+|JAMBONES_REDIS_HOST| redis host |yes|
+|JAMBONES_REDIS_PORT| redis port |no|
+|RATE_LIMIT_WINDOWS_MINS| rate limit window |no|
+|RATE_LIMIT_MAX_PER_WINDOW| number of requests per window |no|
+|JAMBONES_TRUST_PROXY| trust proxies, must be a number |no|
+|JAMBONES_API_VERSION| api version |no|
+|JAMBONES_TIME_SERIES_HOST| influxdb host |yes|
+|JAMBONES_CLUSTER_ID| cluster id |no|
+|HOMER_BASE_URL| HOMER URL |no|
+|HOMER_USERNAME| HOMER username |no|
+|HOMER_PASSWORD| HOMER password |no|
+|K8S| service running as kubernetes service |no|
+|K8S_FEATURE_SERVER_SERVICE_NAME| feature server name(required for K8S) |no|
+|K8S_FEATURE_SERVER_SERVICE_PORT| feature server port(required for K8S) |no|
+|JAMBONZ_RECORD_WS_USERNAME| recording websocket username|no|
+|JAMBONZ_RECORD_WS_PASSWORD| recording websocket password|no|
 
 #### Database dependency
 A mysql database is used to store long-lived objects such as Accounts, Applications, etc. To create the database schema, use or review the scripts in the 'db' folder, particularly:
-- [create_db.sql](db/create_db.sql), which creates the database and associated user (you may want to edit the username and password),
 - [jambones-sql.sql](db/jambones-sql.sql), which creates the schema,
-- [create-admin-token.sql](db/create-admin-token.sql), which creates an admin-level auth token that can be used for testing/exercising the API.
+- [seed-production-database-open-source.sql](db/seed-production-database-open-source.sql), which seeds the database with initial dataset(accounts, permissions, api keys, applications etc).
+- [create-admin-user.sql](db/create-admin-user.sql), which creates admin user with password set to "admin". The password will be forced to change after the first login.
 
 > Note: due to the dependency on the npmjs [mysql](https://www.npmjs.com/package/mysql) package, the mysql database must be configured to use sql [native authentication](https://medium.com/@crmcmullen/how-to-run-mysql-8-0-with-native-password-authentication-502de5bac661).
 

app.js

@@ -1,26 +1,29 @@
 const assert = require('assert');
-const opts = Object.assign({
-  timestamp: () => {
-    return `, "time": "${new Date().toISOString()}"`;
-  }
-}, {
-  level: process.env.JAMBONES_LOGLEVEL || 'info'
-});
-const logger = require('pino')(opts);
+const logger = require('./lib/logger');
 const express = require('express');
 const app = express();
 const helmet = require('helmet');
+const nocache = require('nocache');
 const rateLimit = require('express-rate-limit');
 const cors = require('cors');
 const passport = require('passport');
 const routes = require('./lib/routes');
+const Registrar = require('@jambonz/mw-registrar');
 
 assert.ok(process.env.JAMBONES_MYSQL_HOST &&
   process.env.JAMBONES_MYSQL_USER &&
   process.env.JAMBONES_MYSQL_PASSWORD &&
   process.env.JAMBONES_MYSQL_DATABASE, 'missing JAMBONES_MYSQL_XXX env vars');
-assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+if (process.env.JAMBONES_REDIS_SENTINELS) {
+  assert.ok(process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+    'missing JAMBONES_REDIS_SENTINEL_MASTER_NAME env var, JAMBONES_REDIS_SENTINEL_PASSWORD env var is optional');
+} else {
+  assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+}
 assert.ok(process.env.JAMBONES_TIME_SERIES_HOST, 'missing JAMBONES_TIME_SERIES_HOST env var');
+assert.ok(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET, 'missing ENCRYPTION_SECRET env var');
+assert.ok(process.env.JWT_SECRET, 'missing JWT_SECRET env var');
+
 const {
   queryCdrs,
   queryCdrsSP,
@@ -33,19 +36,27 @@ const {
   logger, process.env.JAMBONES_TIME_SERIES_HOST
 );
 const {
+  client,
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
   retrieveKey,
   deleteKey,
-  getTtsVoices
-} = require('@jambonz/realtimedb-helpers')({
-  host: process.env.JAMBONES_REDIS_HOST || 'localhost',
-  port: process.env.JAMBONES_REDIS_PORT || 6379
-}, logger);
+  incrKey,
+  listConferences
+} = require('./lib/helpers/realtimedb-helpers');
+const {
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache,
+  getAwsAuthToken,
+  getVerbioAccessToken,
+  synthAudio
+} = require('@jambonz/speech-utils')({}, logger);
 const {
   lookupAppBySid,
   lookupAccountBySid,
@@ -53,7 +64,8 @@ const {
   lookupAppByPhoneNumber,
   lookupCarrierBySid,
   lookupSipGatewayBySid,
-  lookupSmppGatewayBySid
+  lookupSmppGatewayBySid,
+  lookupClientByAccountAndUsername
 } = require('@jambonz/db-helpers')({
   host: process.env.JAMBONES_MYSQL_HOST,
   user: process.env.JAMBONES_MYSQL_USER,
@@ -64,22 +76,33 @@ const {
 }, logger);
 const PORT = process.env.HTTP_PORT || 3000;
 const authStrategy = require('./lib/auth')(logger, retrieveKey);
+const {delayLoginMiddleware} = require('./lib/middleware');
+const Websocket = require('ws');
 
 passport.use(authStrategy);
 
 app.locals = app.locals || {};
 app.locals = {
   ...app.locals,
+  registrar: new Registrar(logger, client),
   logger,
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
+  listConferences,
   purgeCalls,
   retrieveSet,
   addKey,
+  incrKey,
   retrieveKey,
   deleteKey,
   getTtsVoices,
+  getTtsSize,
+  getAwsAuthToken,
+  getVerbioAccessToken,
+  purgeTtsCache,
+  synthAudio,
   lookupAppBySid,
   lookupAccountBySid,
   lookupAccountByPhoneNumber,
@@ -87,6 +110,7 @@ app.locals = {
   lookupCarrierBySid,
   lookupSipGatewayBySid,
   lookupSmppGatewayBySid,
+  lookupClientByAccountAndUsername,
   queryCdrs,
   queryCdrsSP,
   queryAlerts,
@@ -110,6 +134,12 @@ const limiter = rateLimit({
   legacyHeaders: false, // Disable the `X-RateLimit-*` headers
 });
 
+// Setup websocket for recording audio
+const recordWsServer = require('./lib/record');
+const wsServer = new Websocket.Server({ noServer: true });
+wsServer.setMaxListeners(0);
+wsServer.on('connection', recordWsServer.bind(null, logger));
+
 if (process.env.JAMBONES_TRUST_PROXY) {
   const proxyCount = parseInt(process.env.JAMBONES_TRUST_PROXY);
   if (!isNaN(proxyCount) && proxyCount > 0) {
@@ -124,9 +154,11 @@ if (process.env.JAMBONES_TRUST_PROXY) {
 app.use(limiter);
 app.use(helmet());
 app.use(helmet.hidePoweredBy());
+app.use(nocache());
 app.use(passport.initialize());
 app.use(cors());
 app.use(express.urlencoded({extended: true}));
+app.use(delayLoginMiddleware);
 app.use(unless(['/stripe'], express.json()));
 app.use('/v1', unless(
   [
@@ -148,7 +180,52 @@ app.use((err, req, res, next) => {
   });
 });
 logger.info(`listening for HTTP traffic on port ${PORT}`);
-app.listen(PORT);
+const server = app.listen(PORT);
+
+
+const isValidWsKey = (hdr) => {
+  const username = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
+  const password = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
+  if (username && password) {
+    if (!hdr) {
+      // auth header is missing
+      return false;
+    }
+    const token = Buffer.from(`${username}:${password}`).toString('base64');
+    const arr = /^Basic (.*)$/.exec(hdr);
+    if (!Array.isArray(arr)) {
+      // malformed auth header
+      return false;
+    }
+    return arr[1] === token;
+  }
+  return true;
+};
+
+server.on('upgrade', (request, socket, head) => {
+  logger.debug({
+    url: request.url,
+    headers: request.headers,
+  }, 'received upgrade request');
+
+  /* verify the path starts with /transcribe */
+  if (!request.url.includes('/record/')) {
+    logger.info(`unhandled path: ${request.url}`);
+    return socket.write('HTTP/1.1 404 Not Found \r\n\r\n', () => socket.destroy());
+  }
+
+  /* verify the api key */
+  if (!isValidWsKey(request.headers['authorization'])) {
+    logger.info(`invalid auth header: ${request.headers['authorization'] || 'authorization header missing'}`);
+    return socket.write('HTTP/1.1 403 Forbidden \r\n\r\n', () => socket.destroy());
+  }
+
+  /* complete the upgrade */
+  wsServer.handleUpgrade(request, socket, head, (ws) => {
+    logger.info(`upgraded to websocket, url: ${request.url}`);
+    wsServer.emit('connection', ws, request.url);
+  });
+});
 
 // purge old calls from active call set every 10 mins
 async function purge() {

db/create-admin-user.sql

@@ -0,0 +1,10 @@
+/* hashed password is "admin" */
+insert into users (user_sid, name, email, hashed_password, force_change, provider, email_validated)
+values ('12c80508-edf9-4b22-8d09-55abd02648eb', 'admin', 'joe@foo.bar', '$argon2i$v=19$m=65536,t=3,p=4$c2FsdHNhbHRzYWx0c2FsdA$x5OO6gXFXS25oqUU2JvbYqrSgRxBujNUJBq6xv9EgjM', 1, 'local', 1);
+
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('8919e0dc-4d69-4de5-be56-a121598d9093', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc342a-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('d6fdf064-0a65-4b17-8b10-5500e956a159', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3a10-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('f68185dd-0486-4767-a77d-a0b84c1b236e' ,'12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3c5e-546a-11ed-bdc3-0242ac120002');

db/jambones-sql.sql

@@ -14,8 +14,12 @@ DROP TABLE IF EXISTS beta_invite_codes;
 
 DROP TABLE IF EXISTS call_routes;
 
+DROP TABLE IF EXISTS clients;
+
 DROP TABLE IF EXISTS dns_records;
 
+DROP TABLE IF EXISTS lcr;
+
 DROP TABLE IF EXISTS lcr_carrier_set_entry;
 
 DROP TABLE IF EXISTS lcr_routes;
@@ -50,8 +54,12 @@ DROP TABLE IF EXISTS signup_history;
 
 DROP TABLE IF EXISTS smpp_addresses;
 
+DROP TABLE IF EXISTS google_custom_voices;
+
 DROP TABLE IF EXISTS speech_credentials;
 
+DROP TABLE IF EXISTS system_information;
+
 DROP TABLE IF EXISTS users;
 
 DROP TABLE IF EXISTS smpp_gateways;
@@ -124,6 +132,19 @@ application_sid CHAR(36) NOT NULL,
 PRIMARY KEY (call_route_sid)
 ) COMMENT='a regex-based pattern match for call routing';
 
+CREATE TABLE clients
+(
+client_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+is_active BOOLEAN NOT NULL DEFAULT 1,
+username VARCHAR(64),
+password VARCHAR(1024),
+allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1,
+allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1,
+PRIMARY KEY (client_sid)
+);
+
 CREATE TABLE dns_records
 (
 dns_record_sid CHAR(36) NOT NULL UNIQUE ,
@@ -136,11 +157,23 @@ PRIMARY KEY (dns_record_sid)
 CREATE TABLE lcr_routes
 (
 lcr_route_sid CHAR(36),
+lcr_sid CHAR(36) NOT NULL,
 regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
 description VARCHAR(1024),
-priority INTEGER NOT NULL UNIQUE  COMMENT 'lower priority routes are attempted first',
+priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
 PRIMARY KEY (lcr_route_sid)
-) COMMENT='Least cost routing table';
+) COMMENT='An ordered list of  digit patterns in an LCR table.  The patterns are tested in sequence until one matches';
+
+CREATE TABLE lcr
+(
+lcr_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+is_active BOOLEAN NOT NULL DEFAULT 1,
+default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+service_provider_sid CHAR(36),
+account_sid CHAR(36),
+PRIMARY KEY (lcr_sid)
+) COMMENT='An LCR (least cost routing) table that is used by a service provider or account to make decisions about routing outbound calls when multiple carriers are available.';
 
 CREATE TABLE password_settings
 (
@@ -248,7 +281,10 @@ CREATE TABLE sbc_addresses
 sbc_address_sid CHAR(36) NOT NULL UNIQUE ,
 ipv4 VARCHAR(255) NOT NULL,
 port INTEGER NOT NULL DEFAULT 5060,
+tls_port INTEGER,
+wss_port INTEGER,
 service_provider_sid CHAR(36),
+last_updated DATETIME,
 PRIMARY KEY (sbc_address_sid)
 );
 
@@ -304,9 +340,29 @@ last_tested DATETIME,
 tts_tested_ok BOOLEAN,
 stt_tested_ok BOOLEAN,
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+label VARCHAR(64),
 PRIMARY KEY (speech_credential_sid)
 );
 
+CREATE TABLE google_custom_voices
+(
+google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+speech_credential_sid CHAR(36) NOT NULL,
+model VARCHAR(512) NOT NULL,
+reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+name VARCHAR(64) NOT NULL,
+PRIMARY KEY (google_custom_voice_sid)
+);
+
+CREATE TABLE system_information
+(
+domain_name VARCHAR(255),
+sip_domain_name VARCHAR(255),
+monitoring_domain_name VARCHAR(255),
+private_network_cidr VARCHAR(8192),
+log_level ENUM('info', 'debug') NOT NULL DEFAULT 'info'
+);
+
 CREATE TABLE users
 (
 user_sid CHAR(36) NOT NULL UNIQUE ,
@@ -357,6 +413,7 @@ smpp_inbound_password VARCHAR(64),
 register_from_user VARCHAR(128),
 register_from_domain VARCHAR(255),
 register_public_ip_in_contact BOOLEAN NOT NULL DEFAULT false,
+register_status VARCHAR(4096),
 PRIMARY KEY (voip_carrier_sid)
 ) COMMENT='A Carrier or customer PBX that can send or receive calls';
 
@@ -385,7 +442,7 @@ PRIMARY KEY (smpp_gateway_sid)
 CREATE TABLE phone_numbers
 (
 phone_number_sid CHAR(36) UNIQUE ,
-number VARCHAR(132) NOT NULL UNIQUE ,
+number VARCHAR(132) NOT NULL,
 voip_carrier_sid CHAR(36),
 account_sid CHAR(36),
 application_sid CHAR(36),
@@ -398,11 +455,15 @@ CREATE TABLE sip_gateways
 sip_gateway_sid CHAR(36),
 ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
 netmask INTEGER NOT NULL DEFAULT 32,
-port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+port INTEGER COMMENT 'sip signaling port',
 inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+send_options_ping BOOLEAN NOT NULL DEFAULT 0,
+use_sips_scheme BOOLEAN NOT NULL DEFAULT 0,
+pad_crypto BOOLEAN NOT NULL DEFAULT 0,
+protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
 
@@ -438,10 +499,21 @@ messaging_hook_sid CHAR(36) COMMENT 'webhook to call for inbound SMS/MMS ',
 app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
-speech_synthesis_voice VARCHAR(64),
+speech_synthesis_voice VARCHAR(256),
+speech_synthesis_label VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
+speech_recognizer_label VARCHAR(64),
+use_for_fallback_speech BOOLEAN DEFAULT false,
+fallback_speech_synthesis_vendor VARCHAR(64),
+fallback_speech_synthesis_language VARCHAR(12),
+fallback_speech_synthesis_voice VARCHAR(256),
+fallback_speech_synthesis_label VARCHAR(64),
+fallback_speech_recognizer_vendor VARCHAR(64),
+fallback_speech_recognizer_language VARCHAR(64),
+fallback_speech_recognizer_label VARCHAR(64),
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
 ) COMMENT='A defined set of behaviors to be applied to phone calls ';
 
@@ -479,6 +551,10 @@ subspace_client_secret VARCHAR(255),
 subspace_sip_teleport_id VARCHAR(255),
 subspace_sip_teleport_destinations VARCHAR(255),
 siprec_hook_sid CHAR(36),
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
+record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
+bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
+enable_debug_log BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (account_sid)
 ) COMMENT='An enterprise that uses the platform for comm services';
 
@@ -499,9 +575,20 @@ ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERE
 
 ALTER TABLE call_routes ADD FOREIGN KEY application_sid_idxfk (application_sid) REFERENCES applications (application_sid);
 
+CREATE INDEX client_sid_idx ON clients (client_sid);
+ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid);
+
 CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
 ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid);
+ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid);
+
+CREATE INDEX lcr_sid_idx ON lcr (lcr_sid);
+ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid);
+
+CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid);
+CREATE INDEX account_sid_idx ON lcr (account_sid);
 CREATE INDEX permission_sid_idx ON permissions (permission_sid);
 CREATE INDEX predefined_carrier_sid_idx ON predefined_carriers (predefined_carrier_sid);
 CREATE INDEX predefined_sip_gateway_sid_idx ON predefined_sip_gateways (predefined_sip_gateway_sid);
@@ -555,8 +642,6 @@ CREATE INDEX smpp_address_sid_idx ON smpp_addresses (smpp_address_sid);
 CREATE INDEX service_provider_sid_idx ON smpp_addresses (service_provider_sid);
 ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-CREATE UNIQUE INDEX speech_credentials_idx_1 ON speech_credentials (vendor,account_sid);
-
 CREATE INDEX speech_credential_sid_idx ON speech_credentials (speech_credential_sid);
 CREATE INDEX service_provider_sid_idx ON speech_credentials (service_provider_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
@@ -564,6 +649,10 @@ ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (ser
 CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
 ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid);
+CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid);
+ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE;
+
 CREATE INDEX user_sid_idx ON users (user_sid);
 CREATE INDEX email_idx ON users (email);
 CREATE INDEX phone_idx ON users (phone);
@@ -593,6 +682,8 @@ CREATE INDEX smpp_gateway_sid_idx ON smpp_gateways (smpp_gateway_sid);
 CREATE INDEX voip_carrier_sid_idx ON smpp_gateways (voip_carrier_sid);
 ALTER TABLE smpp_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
+CREATE UNIQUE INDEX phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid);
+
 CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
 CREATE INDEX number_idx ON phone_numbers (number);
 CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);
@@ -648,4 +739,4 @@ ALTER TABLE accounts ADD FOREIGN KEY device_calling_application_sid_idxfk (devic
 
 ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid);
 
-SET FOREIGN_KEY_CHECKS=1;
+SET FOREIGN_KEY_CHECKS=1;

db/seed-integration-test.sql

@@ -22,17 +22,25 @@ values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9
 
 -- create one service provider and one account
 insert into service_providers (service_provider_sid, name, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.us');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.cloud');
 
 insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
 values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'wh_secret_cJqgtMDPzDhhnjmaJH6Mtk');
 
+-- create account level api key
+insert into api_keys (api_key_sid, token, service_provider_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fa', '38700987-c7a4-4685-a5bb-af378f9734da', '9351f46a-678c-43f5-b8a6-d4eb58d131af');
+
+-- create SP level api key
+insert into api_keys (api_key_sid, token, account_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fs', '38700987-c7a4-4685-a5bb-af378f9734ds', '2708b1b3-2736-40ea-b502-c53d8396247f');
+
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES
@@ -79,6 +87,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways

db/seed-production-database-open-source.sql

@@ -24,10 +24,9 @@ values ('09e92f3c-9d73-4303-b63f-3668574862ce', '1cf2f4f4-64c4-4249-9a3e-5bb4cb5
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
-
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES
 ('7087fe50-8acb-4f3b-b820-97b573723aab', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'hello world', 'd31568d0-b193-4a05-8ff6-778369bc6efe', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US'),
@@ -68,14 +67,12 @@ VALUES
 ('d2ccfcb1-9198-4fe9-a0ca-6e49395837c4', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.172.60.0', 30, 5060, 1, 0),
 ('6b1d0032-4430-41f1-87c6-f22233d394ef', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.244.51.0', 30, 5060, 1, 0),
 ('0de40217-8bd5-4aa8-a9fd-1994282953c6', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.192', 30, 5060, 1, 0),
-('48b108e3-1ce7-4f18-a4cb-e41e63688bdf', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.193', 30, 5060, 1, 0),
-('d9131a69-fe44-4c2a-ba82-4adc81f628dd', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.194', 30, 5060, 1, 0),
-('34a6a311-4bd6-49ca-aa77-edd3cb92c6e1', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.171.127.195', 30, 5060, 1, 0),
 ('37bc0b20-b53c-4c31-95a6-f82b1c3713e3', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '35.156.191.128', 30, 5060, 1, 0),
 ('39791f4e-b612-4882-a37e-e92711a39f3f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.65.63.192', 30, 5060, 1, 0),
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways

db/seed-production-database.sql

@@ -53,6 +53,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways

db/upgrade-jambonz-db.js

@@ -87,6 +87,121 @@ const sql = {
     'ALTER TABLE user_permissions ADD FOREIGN KEY user_sid_idxfk (user_sid) REFERENCES users (user_sid) ON DELETE CASCADE',
     'ALTER TABLE user_permissions ADD FOREIGN KEY permission_sid_idxfk (permission_sid) REFERENCES permissions (permission_sid)',
     'ALTER TABLE `users` ADD COLUMN `is_active` BOOLEAN NOT NULL default true',
+  ],
+  '8003': [
+    'SET FOREIGN_KEY_CHECKS=0',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_status` VARCHAR(4096)',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `last_updated` DATETIME',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `tls_port` INTEGER',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `wss_port` INTEGER',
+    `CREATE TABLE system_information
+    (
+    domain_name VARCHAR(255),
+    sip_domain_name VARCHAR(255),
+    monitoring_domain_name VARCHAR(255)
+    )`,
+    'DROP TABLE IF EXISTS `lcr_routes`',
+    'DROP TABLE IF EXISTS `lcr_carrier_set_entry`',
+    `CREATE TABLE lcr_routes
+    (
+    lcr_route_sid CHAR(36),
+    lcr_sid CHAR(36) NOT NULL,
+    regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
+    description VARCHAR(1024),
+    priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
+    PRIMARY KEY (lcr_route_sid)
+    )`,
+    `CREATE TABLE lcr
+    (
+    lcr_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+    service_provider_sid CHAR(36),
+    account_sid CHAR(36),
+    PRIMARY KEY (lcr_sid)
+    )`,
+    `CREATE TABLE lcr_carrier_set_entry
+    (
+    lcr_carrier_set_entry_sid CHAR(36),
+    workload INTEGER NOT NULL DEFAULT 1 COMMENT 'represents a proportion of traffic to send through the associated carrier; can be used for load balancing traffic across carriers with a common priority for a destination',
+    lcr_route_sid CHAR(36) NOT NULL,
+    voip_carrier_sid CHAR(36) NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 0 COMMENT 'lower priority carriers are attempted first',
+    PRIMARY KEY (lcr_carrier_set_entry_sid)
+    )`,
+    'CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid)',
+    'ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid)',
+    'CREATE INDEX lcr_sid_idx ON lcr (lcr_sid)',
+    'ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid)',
+    'CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid)',
+    'CREATE INDEX account_sid_idx ON lcr (account_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY lcr_route_sid_idxfk (lcr_route_sid) REFERENCES lcr_routes (lcr_route_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_3 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid)',
+    'SET FOREIGN_KEY_CHECKS=1',
+  ],
+  '8004': [
+    'alter table accounts add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table accounts add column bucket_credential VARCHAR(8192)',
+    'alter table accounts add column record_format VARCHAR(16) NOT NULL DEFAULT \'mp3\'',
+    'alter table applications add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table phone_numbers DROP INDEX number',
+    'create unique index phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid)',
+    `CREATE TABLE clients
+    (
+    client_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    username VARCHAR(64),
+    password VARCHAR(1024),
+    PRIMARY KEY (client_sid)
+    )`,
+    'CREATE INDEX client_sid_idx ON clients (client_sid)',
+    'ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid)',
+    'ALTER TABLE sip_gateways ADD COLUMN protocol ENUM(\'udp\',\'tcp\',\'tls\', \'tls/srtp\') DEFAULT \'udp\''
+  ],
+  '8005': [
+    'DROP INDEX speech_credentials_idx_1 ON speech_credentials',
+    'ALTER TABLE speech_credentials ADD COLUMN label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN use_for_fallback_speech BOOLEAN DEFAULT false',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_language VARCHAR(12)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_voice VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_language VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE sip_gateways ADD COLUMN pad_crypto BOOLEAN NOT NULL DEFAULT 0',
+    'ALTER TABLE sip_gateways MODIFY port INTEGER',
+    `CREATE TABLE google_custom_voices
+    (
+    google_custom_voice_sid CHAR(36) NOT NULL UNIQUE ,
+    speech_credential_sid CHAR(36) NOT NULL,
+    model VARCHAR(512) NOT NULL,
+    reported_usage ENUM('REPORTED_USAGE_UNSPECIFIED','REALTIME','OFFLINE') DEFAULT 'REALTIME',
+    name VARCHAR(64) NOT NULL,
+    PRIMARY KEY (google_custom_voice_sid)
+    )
+    `,
+    'CREATE INDEX google_custom_voice_sid_idx ON google_custom_voices (google_custom_voice_sid)',
+    'CREATE INDEX speech_credential_sid_idx ON google_custom_voices (speech_credential_sid)',
+    'ALTER TABLE google_custom_voices ADD FOREIGN KEY speech_credential_sid_idxfk (speech_credential_sid) REFERENCES speech_credentials (speech_credential_sid) ON DELETE CASCADE',
+    'ALTER TABLE clients ADD COLUMN allow_direct_queue_calling BOOLEAN NOT NULL DEFAULT 1',
+    'ALTER TABLE clients ADD COLUMN allow_direct_user_calling BOOLEAN NOT NULL DEFAULT 1',
+    'ALTER TABLE clients ADD COLUMN allow_direct_app_calling BOOLEAN NOT NULL DEFAULT 1',
+  ],
+  9000: [
+    'ALTER TABLE sip_gateways ADD COLUMN send_options_ping BOOLEAN NOT NULL DEFAULT 0',
+    'ALTER TABLE applications MODIFY COLUMN speech_synthesis_voice VARCHAR(256)',
+    'ALTER TABLE applications MODIFY COLUMN fallback_speech_synthesis_voice VARCHAR(256)',
+    'ALTER TABLE sip_gateways ADD COLUMN use_sips_scheme BOOLEAN NOT NULL DEFAULT 0',
+  ],
+  9002: [
+    'ALTER TABLE system_information ADD COLUMN private_network_cidr VARCHAR(8192)',
+    'ALTER TABLE system_information ADD COLUMN log_level ENUM(\'info\', \'debug\') NOT NULL DEFAULT \'info\'',
+    'ALTER TABLE accounts ADD COLUMN enable_debug_log BOOLEAN NOT NULL DEFAULT false',
   ]
 };
 
@@ -116,6 +231,11 @@ const doIt = async() => {
         if (val < 7006) upgrades.push(...sql['7006']);
         if (val < 7007) upgrades.push(...sql['7007']);
         if (val < 8000) upgrades.push(...sql['8000']);
+        if (val < 8003) upgrades.push(...sql['8003']);
+        if (val < 8004) upgrades.push(...sql['8004']);
+        if (val < 8005) upgrades.push(...sql['8005']);
+        if (val < 9000) upgrades.push(...sql['9000']);
+        if (val < 9002) upgrades.push(...sql['9002']);
 
         // perform all upgrades
         logger.info({upgrades}, 'applying schema upgrades..');

db/webapp-tests.sql

@@ -2,7 +2,7 @@ SET FOREIGN_KEY_CHECKS=0;
 
 -- create one service provider
 insert into service_providers (service_provider_sid, name, description, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.us', 'jambonz.us service provider', 'sip.yakeeda.com');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.cloud', 'jambonz.cloud service provider', 'sip.yakeeda.com');
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
 
@@ -19,8 +19,8 @@ insert into sip_gateways (sip_gateway_sid, voip_carrier_sid, ipv4, port, inbound
 values ('46b727eb-c7dc-44fa-b063-96e48d408e4a', '5145b436-2f38-4029-8d4c-fd8c67831c7a', '3.3.3.3', 5060, 1, 1, 1);
 
 -- create the test application and test phone number
-insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.us/testCall', 'POST');
-insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.us/callStatus', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.cloud/testCall', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.cloud/callStatus', 'POST');
 insert into applications (application_sid, name, service_provider_sid, call_hook_sid, call_status_hook_sid, 
 speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 values ('7a489343-02ed-471e-8df0-fc5e1b98ce8f', 'Test application', '2708b1b3-2736-40ea-b502-c53d8396247f', 

lib/auth/index.js

@@ -1,41 +1,42 @@
 const Strategy = require('passport-http-bearer').Strategy;
 const {getMysqlConnection} = require('../db');
-const {hashString} = require('../utils/password-utils');
 const debug = require('debug')('jambonz:api-server');
+const {cacheClient} = require('../helpers');
 const jwt = require('jsonwebtoken');
 const sql = `
   SELECT *
   FROM api_keys
   WHERE api_keys.token = ?`;
 
-function makeStrategy(logger, retrieveKey) {
+function makeStrategy(logger) {
   return new Strategy(
     async function(token, done) {
-      //logger.debug(`validating with token ${token}`);
       jwt.verify(token, process.env.JWT_SECRET, async(err, decoded) => {
         if (err) {
           if (err.name === 'TokenExpiredError') {
             logger.debug('jwt expired');
             return done(null, false);
           }
-          /* its not a jwt obtained through login, check api leys */
+          /* its not a jwt obtained through login, check api keys */
           checkApiTokens(logger, token, done);
         }
         else {
-          /* validated -- make sure it is not on blacklist */
           try {
-            const s = `jwt:${hashString(token)}`;
-            const result = await retrieveKey(s);
-            if (result) {
-              debug(`result from searching for ${s}: ${result}`);
+            const {user_sid} = decoded;
+            /* Valid jwt tokens are stored in redis by hashed user_id */
+            const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+            const result = await cacheClient.get(redisKey);
+            if (result === null) {
+              debug(`result from searching for ${redisKey}: ${result}`);
               logger.info('jwt invalidated after logout');
               return done(null, false);
             }
-          } catch (err) {
+          } catch (error) {
             debug(err);
-            logger.info({err}, 'Error checking blacklist for jwt');
+            logger.info({err}, 'Error checking redis for jwt');
           }
-          const {user_sid, service_provider_sid, account_sid, email, name, scope, permissions} = decoded;
+          const { user_sid, service_provider_sid, account_sid, email, name, scope, permissions } = decoded;
+
           const user = {
             service_provider_sid,
             account_sid,
@@ -99,7 +100,7 @@ const checkApiTokens = (logger, token, done) => {
         hasServiceProviderAuth: scope === 'service_provider',
         hasAccountAuth: scope === 'account'
       };
-      logger.info(user, `successfully validated with scope ${scope}`);
+      logger.debug({user}, `successfully validated with scope ${scope}`);
       return done(null, user, {scope});
     });
   });

lib/helpers/cache-client.js

@@ -0,0 +1,82 @@
+const {
+  addKey: addKeyRedis,
+  deleteKey: deleteKeyRedis,
+  retrieveKey: retrieveKeyRedis,
+} = require('./realtimedb-helpers');
+const { hashString } = require('../utils/password-utils');
+const logger = require('../logger');
+
+class CacheClient {
+  constructor() { }
+
+  async set(params) {
+    const {
+      redisKey,
+      value = '1',
+      time = 3600,
+    } = params || {};
+
+    try {
+      await addKeyRedis(redisKey, value, time);
+
+    } catch (err) {
+      logger.error('CacheClient.get set', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        ...params
+      });
+    }
+  }
+
+  async get(redisKey) {
+    try {
+      const result = await retrieveKeyRedis(redisKey);
+      return result;
+    } catch (err) {
+      logger.error('CacheClient.get error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        redisKey
+      });
+    }
+  }
+
+
+  async delete(key) {
+    try {
+      await deleteKeyRedis(key);
+      logger.debug('CacheClient.delete key from redis', { key });
+    } catch (err) {
+      logger.error('CacheClient.delete error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        key
+      });
+    }
+  }
+
+  generateRedisKey(type, key, version) {
+    let suffix = '';
+    if (version) {
+      suffix = `:version:${version}`;
+    }
+
+    switch (type) {
+      case 'reset-link':
+        return `reset-link:${key}`;
+      case 'jwt':
+      default:
+        return `jwt:${hashString(key)}${suffix}`;
+    }
+  }
+}
+
+const cacheClient = new CacheClient();
+
+module.exports = { cacheClient };

lib/helpers/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require('./cache-client'),
+  ...require('./realtimedb-helpers'),
+};

lib/helpers/realtimedb-helpers.js

@@ -0,0 +1,33 @@
+const logger = require('../logger');
+
+const {
+  client,
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  incrKey,
+  client: redisClient,
+  listConferences
+} = require('@jambonz/realtimedb-helpers')({}, logger);
+
+module.exports = {
+  client,
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  redisClient,
+  incrKey,
+  listConferences
+};

lib/logger.js

@@ -0,0 +1,7 @@
+const opts = {
+  level: process.env.JAMBONES_LOGLEVEL || 'info'
+};
+const pino = require('pino');
+const logger = pino(opts, pino.destination(1, {sync: false}));
+
+module.exports = logger;

lib/middleware.js

@@ -0,0 +1,32 @@
+const logger = require('./logger');
+
+function delayLoginMiddleware(req, res, next) {
+  if (req.path.includes('/login') || req.path.includes('/signin')) {
+    const min = 200;
+    const max = 1000;
+    /* Random delay between 200 - 1000ms */
+    const sendStatusDelay = Math.floor(Math.random() * (max - min + 1)) + min;
+
+    /* the res.json take longer, we decrease the max delay slightly to 0-800ms */
+    const jsonDelay = Math.floor(Math.random() * 800);
+    logger.debug(`delayLoginMiddleware: sendStatus ${sendStatusDelay} - json ${jsonDelay}`);
+    const sendStatus = res.sendStatus;
+    const json = res.json;
+
+    res.sendStatus = function(status) {
+      setTimeout(() => {
+        sendStatus.call(res, status);
+      }, sendStatusDelay);
+    };
+    res.json = function(body) {
+      setTimeout(() => {
+        json.call(res, body);
+      }, jsonDelay);
+    };
+  }
+  next();
+}
+
+module.exports = {
+  delayLoginMiddleware
+};

lib/models/account.js

@@ -4,7 +4,7 @@ const {getMysqlConnection} = require('../db');
 const {promisePool} = require('../db');
 const { v4: uuid } = require('uuid');
 
-const {encrypt} = require('../utils/encrypt-decrypt');
+const {encrypt, decrypt} = require('../utils/encrypt-decrypt');
 
 const retrieveSql = `SELECT * from accounts acc
 LEFT JOIN webhooks AS rh
@@ -34,7 +34,7 @@ AND effective_end_date IS NULL
 AND pending=0`;
 
 const updatePaymentInfoSql = `UPDATE account_subscriptions 
-SET last4 = ?, exp_month = ?, exp_year = ?, card_type = ? 
+SET last4 = ?, stripe_payment_method_id=?, exp_month = ?, exp_year = ?, card_type = ? 
 WHERE account_sid = ? 
 AND effective_end_date IS NULL`;
 
@@ -55,6 +55,13 @@ WHERE account_sid = ?
 AND effective_end_date IS NULL 
 AND pending = 0`;
 
+const extractBucketCredential = (obj) => {
+  const {bucket_credential} = obj;
+  if (bucket_credential) {
+    obj.bucket_credential = JSON.parse(decrypt(bucket_credential));
+  }
+};
+
 function transmogrifyResults(results) {
   return results.map((row) => {
     const obj = row.acc;
@@ -75,6 +82,8 @@ function transmogrifyResults(results) {
     else obj.queue_event_hook = null;
     delete obj.queue_event_hook_sid;
 
+    extractBucketCredential(obj);
+
     return obj;
   });
 }
@@ -190,17 +199,18 @@ class Account extends Model {
     debug(r3, 'Account.activateSubscription - replaced old subscription');
 
     /* update account.plan to paid, if it isnt already */
+    /* update account.is_active to 1, if account is deactivated */
     await promisePool.execute(
-      'UPDATE accounts SET plan_type = \'paid\' WHERE account_sid = ?',
+      'UPDATE accounts SET plan_type = \'paid\', is_active = 1 WHERE account_sid = ?',
       [account_sid]);
     return true;
   }
 
   static async updatePaymentInfo(logger, account_sid, pm) {
-    const {card} = pm;
+    const {id, card} = pm;
     const last4_encrypted = encrypt(card.last4);
     await promisePool.execute(updatePaymentInfoSql,
-      [last4_encrypted, card.exp_month, card.exp_year, card.brand, account_sid]);
+      [last4_encrypted, id, card.exp_month, card.exp_year, card.brand, account_sid]);
   }
 
   static async provisionPendingSubscription(logger, account_sid, products, payment_method, subscription_id) {
@@ -238,7 +248,6 @@ class Account extends Model {
     }));
     return account_subscription_sid;
   }
-
 }
 
 Account.table = 'accounts';
@@ -318,6 +327,18 @@ Account.fields = [
     name: 'siprec_hook_sid',
     type: 'string',
   },
+  {
+    name: 'record_all_calls',
+    type: 'number'
+  },
+  {
+    name: 'record_format',
+    type: 'string'
+  },
+  {
+    name: 'bucket_credential',
+    type: 'string'
+  }
 ];
 
 module.exports = Account;

lib/models/application.js

@@ -120,6 +120,10 @@ Application.fields = [
   {
     name: 'messaging_hook_sid',
     type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number',
   }
 ];
 

lib/models/client.js

@@ -0,0 +1,70 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Client extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(service_provider_sid) {
+    const sql = `SELECT c.client_sid, c.account_sid, c.is_active, c.username, c.hashed_password
+      FROM ${this.table} AS c LEFT JOIN accounts AS acc ON c.account_sid = acc.account_sid
+      LEFT JOIN service_providers AS sp ON sp.service_provider_sid = accs.service_provider_sid
+      WHERE sp.service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, service_provider_sid);
+    return rows;
+  }
+
+  static async retrieveByAccountSidAndUserName(account_sid, username) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ? AND username = ?`;
+    const [rows] = await promisePool.query(sql, [account_sid, username]);
+    return rows;
+  }
+}
+
+Client.table = 'clients';
+Client.fields = [
+  {
+    name: 'client_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+  {
+    name: 'username',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'password',
+    type: 'string'
+  },
+  {
+    name: 'allow_direct_app_calling',
+    type: 'number'
+  },
+  {
+    name: 'allow_direct_queue_calling',
+    type: 'number'
+  },
+  {
+    name: 'allow_direct_user_calling',
+    type: 'number'
+  }
+];
+
+module.exports = Client;

lib/models/google-custom-voice.js

@@ -0,0 +1,61 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class GoogleCustomVoice extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllBySpeechCredentialSid(speech_credential_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE speech_credential_sid = ?`;
+    const [rows] = await promisePool.query(sql, speech_credential_sid);
+    return rows;
+  }
+
+  static async deleteAllBySpeechCredentialSid(speech_credential_sid) {
+    const sql = `DELETE FROM ${this.table} WHERE speech_credential_sid = ?`;
+    const [rows] = await promisePool.query(sql, speech_credential_sid);
+    return rows;
+  }
+
+  static async retrieveAllByLabel(service_provider_sid, account_sid, label) {
+    let sql;
+    if (account_sid) {
+      sql = `SELECT gcv.* FROM ${this.table} gcv 
+LEFT JOIN speech_credentials sc ON gcv.speech_credential_sid = sc.speech_credential_sid 
+WHERE sc.account_sid = ? OR (sc.account_sid is NULL && sc.service_provider_sid = ?) 
+${label ? 'AND label = ?' : 'AND label is NULL'}`;
+    } else {
+      sql = `SELECT gcv.* FROM ${this.table} gcv 
+LEFT JOIN speech_credentials sc ON gcv.speech_credential_sid = sc.speech_credential_sid 
+WHERE sc.service_provider_sid = ? ${label ? 'AND label = ?' : 'AND label is NULL'}`;
+    }
+    const [rows] = await promisePool.query(sql, [...(account_sid ?
+      [account_sid, service_provider_sid] : [service_provider_sid]), label]);
+    return rows;
+  }
+}
+GoogleCustomVoice.table = 'google_custom_voices';
+GoogleCustomVoice.fields = [
+  {
+    name: 'google_custom_voice_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'model',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'reported_usage',
+    type: 'number'
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  }
+];
+
+module.exports = GoogleCustomVoice;

lib/models/lcr-carrier-set-entry.js

@@ -0,0 +1,47 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class LcrCarrierSetEntry extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrRouteSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_route_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrRouteSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_route_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+}
+
+LcrCarrierSetEntry.table = 'lcr_carrier_set_entry';
+LcrCarrierSetEntry.fields = [
+  {
+    name: 'lcr_carrier_set_entry_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'workload',
+    type: 'number'
+  },
+  {
+    name: 'lcr_route_sid',
+    type: 'string'
+  },
+  {
+    name: 'voip_carrier_sid',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrCarrierSetEntry;

lib/models/lcr-route.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+
+class LcrRoutes extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+
+  static async countAllByLcrSid(sid) {
+    const sql = `SELECT COUNT(*) AS count FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.length ? rows[0].count : 0;
+  }
+}
+
+LcrRoutes.table = 'lcr_routes';
+LcrRoutes.fields = [
+  {
+    name: 'lcr_route_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
+  },
+  {
+    name: 'regex',
+    type: 'string'
+  },
+  {
+    name: 'description',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrRoutes;

lib/models/lcr.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Lcr extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async releaseDefaultEntry(sid) {
+    const sql = `UPDATE ${this.table} SET default_carrier_set_entry_sid = null WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+}
+
+Lcr.table = 'lcr';
+Lcr.fields = [
+  {
+    name: 'lcr_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  },
+  {
+    name: 'default_carrier_set_entry_sid',
+    type: 'string'
+  }
+];
+
+module.exports = Lcr;

lib/models/model.js

@@ -107,7 +107,7 @@ class Model extends Emitter {
       if (pk.name in obj) throw new DbErrorBadRequest(`primary key ${pk.name} is immutable`);
       getMysqlConnection((err, conn) => {
         if (err) return reject(err);
-        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = '${sid}'`, obj, (err, results, fields) => {
+        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = ?`, [obj, sid], (err, results, fields) => {
           conn.release();
           if (err) return reject(err);
           resolve(results.affectedRows);

lib/models/phone-number.js

@@ -1,6 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
-const sql = 'SELECT * from phone_numbers WHERE account_sid = ?';
+const sqlRetrieveAll = 'SELECT * from phone_numbers WHERE account_sid = ? ORDER BY number';
+const sqlRetrieveOne = 'SELECT * from phone_numbers WHERE phone_number_sid = ? AND account_sid = ? ORDER BY number';
 const sqlSP = `SELECT * 
 FROM phone_numbers 
 WHERE account_sid IN 
@@ -8,7 +9,7 @@ WHERE account_sid IN
   SELECT account_sid 
   FROM accounts 
   WHERE service_provider_sid = ?
-)`;
+) ORDER BY number`;
 
 class PhoneNumber extends Model {
   constructor() {
@@ -16,8 +17,8 @@ class PhoneNumber extends Model {
   }
 
   static async retrieveAll(account_sid) {
-    if (!account_sid) return super.retrieveAll();
-    const [rows] = await promisePool.query(sql, account_sid);
+    if (!account_sid) return await super.retrieveAll();
+    const [rows] = await promisePool.query(sqlRetrieveAll, account_sid);
     return rows;
   }
   static async retrieveAllForSP(service_provider_sid) {
@@ -30,7 +31,7 @@ class PhoneNumber extends Model {
    */
   static async retrieve(sid, account_sid) {
     if (!account_sid) return super.retrieve(sid);
-    const [rows] = await promisePool.query(`${sql} AND phone_number_sid = ?`, [account_sid, sid]);
+    const [rows] = await promisePool.query(sqlRetrieveOne, [sid, account_sid]);
     return rows;
   }
 }

lib/models/service-provider.js

@@ -89,6 +89,10 @@ ServiceProvider.fields = [
   {
     name: 'ms_teams_fqdn',
     type: 'string',
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
   }
 
 ];

lib/models/sip-gateway.js

@@ -51,6 +51,10 @@ SipGateway.fields = [
     name: 'is_active',
     type: 'number'
   },
+  {
+    name: 'pad_crypto',
+    type: 'number'
+  },
   {
     name: 'account_sid',
     type: 'string'
@@ -58,6 +62,10 @@ SipGateway.fields = [
   {
     name: 'application_sid',
     type: 'string'
+  },
+  {
+    name: 'protocol',
+    type: 'string'
   }
 ];
 

lib/models/speech-credential.js

@@ -1,7 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
 const retrieveSql = 'SELECT * from speech_credentials WHERE account_sid = ?';
-const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ?';
+const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ? and account_sid is null';
 
 class SpeechCredential extends Model {
   constructor() {
@@ -20,6 +20,19 @@ class SpeechCredential extends Model {
     return rows;
   }
 
+  static async getSpeechCredentialsByVendorAndLabel(service_provider_sid, account_sid, vendor, label) {
+    let sql;
+    if (account_sid) {
+      sql = `SELECT * FROM speech_credentials WHERE account_sid = ? AND vendor = ?
+        AND label ${label ? '= ?' : 'is NULL'}`;
+    } else {
+      sql = `SELECT * FROM speech_credentials WHERE service_provider_sid = ? AND vendor = ?
+        AND label ${label ? '= ?' : 'is NULL'}`;
+    }
+    const [rows] = await promisePool.query(sql, [account_sid ? account_sid : service_provider_sid, vendor, label]);
+    return rows;
+  }
+
   static async disableStt(account_sid) {
     await promisePool.execute('UPDATE speech_credentials SET use_for_stt = 0 WHERE account_sid = ?', [account_sid]);
   }
@@ -86,6 +99,10 @@ SpeechCredential.fields = [
   {
     name: 'last_tested',
     type: 'date'
+  },
+  {
+    name: 'label',
+    type: 'string'
   }
 ];
 

lib/models/system-information.js

@@ -0,0 +1,42 @@
+const Model = require('./model');
+const { promisePool } = require('../db');
+class SystemInformation extends Model {
+  constructor() {
+    super();
+  }
+
+  static async add(body) {
+    let [sysInfo] = await this.retrieveAll();
+    if (sysInfo) {
+      const sql = `UPDATE ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    } else {
+      const sql = `INSERT INTO ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    }
+    [sysInfo] = await this.retrieveAll();
+    return sysInfo;
+  }
+}
+
+SystemInformation.table = 'system_information';
+SystemInformation.fields = [
+  {
+    name: 'domain_name',
+    type: 'string',
+  },
+  {
+    name: 'sip_domain_name',
+    type: 'string',
+  },
+  {
+    name: 'monitoring_domain_name',
+    type: 'string',
+  },
+  {
+    name: 'private_network_cidr',
+    type: 'string',
+  },
+];
+
+module.exports = SystemInformation;

lib/models/voip-carrier.js

@@ -11,10 +11,16 @@ class VoipCarrier extends Model {
   static async retrieveAll(account_sid) {
     if (!account_sid) return super.retrieveAll();
     const [rows] = await promisePool.query(retrieveSql, account_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
     return rows;
   }
   static async retrieveAllForSP(service_provider_sid) {
     const [rows] = await promisePool.query(retrieveSqlForSP, service_provider_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
     return rows;
   }
 }
@@ -55,6 +61,10 @@ VoipCarrier.fields = [
     name: 'requires_register',
     type: 'number'
   },
+  {
+    name: 'register_use_tls',
+    type: 'number'
+  },
   {
     name: 'register_username',
     type: 'string'
@@ -122,6 +132,10 @@ VoipCarrier.fields = [
   {
     name: 'register_public_ip_in_contact',
     type: 'number'
+  },
+  {
+    name: 'register_status',
+    type: 'string'
   }
 ];
 

lib/record/azure-storage.js

@@ -0,0 +1,71 @@
+const { Writable } = require('stream');
+const { BlobServiceClient } = require('@azure/storage-blob');
+const { v4: uuidv4 } = require('uuid');
+const streamBuffers = require('stream-buffers');
+
+class AzureStorageUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+    this.blockBlobClient = blobServiceClient.getContainerClient(opts.bucketName).getBlockBlobClient(opts.Key);
+
+    this.metadata = opts.metadata;
+    this.blocks = [];
+
+    this.bufferSize = 2 * 1024 * 1024; // Buffer size set to 2MB
+    this.buffer = new streamBuffers.WritableStreamBuffer({
+      initialSize: this.bufferSize,
+      incrementAmount: this.bufferSize
+    });
+  }
+
+  async _write(chunk, encoding, callback) {
+    this.buffer.write(chunk, encoding);
+
+    if (this.buffer.size() >= this.bufferSize) {
+      const blockID = uuidv4().replace(/-/g, '');
+      this.blocks.push(blockID);
+
+      try {
+        const dataToWrite = this.buffer.getContents();
+        await this.blockBlobClient.stageBlock(blockID, dataToWrite, dataToWrite.length);
+        callback();
+      } catch (error) {
+        callback(error);
+      }
+    } else {
+      callback();
+    }
+  }
+
+  async _final(callback) {
+    // Write any remaining data in buffer
+    if (this.buffer.size() > 0) {
+      const remainingData = this.buffer.getContents();
+      const blockID = uuidv4().replace(/-/g, '');
+      this.blocks.push(blockID);
+
+      try {
+        await this.blockBlobClient.stageBlock(blockID, remainingData, remainingData.length);
+      } catch (error) {
+        callback(error);
+        return;
+      }
+    }
+
+    try {
+      await this.blockBlobClient.commitBlockList(this.blocks);
+      // remove all null/undefined props
+      const filteredObj = Object.entries(this.metadata).reduce((acc, [key, val]) => {
+        if (val !== undefined && val !== null) acc[key] = val;
+        return acc;
+      }, {});
+      await this.blockBlobClient.setMetadata(filteredObj);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = AzureStorageUploadStream;

lib/record/encoder.js

@@ -0,0 +1,61 @@
+const { Transform } = require('stream');
+const lamejs = require('@jambonz/lamejs');
+
+class PCMToMP3Encoder extends Transform {
+  constructor(options, logger) {
+    super(options);
+
+    const channels = options.channels || 1;
+    const sampleRate = options.sampleRate || 8000;
+    const bitRate = options.bitRate || 128;
+
+    this.encoder = new lamejs.Mp3Encoder(channels, sampleRate, bitRate);
+    this.channels = channels;
+    this.logger = logger;
+  }
+
+  _transform(chunk, encoding, callback) {
+    try {
+      // Convert chunk buffer into Int16Array for lamejs
+      const samples = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
+
+      // Split input samples into left and right channel arrays if stereo
+      let leftChannel, rightChannel;
+      if (this.channels === 2) {
+        leftChannel = new Int16Array(samples.length / 2);
+        rightChannel = new Int16Array(samples.length / 2);
+
+        for (let i = 0; i < samples.length; i += 2) {
+          leftChannel[i / 2] = samples[i];
+          rightChannel[i / 2] = samples[i + 1];
+        }
+      } else {
+        leftChannel = samples;
+      }
+
+      // Encode the input data
+      const mp3Data = this.encoder.encodeBuffer(leftChannel, rightChannel);
+
+      if (mp3Data.length > 0) {
+        this.push(Buffer.from(mp3Data));
+      }
+      callback();
+    } catch (err) {
+      this.logger.error(
+        { err },
+        'Error while mp3 transform');
+    }
+  }
+
+  _flush(callback) {
+    // Finalize encoding and flush the internal buffers
+    const mp3Data = this.encoder.flush();
+
+    if (mp3Data.length > 0) {
+      this.push(Buffer.from(mp3Data));
+    }
+    callback();
+  }
+}
+
+module.exports = PCMToMP3Encoder;

lib/record/google-storage.js

@@ -0,0 +1,62 @@
+const { Storage } = require('@google-cloud/storage');
+const { Writable } = require('stream');
+const streamBuffers = require('stream-buffers');
+
+class GoogleStorageUploadStream extends Writable {
+
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.metadata = opts.metadata;
+
+    const storage = new Storage(opts.bucketCredential);
+    this.gcsFile = storage.bucket(opts.bucketName).file(opts.Key);
+    this.writeStream = this.gcsFile.createWriteStream();
+
+    this.bufferSize = 2 * 1024 * 1024; // Buffer size set to 2MB
+    this.buffer = new streamBuffers.WritableStreamBuffer({
+      initialSize: this.bufferSize,
+      incrementAmount: this.bufferSize
+    });
+
+    this.writeStream.on('error', (err) => this.logger.error(err));
+    this.writeStream.on('finish', () => {
+      this.logger.info('Google storage Upload completed.');
+      this._addMetadata();
+    });
+  }
+
+  _write(chunk, encoding, callback) {
+    this.buffer.write(chunk, encoding);
+
+    // Write to GCS when buffer reaches desired size
+    if (this.buffer.size() >= this.bufferSize) {
+      const dataToWrite = this.buffer.getContents();
+      this.writeStream.write(dataToWrite, callback);
+    } else {
+      callback();
+    }
+  }
+
+  _final(callback) {
+    // Write any remaining data in the buffer to GCS
+    if (this.buffer.size() > 0) {
+      const remainingData = this.buffer.getContents();
+      this.writeStream.write(remainingData);
+    }
+
+    this.writeStream.end();
+    this.writeStream.once('finish', callback);
+  }
+
+  async _addMetadata() {
+    try {
+      await this.gcsFile.setMetadata({metadata: this.metadata});
+      this.logger.info('Google storage Upload and metadata setting completed.');
+    } catch (err) {
+      this.logger.error(err, 'Google storage An error occurred while setting metadata');
+    }
+  }
+}
+
+module.exports = GoogleStorageUploadStream;

lib/record/index.js

@@ -0,0 +1,6 @@
+
+async function record(logger, socket) {
+  return require('./upload')(logger, socket);
+}
+
+module.exports = record;

lib/record/s3-multipart-upload-stream.js

@@ -0,0 +1,103 @@
+const { Writable } = require('stream');
+const {
+  S3Client,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+} = require('@aws-sdk/client-s3');
+
+class S3MultipartUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.bucketName = opts.bucketName;
+    this.objectKey = opts.Key;
+    this.uploadId = null;
+    this.partNumber = 1;
+    this.multipartETags = [];
+    this.buffer = Buffer.alloc(0);
+    this.minPartSize = 5 * 1024 * 1024; // 5 MB
+    this.s3 = new S3Client(opts.bucketCredential);
+    this.metadata = opts.metadata;
+  }
+
+  async _initMultipartUpload() {
+    const command = new CreateMultipartUploadCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      Metadata: this.metadata
+    });
+    const response = await this.s3.send(command);
+    return response.UploadId;
+  }
+
+  async _uploadBuffer() {
+    const uploadPartCommand = new UploadPartCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      PartNumber: this.partNumber,
+      UploadId: this.uploadId,
+      Body: this.buffer,
+    });
+
+    const uploadPartResponse = await this.s3.send(uploadPartCommand);
+    this.multipartETags.push({
+      ETag: uploadPartResponse.ETag,
+      PartNumber: this.partNumber,
+    });
+    this.partNumber += 1;
+  }
+
+  async _write(chunk, encoding, callback) {
+    try {
+      if (!this.uploadId) {
+        this.uploadId = await this._initMultipartUpload();
+      }
+
+      this.buffer = Buffer.concat([this.buffer, chunk]);
+
+      if (this.buffer.length >= this.minPartSize) {
+        await this._uploadBuffer();
+        this.buffer = Buffer.alloc(0);
+      }
+
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _finalize(err) {
+    try {
+      if (this.buffer.length > 0) {
+        await this._uploadBuffer();
+      }
+
+      const completeMultipartUploadCommand = new CompleteMultipartUploadCommand({
+        Bucket: this.bucketName,
+        Key: this.objectKey,
+        MultipartUpload: {
+          Parts: this.multipartETags.sort((a, b) => a.PartNumber - b.PartNumber),
+        },
+        UploadId: this.uploadId,
+      });
+
+      await this.s3.send(completeMultipartUploadCommand);
+      this.logger.info('Finished upload to S3');
+    } catch (error) {
+      this.logger.error('Error completing multipart upload:', error);
+      throw error;
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this._finalize();
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = S3MultipartUploadStream;

lib/record/upload.js

@@ -0,0 +1,97 @@
+const Account = require('../models/account');
+const Websocket = require('ws');
+const PCMToMP3Encoder = require('./encoder');
+const wav = require('wav');
+const { getUploader } = require('./utils');
+const { pipeline } = require('stream');
+
+async function upload(logger, socket) {
+  socket._recvInitialMetadata = false;
+  socket.on('message', async function(data, isBinary) {
+    try {
+      if (!isBinary && !socket._recvInitialMetadata) {
+        socket._recvInitialMetadata = true;
+        logger.debug(`initial metadata: ${data}`);
+        const obj = JSON.parse(data.toString());
+        logger.info({ obj }, 'received JSON message from jambonz');
+        const { sampleRate, accountSid, callSid, direction, from, to,
+          callId, applicationSid, originatingSipIp, originatingSipTrunkName } = obj;
+        const account = await Account.retrieve(accountSid);
+        if (account && account.length && account[0].bucket_credential) {
+          const obj = account[0].bucket_credential;
+          // add tags to metadata
+          const metadata = {
+            accountSid,
+            callSid,
+            direction,
+            from,
+            to,
+            callId,
+            applicationSid,
+            originatingSipIp,
+            originatingSipTrunkName,
+            sampleRate: `${sampleRate}`
+          };
+          if (obj.tags && obj.tags.length) {
+            obj.tags.forEach((tag) => {
+              metadata[tag.Key] = tag.Value;
+            });
+          }
+          // create S3 path
+          const day = new Date();
+          let key = `${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
+          key += `/${day.getDate().toString().padStart(2, '0')}/${callSid}.${account[0].record_format}`;
+
+          // Uploader
+          const uploadStream = getUploader(key, metadata, obj, logger);
+          if (!uploadStream) {
+            logger.info('There is no available record uploader, close the socket.');
+            socket.close();
+          }
+
+          /**encoder */
+          let encoder;
+          if (account[0].record_format === 'wav') {
+            encoder = new wav.Writer({ channels: 2, sampleRate, bitDepth: 16 });
+          } else {
+            // default is mp3
+            encoder = new PCMToMP3Encoder({
+              channels: 2,
+              sampleRate: sampleRate,
+              bitrate: 128
+            }, logger);
+          }
+
+          /* start streaming data */
+          pipeline(
+            Websocket.createWebSocketStream(socket),
+            encoder,
+            uploadStream,
+            (error) => {
+              if (error) {
+                logger.error({ error }, 'pipeline error, cannot upload data to storage');
+                socket.close();
+              }
+            }
+          );
+        } else {
+          logger.info(`account ${accountSid} does not have any bucket credential, close the socket`);
+          socket.close();
+        }
+      }
+    } catch (err) {
+      logger.error({ err, data }, 'error parsing message during connection');
+    }
+  });
+  socket.on('error', function(err) {
+    logger.error({ err }, 'record upload: error');
+  });
+  socket.on('close', (data) => {
+    logger.info({ data }, 'record upload: close');
+  });
+  socket.on('end', function(err) {
+    logger.error({ err }, 'record upload: socket closed from jambonz');
+  });
+}
+
+module.exports = upload;

lib/record/utils.js

@@ -0,0 +1,58 @@
+const AzureStorageUploadStream = require('./azure-storage');
+const GoogleStorageUploadStream = require('./google-storage');
+const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
+
+const getUploader = (key, metadata, bucket_credential, logger) => {
+  const uploaderOpts = {
+    bucketName: bucket_credential.name,
+    Key: key,
+    metadata
+  };
+  try {
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+        uploaderOpts.bucketCredential = {
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: bucket_credential.region || 'us-east-1'
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 's3_compatible':
+        uploaderOpts.bucketCredential = {
+          endpoint: bucket_credential.endpoint,
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: 'us-east-1',
+          forcePathStyle: true
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 'google':
+        const serviceKey = JSON.parse(bucket_credential.service_key);
+        uploaderOpts.bucketCredential = {
+          projectId: serviceKey.project_id,
+          credentials: {
+            client_email: serviceKey.client_email,
+            private_key: serviceKey.private_key
+          }
+        };
+        return new GoogleStorageUploadStream(logger, uploaderOpts);
+      case 'azure':
+        uploaderOpts.connection_string = bucket_credential.connection_string;
+        return new AzureStorageUploadStream(logger, uploaderOpts);
+      default:
+        logger.error(`unknown bucket vendor: ${bucket_credential.vendor}`);
+        break;
+    }
+  } catch (err) {
+    logger.error(`Error creating uploader, vendor: ${bucket_credential.vendor}, reason: ${err.message}`);
+  }
+  return null;
+};
+
+module.exports = {
+  getUploader
+};

lib/routes/api/accounts.js

@@ -1,4 +1,5 @@
 const router = require('express').Router();
+const assert = require('assert');
 const request = require('request');
 const {DbErrorBadRequest, DbErrorForbidden, DbErrorUnprocessableRequest} = require('../../utils/errors');
 const Account = require('../../models/account');
@@ -12,9 +13,18 @@ const { v4: uuidv4 } = require('uuid');
 const snakeCase = require('../../utils/snake-case');
 const sysError = require('../error');
 const {promisePool} = require('../../db');
-const {hasAccountPermissions, parseAccountSid, enableSubspace, disableSubspace} = require('./utils');
+const {
+  hasAccountPermissions,
+  parseAccountSid,
+  parseCallSid,
+  enableSubspace,
+  disableSubspace,
+  parseVoipCarrierSid
+} = require('./utils');
 const short = require('short-uuid');
 const VoipCarrier = require('../../models/voip-carrier');
+const { encrypt } = require('../../utils/encrypt-decrypt');
+const { testS3Storage, testGoogleStorage, testAzureStorage } = require('../../utils/storage-utils');
 const translator = short();
 
 let idx = 0;
@@ -31,32 +41,41 @@ const getFsUrl = async(logger, retrieveSet, setName) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/createCall`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/createCall`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
-};
+const validateRequest = async(req, account_sid) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return;
+    }
 
-const validateUpdateForCarrier = async(req) => {
-  const account_sid = parseAccountSid(req);
-  if (req.user.hasScope('admin')) return ;
-  if (req.user.hasScope('account')) {
-    if (account_sid === req.user.account_sid) return ;
-    throw new DbErrorForbidden('insufficient permissions to update account');
-  }
-  if (req.user.hasScope('service_provider')) {
-    const [r] = await promisePool.execute(
-      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]);
-    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) return;
-    throw new DbErrorForbidden('insufficient permissions to update account');
+    if (req.user.hasScope('account')) {
+      if (account_sid === req.user.account_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+      );
+
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  } catch (error) {
+    throw error;
   }
 };
 
@@ -67,10 +86,12 @@ router.use('/:sid/Charges', hasAccountPermissions, require('./charges'));
 router.use('/:sid/SipRealms', hasAccountPermissions, require('./sip-realm'));
 router.use('/:sid/PredefinedCarriers', hasAccountPermissions, require('./add-from-predefined-carrier'));
 router.use('/:sid/Limits', hasAccountPermissions, require('./limits'));
+router.use('/:sid/TtsCache', hasAccountPermissions, require('./tts-cache'));
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const results = await Application.retrieveAll(null, account_sid);
     res.status(200).json(results);
   } catch (err) {
@@ -81,6 +102,7 @@ router.get('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const results = await VoipCarrier.retrieveAll(account_sid);
     res.status(200).json(results);
   } catch (err) {
@@ -89,13 +111,18 @@ router.get('/:sid/VoipCarriers', async(req, res) => {
 });
 router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
   const logger = req.app.locals.logger;
+
   try {
-    await validateUpdateForCarrier(req);
-    const rowsAffected = await VoipCarrier.update(req.params.voip_carrier_sid, req.body);
+    const sid = parseVoipCarrierSid(req);
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
+    const rowsAffected = await VoipCarrier.update(sid, req.body);
     if (rowsAffected === 0) {
       return res.sendStatus(404);
     }
-    res.status(204).end();
+
+    return res.status(204).end();
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -106,6 +133,8 @@ router.post('/:sid/VoipCarriers', async(req, res) => {
   const payload = req.body;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
     logger.debug({payload}, 'POST /:sid/VoipCarriers');
     const uuid = await VoipCarrier.make({
       account_sid,
@@ -117,6 +146,88 @@ router.post('/:sid/VoipCarriers', async(req, res) => {
   }
 });
 
+router.get('/:sid/RegisteredSipUsers', async(req, res) => {
+  const {logger, registrar} = req.app.locals;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const result = await Account.retrieve(account_sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${account_sid}`);
+    }
+    if (!result[0].sip_realm) {
+      throw new DbErrorBadRequest('account does not have sip_realm configuration');
+    }
+    const users = await registrar.getRegisteredUsersForRealm(result[0].sip_realm);
+    res.status(200).json(users.map((u) => `${u}@${result[0].sip_realm}`));
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.post('/:sid/RegisteredSipUsers', async(req, res) => {
+  const {logger, registrar} = req.app.locals;
+  const users = req.body;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const result = await Account.retrieve(account_sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${account_sid}`);
+    }
+    if (!result[0].sip_realm) {
+      throw new DbErrorBadRequest('account does not have sip_realm configuration');
+    }
+    if (!users || !Array.isArray(users) || users.length === 0) {
+      return res.status(200).json(await registrar.getRegisteredUsersDetailsForRealm(result[0].sip_realm));
+    }
+    const ret = [];
+    for (const u of users) {
+      const user = await registrar.query(`${u}@${result[0].sip_realm}`) || {
+        name: u,
+        contact:  null,
+        expiryTime:  0,
+        protocol: null
+      };
+      ret.push({
+        name: u,
+        ...user,
+        registered_status: user.expiryTime > 0 ? 'active' : 'inactive',
+      });
+    }
+    res.status(200).json(ret);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid/RegisteredSipUsers/:client', async(req, res) => {
+  const {logger, registrar, lookupClientByAccountAndUsername} = req.app.locals;
+  const client = req.params.client;
+  try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const result = await Account.retrieve(account_sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${account_sid}`);
+    }
+    const user = await registrar.query(`${client}@${result[0].sip_realm}`);
+    const [clientDb] = await lookupClientByAccountAndUsername(account_sid, client);
+    res.status(200).json({
+      name: client,
+      contact: user ? user.contact : null,
+      expiryTime: user ? user.expiryTime : 0,
+      protocol: user ? user.protocol : null,
+      allow_direct_app_calling: clientDb ? clientDb.allow_direct_app_calling : 0,
+      allow_direct_queue_calling: clientDb ? clientDb.allow_direct_queue_calling : 0,
+      allow_direct_user_calling: clientDb ? clientDb.allow_direct_user_calling : 0,
+      registered_status: user ? 'active' : 'inactive'
+    });
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 function coerceNumbers(callInfo) {
   if (Array.isArray(callInfo)) {
     return callInfo.map((ci) => {
@@ -147,11 +258,15 @@ function validateUpdateCall(opts) {
     'child_call_hook',
     'call_status',
     'listen_status',
+    'transcribe_status',
     'conf_hold_status',
     'conf_mute_status',
     'mute_status',
     'sip_request',
-    'record'
+    'record',
+    'tag',
+    'dtmf',
+    'conferenceParticipantAction'
   ]
     .reduce((acc, prop) => (opts[prop] ? ++acc : acc), 0);
 
@@ -187,15 +302,34 @@ function validateUpdateCall(opts) {
     throw new DbErrorBadRequest('invalid conf_mute_status');
   }
   if (opts.sip_request &&
-    (!opts.sip_request.method && !opts.sip_request.content_type ||  !opts.sip_request.content_type)) {
-    throw new DbErrorBadRequest('sip_request requires content_type and content properties');
+    (!opts.sip_request.method || !opts.sip_request.content_type ||  !opts.sip_request.content)) {
+    throw new DbErrorBadRequest('sip_request requires method, content_type and content properties');
   }
   if (opts.record && !opts.record.action) {
     throw new DbErrorBadRequest('record requires action property');
   }
+  if (opts.dtmf && !opts.dtmf.digit) {
+    throw new DbErrorBadRequest('invalid dtmf');
+  }
   if ('startCallRecording' === opts.record?.action && !opts.record.siprecServerURL) {
     throw new DbErrorBadRequest('record requires siprecServerURL property when starting recording');
   }
+  if (opts.tag && (typeof opts.tag !== 'object' || Array.isArray(opts.tag) || opts.tag === null)) {
+    throw new DbErrorBadRequest('invalid tag data');
+  }
+  if (opts.conferenceParticipantAction) {
+    if (!['tag', 'untag', 'coach', 'uncoach', 'mute', 'unmute', 'hold', 'unhold']
+      .includes(opts.conferenceParticipantAction.action)) {
+      throw new DbErrorBadRequest(
+        `conferenceParticipantAction invalid action property ${opts.conferenceParticipantAction.action}`);
+    }
+    if ('tag' == opts.conferenceParticipantAction.action && !opts.conferenceParticipantAction.tag) {
+      throw new DbErrorBadRequest('conferenceParticipantAction requires tag property when action is \'tag\'');
+    }
+    if ('coach' == opts.conferenceParticipantAction.action && !opts.conferenceParticipantAction.tag) {
+      throw new DbErrorBadRequest('conferenceParticipantAction requires tag property when action is \'coach\'');
+    }
+  }
 }
 
 function validateTo(to) {
@@ -215,6 +349,7 @@ function validateTo(to) {
   }
   throw new DbErrorBadRequest(`missing or invalid to property: ${JSON.stringify(to)}`);
 }
+
 async function validateCreateCall(logger, sid, req) {
   const {lookupAppBySid} = req.app.locals;
   const obj = req.body;
@@ -231,6 +366,7 @@ async function validateCreateCall(logger, sid, req) {
       const application = await lookupAppBySid(obj.application_sid);
       Object.assign(obj, {
         call_hook: application.call_hook,
+        app_json: application.app_json,
         call_status_hook: application.call_status_hook,
         speech_synthesis_vendor: application.speech_synthesis_vendor,
         speech_synthesis_language: application.speech_synthesis_language,
@@ -315,7 +451,7 @@ async function validateCreateMessage(logger, sid, req) {
 async function validateAdd(req) {
   /* account-level token can not be used to add accounts */
   if (req.user.hasAccountAuth) {
-    throw new DbErrorUnprocessableRequest('insufficient permissions to create accounts');
+    throw new DbErrorForbidden('insufficient permissions');
   }
   if (req.user.hasServiceProviderAuth && req.user.service_provider_sid) {
     /* service providers can only create accounts under themselves */
@@ -334,9 +470,10 @@ async function validateAdd(req) {
     throw new DbErrorBadRequest('\'queue_event_hook\' must be an object when adding an account');
   }
 }
+
 async function validateUpdate(req, sid) {
   if (req.user.hasAccountAuth && req.user.account_sid !== sid) {
-    throw new DbErrorUnprocessableRequest('insufficient privileges to update this account');
+    throw new DbErrorForbidden('insufficient privileges');
   }
   if (req.user.hasAccountAuth && req.body.sip_realm) {
     throw new DbErrorBadRequest('use POST /Accounts/:sid/sip_realm/:realm to set or change the sip realm');
@@ -344,12 +481,23 @@ async function validateUpdate(req, sid) {
 
   if (req.user.service_provider_sid && !req.user.hasScope('admin')) {
     const result = await Account.retrieve(sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${sid}`);
+    }
     if (result[0].service_provider_sid !== req.user.service_provider_sid) {
-      throw new DbErrorUnprocessableRequest('cannot update account from different service provider');
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
+  if (req.user.hasScope('admin')) {
+    /* check to be sure that the account_sid exists */
+    const result = await Account.retrieve(sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${sid}`);
     }
   }
   if (req.body.service_provider_sid) throw new DbErrorBadRequest('service_provider_sid may not be modified');
 }
+
 async function validateDelete(req, sid) {
   if (req.user.hasAccountAuth && req.user.account_sid !== sid) {
     throw new DbErrorUnprocessableRequest('insufficient privileges to update this account');
@@ -357,12 +505,11 @@ async function validateDelete(req, sid) {
   if (req.user.service_provider_sid && !req.user.hasScope('admin')) {
     const result = await Account.retrieve(sid);
     if (result[0].service_provider_sid !== req.user.service_provider_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete account from different service provider');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
 }
 
-
 /* add */
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
@@ -404,8 +551,10 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     return res.status(200).json(results[0]);
   }
@@ -417,13 +566,15 @@ router.get('/:sid', async(req, res) => {
 router.get('/:sid/WebhookSecret', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     let {webhook_secret} = results[0];
     if (req.query.regenerate) {
       const secret = `wh_secret_${translator.generate()}`;
-      await Account.update(req.params.sid, {webhook_secret: secret});
+      await Account.update(account_sid, {webhook_secret: secret});
       webhook_secret = secret;
     }
     return res.status(200).json({webhook_secret});
@@ -436,8 +587,9 @@ router.get('/:sid/WebhookSecret', async(req, res) => {
 router.post('/:sid/SubspaceTeleport', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     const {subspace_client_id, subspace_client_secret} = results[0];
     const {destination} = req.body;
@@ -450,7 +602,7 @@ router.post('/:sid/SubspaceTeleport', async(req, res) => {
       destination: dest
     });
     logger.info({destination, teleport}, 'SubspaceTeleport - create teleport');
-    await Account.update(req.params.sid, {
+    await Account.update(account_sid, {
       subspace_sip_teleport_id: teleport.id,
       subspace_sip_teleport_destinations: JSON.stringify(teleport.teleport_entry_points)//hacky
     });
@@ -468,13 +620,14 @@ router.post('/:sid/SubspaceTeleport', async(req, res) => {
 router.delete('/:sid/SubspaceTeleport', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     const {subspace_client_id, subspace_client_secret, subspace_sip_teleport_id} = results[0];
 
     await disableSubspace({subspace_client_id, subspace_client_secret, subspace_sip_teleport_id});
-    await Account.update(req.params.sid, {
+    await Account.update(account_sid, {
       subspace_sip_teleport_id: null,
       subspace_sip_teleport_destinations: null
     });
@@ -485,11 +638,66 @@ router.delete('/:sid/SubspaceTeleport', async(req, res) => {
   }
 });
 
-/* update */
+function encryptBucketCredential(obj) {
+  if (!obj.bucket_credential) return;
+  const {
+    vendor,
+    region,
+    name,
+    access_key_id,
+    secret_access_key,
+    tags,
+    service_key,
+    connection_string,
+    endpoint
+  } = obj.bucket_credential;
+
+  switch (vendor) {
+    case 'aws_s3':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(region, 'invalid aws bucket region: region is required');
+      const awsData = JSON.stringify({vendor, region, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(awsData);
+      break;
+    case 's3_compatible':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(endpoint, 'invalid endpoint uri: endpoint is required');
+      const s3Data = JSON.stringify({vendor, endpoint, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(s3Data);
+      break;
+    case 'google':
+      assert(service_key, 'invalid google cloud storage credential: service_key is required');
+      const googleData = JSON.stringify({vendor, name, service_key, tags});
+      obj.bucket_credential = encrypt(googleData);
+      break;
+    case 'azure':
+      assert(name, 'invalid azure container name: name is required');
+      assert(connection_string, 'invalid azure cloud storage credential: connection_string is required');
+      const azureData = JSON.stringify({vendor, name, connection_string, tags});
+      obj.bucket_credential = encrypt(azureData);
+      break;
+    case 'none':
+      obj.bucket_credential = null;
+      break;
+    default:
+      throw new DbErrorBadRequest(`unknown storage vendor: ${vendor}`);
+  }
+}
+
+/**
+ * update
+ */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
 
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
@@ -528,6 +736,8 @@ router.put('/:sid', async(req, res) => {
       delete obj.registration_hook;
       delete obj.queue_event_hook;
 
+      encryptBucketCredential(obj);
+
       const rowsAffected = await Account.update(sid, obj);
       if (rowsAffected === 0) {
         return res.status(404).end();
@@ -549,12 +759,14 @@ router.put('/:sid', async(req, res) => {
 
 /* delete */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   const sqlDeleteGateways = `DELETE from sip_gateways 
     WHERE voip_carrier_sid IN
     (SELECT voip_carrier_sid from voip_carriers where account_sid = ?)`;
+
   try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
     await validateDelete(req, sid);
 
     const [account] = await promisePool.query('SELECT * FROM accounts WHERE account_sid = ?', sid);
@@ -618,13 +830,57 @@ account_subscriptions WHERE account_sid = ?)
   }
 });
 
-/* retrieve account level api keys */
-router.get('/:sid/ApiKeys', async(req, res) => {
+/* Test Bucket credential Keys */
+router.post('/:sid/BucketCredentialTest', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await ApiKey.retrieveAll(req.params.sid);
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const {vendor, name, region, access_key_id, secret_access_key, service_key, connection_string, endpoint} = req.body;
+    const ret = {
+      status: 'not tested'
+    };
+
+    switch (vendor) {
+      case 'aws_s3':
+        await testS3Storage(logger, {vendor, name, region, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 's3_compatible':
+        await testS3Storage(logger, {vendor, name, endpoint, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 'google':
+        await testGoogleStorage(logger, {vendor, name, service_key});
+        ret.status = 'ok';
+        break;
+      case 'azure':
+        await testAzureStorage(logger, {vendor, name, connection_string});
+        ret.status = 'ok';
+        break;
+      default:
+        throw new DbErrorBadRequest(`Does not support test for ${vendor}`);
+    }
+    return res.status(200).json(ret);
+  }
+  catch (err) {
+    return res.status(200).json({status: 'failed', reason: err.message});
+  }
+});
+
+/**
+ * retrieve account level api keys
+ */
+router.get('/:sid/ApiKeys', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
+
+    const results = await ApiKey.retrieveAll(sid);
     res.status(200).json(results);
-    updateLastUsed(logger, req.params.sid, req).catch((err) => {});
+    updateLastUsed(logger, sid, req).catch((err) => {});
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -634,36 +890,40 @@ router.get('/:sid/ApiKeys', async(req, res) => {
  * create a new Call
  */
 router.post('/:sid/Calls', async(req, res) => {
-  const sid = req.params.sid;
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const {retrieveSet, logger} = req.app.locals;
-
+  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
   const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
+
   if (!serviceUrl) {
-    res.status(480).json({msg: 'no available feature servers at this time'});
-  } else {
-    try {
-      await validateCreateCall(logger, sid, req);
-      updateLastUsed(logger, sid, req).catch((err) => {});
-      request({
-        url: serviceUrl,
-        method: 'POST',
-        json: true,
-        body: Object.assign(req.body, {account_sid: sid})
-      }, (err, response, body) => {
-        if (err) {
-          logger.error(err, `Error sending createCall POST to ${serviceUrl}`);
-          return res.sendStatus(500);
-        }
-        if (response.statusCode !== 201) {
-          logger.error({statusCode: response.statusCode}, `Non-success response returned by createCall ${serviceUrl}`);
-          return res.sendStatus(500);
-        }
-        res.status(201).json(body);
-      });
-    } catch (err) {
-      sysError(logger, res, err);
-    }
+    return res.status(480).json({msg: 'no available feature servers at this time'});
+  }
+
+  try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
+
+    await validateCreateCall(logger, sid, req);
+    updateLastUsed(logger, sid, req).catch((err) => {});
+    request({
+      url: serviceUrl,
+      method: 'POST',
+      json: true,
+      body: Object.assign(req.body, {account_sid: sid})
+    }, (err, response, body) => {
+      if (err) {
+        logger.error(err, `Error sending createCall POST to ${serviceUrl}`);
+        return res.sendStatus(500);
+      }
+
+      if (response.statusCode !== 201) {
+        logger.error({statusCode: response.statusCode}, `Non-success response returned by createCall ${serviceUrl}`);
+        return res.sendStatus(500);
+      }
+
+      return res.status(201).json(body);
+    });
+  } catch (err) {
+    sysError(logger, res, err);
   }
 });
 
@@ -671,11 +931,18 @@ router.post('/:sid/Calls', async(req, res) => {
  * retrieve info for a group of calls under an account
  */
 router.get('/:sid/Calls', async(req, res) => {
-  const accountSid = req.params.sid;
   const {logger, listCalls} = req.app.locals;
-
+  const {direction, from, to, callStatus} = req.query || {};
   try {
-    const calls = await listCalls(accountSid);
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const calls = await listCalls({
+      accountSid,
+      direction,
+      from,
+      to,
+      callStatus
+    });
     logger.debug(`retrieved ${calls.length} calls for account sid ${accountSid}`);
     res.status(200).json(coerceNumbers(snakeCase(calls)));
     updateLastUsed(logger, accountSid, req).catch((err) => {});
@@ -688,11 +955,12 @@ router.get('/:sid/Calls', async(req, res) => {
  * retrieve single call
  */
 router.get('/:sid/Calls/:callSid', async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, retrieveCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     const callInfo = await retrieveCall(accountSid, callSid);
     if (callInfo) {
       logger.debug(callInfo, `retrieved call info for call sid ${callSid}`);
@@ -712,11 +980,12 @@ router.get('/:sid/Calls/:callSid', async(req, res) => {
  * delete call
  */
 router.delete('/:sid/Calls/:callSid', async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, deleteCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     const result = await deleteCall(accountSid, callSid);
     if (result) {
       logger.debug(`successfully deleted call ${callSid}`);
@@ -736,11 +1005,12 @@ router.delete('/:sid/Calls/:callSid', async(req, res) => {
  * update a call
  */
 const updateCall = async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, retrieveCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     validateUpdateCall(req.body);
     const call = await retrieveCall(accountSid, callSid);
     if (call) {
@@ -767,6 +1037,7 @@ const updateCall = async(req, res) => {
 router.post('/:sid/Calls/:callSid', async(req, res) => {
   await updateCall(req, res);
 });
+
 router.put('/:sid/Calls/:callSid', async(req, res) => {
   await updateCall(req, res);
 });
@@ -775,11 +1046,13 @@ router.put('/:sid/Calls/:callSid', async(req, res) => {
  * create a new Message
  */
 router.post('/:sid/Messages', async(req, res) => {
-  const account_sid = parseAccountSid(req);
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const {retrieveSet, logger} = req.app.locals;
 
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
+    const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
     const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
     if (!serviceUrl) res.json({msg: 'no available feature servers at this time'}).status(480);
     await validateCreateMessage(logger, account_sid, req);
@@ -812,4 +1085,39 @@ router.post('/:sid/Messages', async(req, res) => {
   }
 });
 
+/**
+ * retrieve info for a group of queues under an account
+ */
+router.get('/:sid/Queues', async(req, res) => {
+  const {logger, listSortedSets} = req.app.locals;
+  const { search } = req.query || {};
+  try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const queues = search ? await listSortedSets(accountSid, search) : await listSortedSets(accountSid);
+    logger.debug(`retrieved ${queues.length} queues for account sid ${accountSid}`);
+    res.status(200).json(queues);
+    updateLastUsed(logger, accountSid, req).catch((err) => {});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/**
+ * retrieve info for a list of conferences under an account
+ */
+router.get('/:sid/Conferences', async(req, res) => {
+  const {logger, listConferences} = req.app.locals;
+  try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const conferences = await listConferences(accountSid);
+    logger.debug(`retrieved ${conferences.length} queues for account sid ${accountSid}`);
+    res.status(200).json(conferences.map((c) => c.split(':').pop()));
+    updateLastUsed(logger, accountSid, req).catch((err) => {});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 module.exports = router;

lib/routes/api/add-from-predefined-carrier.js

@@ -24,13 +24,13 @@ router.post('/:sid', async(req, res) => {
   let service_provider_sid;
   const {account_sid} = req.user;
 
-  if (!account_sid) {
-    service_provider_sid = parseServiceProviderSid(req);
-  } else {
-    service_provider_sid = req.user.service_provider_sid;
-  }
-
   try {
+    if (!account_sid) {
+      service_provider_sid = parseServiceProviderSid(req);
+    } else {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
     const [template] = await PredefinedCarrier.retrieve(sid);
     logger.debug({template}, `Retrieved template carrier for sid ${sid}`);
     if (!template) return res.sendStatus(404);

lib/routes/api/alerts.js

@@ -1,17 +1,13 @@
 const router = require('express').Router();
 const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
+const { parseServiceProviderSid } = require('./utils');
 
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
   if (arr) return arr[1];
 };
 
-const parseServiceProviderSid = (url) => {
-  const arr = /ServiceProviders\/([^\/]*)/.exec(url);
-  if (arr) return arr[1];
-};
-
 router.get('/', async(req, res) => {
   const {logger, queryAlerts, queryAlertsSP} = req.app.locals;
   try {

lib/routes/api/applications.js

@@ -1,5 +1,5 @@
 const router = require('express').Router();
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const Application = require('../../models/application');
 const Account = require('../../models/account');
 const Webhook = require('../../models/webhook');
@@ -7,11 +7,42 @@ const {promisePool} = require('../../db');
 const decorate = require('./decorate');
 const sysError = require('../error');
 const { validate } = require('@jambonz/verb-specifications');
+const { parseApplicationSid } = require('./utils');
 const preconditions = {
   'add': validateAdd,
   'update': validateUpdate
 };
 
+const validateRequest = async(req, account_sid) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('account')) {
+      if (account_sid === req.user.account_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+      );
+
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  } catch (error) {
+    throw error;
+  }
+};
+
 /* only user-level tokens can add applications */
 async function validateAdd(req) {
   if (req.user.account_sid) {
@@ -22,7 +53,7 @@ async function validateAdd(req) {
     if (!req.body.account_sid) throw new DbErrorBadRequest('missing required field: \'account_sid\'');
     const result = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
     if (result.length === 0) {
-      throw new DbErrorBadRequest('insufficient privileges to create an application under the specified account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
@@ -34,12 +65,26 @@ async function validateAdd(req) {
 }
 
 async function validateUpdate(req, sid) {
-  if (req.user.account_sid) {
-    const app = await Application.retrieve(sid);
-    if (!app || !app.length || app[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorBadRequest('you may not update or delete an application associated with a different account');
+  const app = await Application.retrieve(sid);
+  if (req.user.hasAccountAuth) {
+    if (!app || 0 === app.length || app[0].account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
+
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [app[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  }
+
+
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
     throw new DbErrorBadRequest('\'call_hook\' must be an object when updating an application');
   }
@@ -49,13 +94,24 @@ async function validateUpdate(req, sid) {
 }
 
 async function validateDelete(req, sid) {
+  const result = await Application.retrieve(sid);
   if (req.user.hasAccountAuth) {
-    const result = await Application.retrieve(sid);
     if (!result || 0 === result.length) throw new DbErrorBadRequest('application does not exist');
     if (result[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete application owned by a different account');
+      throw new DbErrorUnprocessableRequest('insufficient permissions');
     }
   }
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [result[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  }
   const assignedPhoneNumbers = await Application.getForeignKeyReferences('phone_numbers.application_sid', sid);
   if (assignedPhoneNumbers > 0) throw new DbErrorUnprocessableRequest('cannot delete application with phone numbers');
 }
@@ -111,10 +167,12 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const application_sid = parseApplicationSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await Application.retrieve(req.params.sid, service_provider_sid, account_sid);
+    const results = await Application.retrieve(application_sid, service_provider_sid, account_sid);
     if (results.length === 0) return res.status(404).end();
+    await validateRequest(req, results[0].account_sid);
     return res.status(200).json(results[0]);
   }
   catch (err) {
@@ -124,9 +182,9 @@ router.get('/:sid', async(req, res) => {
 
 /* delete */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseApplicationSid(req);
     await validateDelete(req, sid);
 
     const [application] = await promisePool.query('SELECT * FROM applications WHERE application_sid = ?', sid);
@@ -165,9 +223,9 @@ router.delete('/:sid', async(req, res) => {
 
 /* update */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseApplicationSid(req);
     await validateUpdate(req, sid);
 
     // create webhooks if provided

lib/routes/api/change-password.js

@@ -1,15 +1,15 @@
 const router = require('express').Router();
-//const debug = require('debug')('jambonz:api-server');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {generateHashedPassword, verifyPassword} = require('../../utils/password-utils');
 const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 const sqlUpdatePassword = `UPDATE users 
 SET hashed_password= ? 
 WHERE user_sid = ?`;
 
 router.post('/', async(req, res) => {
-  const {logger, retrieveKey, deleteKey} = req.app.locals;
+  const {logger} = req.app.locals;
   const {user_sid} = req.user;
   const {old_password, new_password} = req.body;
   try {
@@ -26,10 +26,10 @@ router.post('/', async(req, res) => {
 
       const isCorrect = await verifyPassword(r[0].hashed_password, old_password);
       if (!isCorrect) {
-        const key = `reset-link:${old_password}`;
-        const user_sid = await retrieveKey(key);
+        const key = cacheClient.generateRedisKey('reset-link', old_password);
+        const user_sid = await cacheClient.get(key);
         if (!user_sid) throw new DbErrorBadRequest('old_password is incorrect');
-        await deleteKey(key);
+        await cacheClient.delete(key);
       }
     }
 
@@ -42,6 +42,9 @@ router.post('/', async(req, res) => {
     sysError(logger, res, err);
     return;
   }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+  await cacheClient.delete(redisKey);
 });
 
 module.exports = router;

lib/routes/api/clients.js

@@ -0,0 +1,88 @@
+const router = require('express').Router();
+const decorate = require('./decorate');
+const sysError = require('../error');
+const Client = require('../../models/client');
+const Account = require('../../models/account');
+const { DbErrorBadRequest, DbErrorForbidden } = require('../../utils/errors');
+const { encrypt, decrypt, obscureKey } = require('../../utils/encrypt-decrypt');
+
+const commonCheck = async(req) => {
+  if (req.user.hasAccountAuth) {
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth && req.body.account_sid) {
+    const accounts = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
+    if (accounts.length === 0) {
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  }
+
+  if (req.body.password) {
+    req.body.password = encrypt(req.body.password);
+  }
+};
+
+const validateAdd = async(req) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length && clients[0].client_sid !== sid) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, Client, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Client.retrieveAll() : req.user.hasAccountAuth ?
+        await Client.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Client.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+    const ret = results.map((c) => {
+      c.password = obscureKey(decrypt(c.password), 1);
+      return c;
+    });
+    res.status(200).json(ret);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Client.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const client = results[0];
+    client.password = obscureKey(decrypt(client.password), 1);
+    if (req.user.hasAccountAuth && client.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth) {
+      const accounts = await Account.retrieve(client.account_sid, req.user.service_provider_sid);
+      if (!accounts.length) {
+        return res.sendStatus(404);
+      }
+    }
+    return res.status(200).json(client);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/error.js

@@ -1,6 +1,10 @@
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const { BadRequestError, DbErrorBadRequest, DbErrorUnprocessableRequest } = require('../../utils/errors');
 
 function sysError(logger, res, err) {
+  if (err instanceof BadRequestError) {
+    logger.info(err, err.message);
+    return res.status(400).json({msg: 'Bad request'});
+  }
   if (err instanceof DbErrorBadRequest) {
     logger.info(err, 'invalid client request');
     return res.status(400).json({msg: err.message});

lib/routes/api/forgot-password.js

@@ -4,7 +4,9 @@ const short = require('short-uuid');
 const translator = short();
 const {validateEmail, emailSimpleText} = require('../../utils/email-utils');
 const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
+const assert = require('assert');
 const sql = `SELECT * from users user 
 LEFT JOIN accounts AS acc 
 ON acc.account_sid = user.account_sid 
@@ -25,7 +27,8 @@ function createOauthEmailText(provider) {
 }
 
 function createResetEmailText(link) {
-  const baseUrl = 'http://localhost:3001';
+  assert(process.env.JAMBONZ_BASE_URL, 'process.env.JAMBONZ_BASE_URL is missing');
+  const baseUrl = process.env.JAMBONZ_BASE_URL;
 
   return `Hi there!
 
@@ -45,6 +48,7 @@ function createResetEmailText(link) {
 router.post('/', async(req, res) => {
   const {logger, addKey} = req.app.locals;
   const {email} = req.body;
+
   let obj;
   try {
     if (!email || !validateEmail(email)) {
@@ -53,11 +57,16 @@ router.post('/', async(req, res) => {
 
     const [r] = await promisePool.query({sql, nestTables: true}, email);
     if (0 === r.length) {
-      return res.status(400).json({error: 'email does not exist'});
+      logger.info('user not found');
+      return res.status(400).json({error: 'failed to reset your password'});
     }
     obj = r[0];
-    if (!obj.acc.is_active) {
-      return res.status(400).json({error: 'you may not reset the password of an inactive account'});
+    if (!obj.user.is_active) {
+      logger.info(obj.user.name, 'user is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
+    } else if (obj.acc.account_sid !== null && !obj.acc.is_active) {
+      logger.info(obj.acc.account_sid, 'account is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
     }
     res.sendStatus(204);
   } catch (err) {
@@ -73,10 +82,14 @@ router.post('/', async(req, res) => {
   else {
     /* generate a link for this user to reset, send email */
     const link = translator.generate();
-    addKey(`reset-link:${link}`, obj.user.user_sid, 3600)
+    const redisKey = cacheClient.generateRedisKey('reset-link', link);
+    addKey(redisKey, obj.user.user_sid, 3600)
       .catch((err) => logger.error({err}, 'Error adding reset link to redis'));
     emailSimpleText(logger, email, 'Reset password request', createResetEmailText(link));
   }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', obj.user.user_sid, 'v2');
+  await cacheClient.delete(redisKey);
 });
 
 module.exports = router;

lib/routes/api/google-custom-voices.js

@@ -0,0 +1,77 @@
+const router = require('express').Router();
+const GoogleCustomVoice = require('../../models/google-custom-voice');
+const SpeechCredential = require('../../models/speech-credential');
+const decorate = require('./decorate');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateCredentialPermission = async(req) => {
+  const credential = await SpeechCredential.retrieve(req.body.speech_credential_sid);
+  if (!credential || credential.length === 0) {
+    throw new DbErrorBadRequest('Invalid speech_credential_sid');
+  }
+  const cred = credential[0];
+
+  if (req.user.hasServiceProviderAuth && cred.service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+  if (req.user.hasAccountAuth && cred.account_sid !== req.user.account_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+};
+
+const validateAdd = async(req) => {
+  if (!req.body.speech_credential_sid) {
+    throw new DbErrorBadRequest('missing speech_credential_sid');
+  }
+
+  await validateCredentialPermission(req);
+};
+
+const validateUpdate = async(req) => {
+  if (req.body.speech_credential_sid) {
+    await validateCredentialPermission(req);
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, GoogleCustomVoice, ['add', 'retrieve', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const account_sid = req.user.account_sid || req.query.account_sid;
+  const service_provider_sid = req.user.service_provider_sid || req.query.service_provider_sid;
+  const speech_credential_sid = req.query.speech_credential_sid;
+  const label = req.query.label;
+  try {
+    let results = [];
+    if (speech_credential_sid) {
+      const [cred] = await SpeechCredential.retrieve(speech_credential_sid);
+      if (!cred) {
+        return res.sendStatus(404);
+      }
+      if (account_sid && cred.account_sid && cred.account_sid !== account_sid) {
+        throw new DbErrorForbidden('Insufficient privileges');
+      }
+      if (service_provider_sid && cred.service_provider_sid && cred.service_provider_sid !== service_provider_sid) {
+        throw new DbErrorForbidden('Insufficient privileges');
+      }
+      results = await GoogleCustomVoice.retrieveAllBySpeechCredentialSid(speech_credential_sid);
+    } else {
+      if (!account_sid && !service_provider_sid) {
+        throw new DbErrorBadRequest('missing account_sid or service_provider_sid in query parameters');
+      }
+      results = await GoogleCustomVoice.retrieveAllByLabel(service_provider_sid, account_sid, label);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/index.js

@@ -16,6 +16,8 @@ const isAdminScope = (req, res, next) => {
 // };
 
 api.use('/BetaInviteCodes', isAdminScope, require('./beta-invite-codes'));
+api.use('/SystemInformation', isAdminScope, require('./system-information'));
+api.use('/TtsCache', isAdminScope, require('./tts-cache'));
 api.use('/ServiceProviders', require('./service-providers'));
 api.use('/VoipCarriers', require('./voip-carriers'));
 api.use('/Webhooks', require('./webhooks'));
@@ -45,6 +47,13 @@ api.use('/Invoices', require('./invoices'));
 api.use('/InviteCodes', require('./invite-codes'));
 api.use('/PredefinedCarriers', require('./predefined-carriers'));
 api.use('/PasswordSettings', require('./password-settings'));
+// Least Cost Routing
+api.use('/Lcrs', require('./lcrs'));
+api.use('/LcrRoutes', require('./lcr-routes'));
+api.use('/LcrCarrierSetEntries', require('./lcr-carrier-set-entries'));
+api.use('/Clients', require('./clients'));
+// Google Custom Voices
+api.use('/GoogleCustomVoices', require('./google-custom-voices'));
 
 // messaging
 api.use('/Smpps', require('./smpps'));   // our smpp server info

lib/routes/api/lcr-carrier-set-entries.js

@@ -0,0 +1,65 @@
+const router = require('express').Router();
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoute = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!req.body.lcr_route_sid) {
+    throw new DbErrorBadRequest('missing lcr_route_sid');
+  }
+  // check lcr_route_sid is exist
+  const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+  if (lcrRoute.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_route_sid');
+  }
+  // check voip_carrier_sid is exist
+  if (!req.body.voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+  const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+  if (!carrier) {
+    throw new DbErrorBadRequest('unknown voip_carrier_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (req.body.lcr_route_sid) {
+    const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+    if (lcrRoute.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_route_sid');
+    }
+  }
+
+  // check voip_carrier_sid is exist
+  if (req.body.voip_carrier_sid) {
+    const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+    if (!carrier) {
+      throw new DbErrorBadRequest('unknown voip_carrier_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, LcrCarrierSetEntry, ['add', 'retrieve', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.query.lcr_route_sid;
+  try {
+    const results = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(lcr_route_sid);
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcr-routes.js

@@ -0,0 +1,96 @@
+const router = require('express').Router();
+const LcrRoute = require('../../models/lcr-route');
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  // check if lcr sid is available
+  if (!req.body.lcr_sid) {
+    throw new DbErrorBadRequest('missing parameter lcr_sid');
+  }
+
+  const lcr = await Lcr.retrieve(req.body.lcr_sid);
+  if (lcr.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  if (req.body.lcr_sid) {
+    const lcr = await Lcr.retrieve(req.body.lcr_sid);
+    if (lcr.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateDelete = async(req, sid) => {
+  // delete all lcr carrier set entries
+  await LcrCarrierSetEntry.deleteByLcrRouteSid(sid);
+};
+
+const checkUserScope = async(req, lcr_sid) => {
+  if (!lcr_sid) {
+    throw new DbErrorBadRequest('missing lcr_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+
+  const lcrList = await Lcr.retrieve(lcr_sid);
+  if (lcrList.length === 0) throw new DbErrorBadRequest('unknown lcr_sid');
+  const lcr = lcrList[0];
+  if (req.user.hasAccountAuth) {
+    if (!lcr.account_sid || lcr.account_sid !== req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+
+  if (req.user.hasServiceProviderAuth) {
+    if (!lcr.service_provider_sid  || lcr.service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete,
+};
+
+decorate(router, LcrRoute, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_sid = req.query.lcr_sid;
+  try {
+    await checkUserScope(req, lcr_sid);
+    const results = await LcrRoute.retrieveAllByLcrSid(lcr_sid);
+    for (const r of results) {
+      r.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(r.lcr_route_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.params.sid;
+  try {
+    const results = await LcrRoute.retrieve(lcr_route_sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const route = results[0];
+    route.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(route.lcr_route_sid);
+    res.status(200).json(route);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcrs.js

@@ -0,0 +1,225 @@
+const router = require('express').Router();
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoutes = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+const ServiceProvider = require('../../models/service-provider');
+
+const validateAssociatedTarget = async(req, sid) => {
+  const {lookupAccountBySid} = req.app.locals;
+  if (req.body.account_sid) {
+    // Add only for account
+    req.body.service_provider_sid = null;
+    const account = await lookupAccountBySid(req.body.account_sid);
+    if (!account) throw new DbErrorBadRequest('unknown account_sid');
+    const lcr = await Lcr.retrieveAllByAccountSid(req.body.account_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Account: ${account.name}  already has an active call routing table.`);
+    }
+  } else if (req.body.service_provider_sid) {
+    const serviceProviders = await ServiceProvider.retrieve(req.body.service_provider_sid);
+    if (serviceProviders.length === 0) throw new DbErrorBadRequest('unknown service_provider_sid');
+    const serviceProvider = serviceProviders[0];
+    const lcr = await Lcr.retrieveAllByServiceProviderSid(req.body.service_provider_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Service Provider: ${serviceProvider.name} already 
+      has an active call routing table.`);
+    }
+  }
+};
+
+const validateAdd = async(req) => {
+  if (req.user.hasAccountAuth) {
+    // Account just create LCR for himself
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth) {
+    // SP just can create LCR for himself
+    req.body.service_provider_sid = req.user.service_provider_sid;
+    req.body.account_sid = null;
+  }
+
+  await validateAssociatedTarget(req);
+  // check if lcr_carrier_set_entry is available
+  if (req.body.lcr_carrier_set_entry) {
+    const e = await LcrCarrierSetEntry.retrieve(req.body.lcr_carrier_set_entry);
+    if (e.length === 0) throw new DbErrorBadRequest('unknown lcr_carrier_set_entry');
+  }
+
+};
+
+const validateUserPermissionForExistingEntity = async(req, sid) => {
+  const r = await Lcr.retrieve(sid);
+  if (r.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+  const lcr = r[0];
+  if (req.user.hasAccountAuth) {
+    if (lcr.account_sid != req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  } else if (req.user.hasServiceProviderAuth) {
+    if (lcr.service_provider_sid != req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await validateUserPermissionForExistingEntity(req, sid);
+  await validateAssociatedTarget(req, sid);
+};
+
+const validateDelete = async(req, sid) => {
+  if (req.user.hasAccountAuth) {
+    /* can only delete Lcr for the user's account */
+    const r = await Lcr.retrieve(sid);
+    const lcr = r.length > 0 ? r[0] : null;
+    if (!lcr || (req.user.account_sid && lcr.account_sid != req.user.account_sid)) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+  await Lcr.releaseDefaultEntry(sid);
+  // fetch lcr route
+  const lcr_routes = await LcrRoutes.retrieveAllByLcrSid(sid);
+  // delete all lcr carrier set entries
+  for (const e of lcr_routes) {
+    await LcrCarrierSetEntry.deleteByLcrRouteSid(e.lcr_route_sid);
+  }
+
+  // delete all lcr routes
+  await LcrRoutes.deleteByLcrSid(sid);
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete
+};
+
+decorate(router, Lcr, ['add', 'update', 'delete'], preconditions);
+
+const validateLcrBatchAdd = async(lcr_sid, body, lookupCarrierBySid) => {
+  for (const lcr_route of body) {
+    lcr_route.lcr_sid = lcr_sid;
+    if (!lcr_route.lcr_carrier_set_entries || lcr_route.lcr_carrier_set_entries.length === 0) {
+      throw new DbErrorBadRequest('Lcr Route batch process require lcr_carrier_set_entries');
+    }
+    for (const entry of lcr_route.lcr_carrier_set_entries) {
+      // check voip_carrier_sid is exist
+      if (!entry.voip_carrier_sid) {
+        throw new DbErrorBadRequest('One of lcr_carrier_set_entries is missing voip_carrier_sid');
+      }
+      const carrier = await lookupCarrierBySid(entry.voip_carrier_sid);
+      if (!carrier) {
+        throw new DbErrorBadRequest('unknown voip_carrier_sid');
+      }
+    }
+  }
+};
+
+
+const addNewLcrRoute = async(lcr_route) => {
+  const lcr_sid = lcr_route.lcr_sid;
+  const lcr_carrier_set_entries = lcr_route.lcr_carrier_set_entries;
+  delete lcr_route.lcr_carrier_set_entries;
+  const lcr_route_sid = await LcrRoutes.make(lcr_route);
+  for (const entry of lcr_carrier_set_entries) {
+    entry.lcr_route_sid = lcr_route_sid;
+    const lcr_carrier_set_entry_sid = await LcrCarrierSetEntry.make(entry);
+    if (lcr_route.priority === 9999) {
+      // this is default lcr set entry
+      const [lcr] = await Lcr.retrieve(lcr_sid);
+      if (lcr) {
+        lcr.default_carrier_set_entry_sid = lcr_carrier_set_entry_sid;
+        delete lcr.lcr_sid;
+        await Lcr.update(lcr_sid, lcr);
+      }
+    }
+  }
+};
+
+router.post('/:sid/Routes', async(req, res) => {
+  const results = await Lcr.retrieve(req.params.sid);
+  if (results.length === 0) return res.sendStatus(404);
+  const {logger, lookupCarrierBySid} = req.app.locals;
+  try {
+    const body = req.body;
+    await validateLcrBatchAdd(req.params.sid, body, lookupCarrierBySid);
+    for (const lcr_route of body) {
+      await addNewLcrRoute(lcr_route, lookupCarrierBySid);
+    }
+    res.sendStatus(204);
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.put('/:sid/Routes', async(req, res) => {
+  const results = await Lcr.retrieve(req.params.sid);
+  if (results.length === 0) return res.sendStatus(404);
+  const {logger, lookupCarrierBySid} = req.app.locals;
+  try {
+    const body = req.body;
+    await validateLcrBatchAdd(req.params.sid, body, lookupCarrierBySid);
+    for (const lcr_route of body) {
+      if (lcr_route.lcr_route_sid) {
+        const lcr_route_sid = lcr_route.lcr_route_sid;
+        delete lcr_route.lcr_route_sid;
+        const lcr_carrier_set_entries = lcr_route.lcr_carrier_set_entries;
+        delete lcr_route.lcr_carrier_set_entries;
+        await LcrRoutes.update(lcr_route_sid, lcr_route);
+        for (const entry of lcr_carrier_set_entries) {
+          const lcr_carrier_set_entry_sid = entry.lcr_carrier_set_entry_sid;
+          delete entry.lcr_carrier_set_entry_sid;
+          await LcrCarrierSetEntry.update(lcr_carrier_set_entry_sid, entry);
+        }
+      } else {
+        // Route is not available yet, let create it now
+        await addNewLcrRoute(lcr_route, lookupCarrierBySid);
+      }
+    }
+    res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Lcr.retrieveAll() : req.user.hasAccountAuth ?
+        await Lcr.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Lcr.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+
+    for (const lcr of results) {
+      lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Lcr.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const lcr = results[0];
+    if (req.user.hasAccountAuth && lcr.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth && lcr.service_provider_sid !== req.user.service_provider_sid) {
+      return res.sendStatus(404);
+    }
+    lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    return res.status(200).json(lcr);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/limits.js

@@ -22,22 +22,25 @@ DELETE FROM account_limits
 WHERE account_sid = ? 
 AND category = ?
 `;
+
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
   const {
     category,
     quantity
   } = req.body;
-  const account_sid = parseAccountSid(req);
-  let service_provider_sid;
-  if (!account_sid) {
-    if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
-      logger.error('POST /SpeechCredentials invalid credentials');
-      return res.sendStatus(403);
-    }
-    service_provider_sid = parseServiceProviderSid(req);
-  }
+
   try {
+    let service_provider_sid;
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) {
+      if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
+        logger.error('POST /SpeechCredentials invalid credentials');
+        return res.sendStatus(403);
+      }
+      service_provider_sid = parseServiceProviderSid(req);
+    }
+
     let uuid;
     if (account_sid) {
       const existing = (await AccountLimits.retrieve(account_sid) || [])
@@ -80,10 +83,11 @@ router.post('/', async(req, res) => {
  */
 router.get('/', async(req, res) => {
   let service_provider_sid;
-  const account_sid = parseAccountSid(req);
-  if (!account_sid) service_provider_sid = parseServiceProviderSid(req);
   const logger = req.app.locals.logger;
+
   try {
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) service_provider_sid = parseServiceProviderSid(req);
     const limits = account_sid ?
       await AccountLimits.retrieve(account_sid) :
       await ServiceProviderLimits.retrieve(service_provider_sid);
@@ -99,10 +103,11 @@ router.get('/', async(req, res) => {
 
 router.delete('/', async(req, res) => {
   const logger = req.app.locals.logger;
-  const account_sid = parseAccountSid(req);
-  const {category} = req.query;
-  const service_provider_sid = parseServiceProviderSid(req);
+
   try {
+    const account_sid = parseAccountSid(req);
+    const {category} = req.query;
+    const service_provider_sid = parseServiceProviderSid(req);
     if (account_sid) {
       if (category) {
         await promisePool.execute(sqlDeleteAccountLimitsByCategory, [account_sid, category]);

lib/routes/api/login.js

@@ -2,6 +2,7 @@ const router = require('express').Router();
 const jwt = require('jsonwebtoken');
 const {verifyPassword} = require('../../utils/password-utils');
 const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
 const Account = require('../../models/account');
 const ServiceProvider = require('../../models/service-provider');
 const sysError = require('../error');
@@ -12,11 +13,9 @@ WHERE up.permission_sid = p.permission_sid
 AND up.user_sid = ? 
 `;
 const retrieveSql = 'SELECT * from users where name = ?';
-const tokenSql = 'SELECT token from api_keys where account_sid IS NULL AND service_provider_sid IS NULL';
-
 
 router.post('/', async(req, res) => {
-  const logger = req.app.locals.logger;
+  const {logger, incrKey, retrieveKey} = req.app.locals;
   const {username, password} = req.body;
   if (!username || !password) {
     logger.info('Bad POST to /login is missing username or password');
@@ -30,15 +29,30 @@ router.post('/', async(req, res) => {
       return res.sendStatus(403);
     }
     logger.info({r}, 'successfully retrieved user account');
-    const isCorrect = await verifyPassword(r[0].hashed_password, password);
-    if (!isCorrect) return res.sendStatus(403);
-    const force_change = !!r[0].force_change;
-    const [t] = await promisePool.query(tokenSql);
-    if (t.length === 0) {
-      logger.error('Database has no admin token provisioned...run reset_admin_password');
-      return res.sendStatus(500);
+
+    const maxLoginAttempts = process.env.LOGIN_ATTEMPTS_MAX_RETRIES || 6;
+    const loginAttempsBlocked = await retrieveKey(`login:${r[0].user_sid}`) >= maxLoginAttempts;
+
+    if (loginAttempsBlocked) {
+      logger.info(`User ${r[0].user_sid} was blocked due to excessive login attempts with incorrect credentials.`);
+      return res.status(403)
+        .json({error: 'Maximum login attempts reached. Please try again later or reset your password.'});
     }
 
+    const isCorrect = await verifyPassword(r[0].hashed_password, password);
+    if (!isCorrect) {
+      const attempTime = process.env.LOGIN_ATTEMPTS_TIME || 1800;
+      const newAttempt = await incrKey(`login:${r[0].user_sid}`, attempTime)
+        .catch((err) => logger.error({err}, 'Error adding logging attempt to redis'));
+      if (newAttempt >= maxLoginAttempts) {
+        logger.info(`User ${r[0].user_sid} is now blocked due to excessive login attempts with incorrect credentials.`);
+        return res.status(403)
+          .json({error: `Maximum login attempts reached. Please try again in ${attempTime} seconds.`});
+      }
+      return res.sendStatus(403);
+    }
+    const force_change = !!r[0].force_change;
+
     const [p] = await promisePool.query(retrievePemissionsSql, r[0].user_sid);
     const permissions = p.map((x) => x.name);
     const obj = {user_sid: r[0].user_sid, scope: 'admin', force_change, permissions};
@@ -71,12 +85,22 @@ router.post('/', async(req, res) => {
       }),
       user_sid: obj.user_sid
     };
+
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60;
     const token = jwt.sign(
       payload,
       process.env.JWT_SECRET,
-      { expiresIn: parseInt(process.env.JWT_EXPIRES_IN || 60) * 60 }
+      { expiresIn }
     );
     res.json({token, ...obj});
+
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', obj.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/logout.js

@@ -1,19 +1,18 @@
 const router = require('express').Router();
 const debug = require('debug')('jambonz:api-server');
-const {hashString} = require('../../utils/password-utils');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 
 router.post('/', async(req, res) => {
-  const {logger, addKey} = req.app.locals;
-  const {jwt} = req.user;
+  const {logger} = req.app.locals;
+  const {user_sid} = req.user;
 
-  debug(`adding jwt to blacklist: ${jwt}`);
+  debug(`logout user and invalidate jwt token for user: ${user_sid}`);
 
   try {
-    /* add key to blacklist */
-    const s = `jwt:${hashString(jwt)}`;
-    const result = await addKey(s, '1', 3600);
-    debug(`result from adding ${s}: ${result}`);
+    const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+    await cacheClient.delete(redisKey);
+
     res.sendStatus(204);
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/password-settings.js

@@ -15,6 +15,9 @@ const validate = (obj) => {
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    if (!req.user.hasAdminAuth) {
+      return res.sendStatus(403);
+    }
     validate(req.body);
     const [existing] = (await PasswordSettings.retrieve() || []);
     if (existing) {

lib/routes/api/phone-numbers.js

@@ -1,8 +1,10 @@
 const router = require('express').Router();
-const {DbErrorUnprocessableRequest, DbErrorBadRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
 const PhoneNumber = require('../../models/phone-number');
 const VoipCarrier = require('../../models/voip-carrier');
+const Account = require('../../models/account');
 const decorate = require('./decorate');
+const {promisePool} = require('../../db');
 const {e164} = require('../../utils/phone-number-utils');
 const preconditions = {
   'add': validateAdd,
@@ -10,6 +12,7 @@ const preconditions = {
   'update': validateUpdate
 };
 const sysError = require('../error');
+const { parsePhoneNumberSid } = require('./utils');
 
 
 /* check for required fields when adding */
@@ -20,6 +23,10 @@ async function validateAdd(req) {
       req.body.account_sid = req.user.account_sid;
     }
 
+    if (req.user.hasServiceProviderAuth) {
+      req.body.service_provider_sid = req.user.service_provider_sid;
+    }
+
     if (!req.body.number) throw new DbErrorBadRequest('number is required');
     const formattedNumber = e164(req.body.number);
     req.body.number = formattedNumber;
@@ -41,11 +48,11 @@ async function checkInUse(req, sid) {
   const phoneNumber = await PhoneNumber.retrieve(sid);
   if (req.user.hasAccountAuth) {
     if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete a phone number that belongs to another account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
   if (!req.user.hasAccountAuth && phoneNumber.account_sid) {
-    throw new DbErrorUnprocessableRequest('cannot delete phone number that is assigned to an account');
+    throw new DbErrorForbidden('insufficient privileges');
   }
 }
 
@@ -57,10 +64,23 @@ async function validateUpdate(req, sid) {
   const phoneNumber = await PhoneNumber.retrieve(sid);
   if (req.user.hasAccountAuth) {
     if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot operate on a phone number that belongs to another account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
+  if (req.user.hasServiceProviderAuth) {
+    let service_provider_sid;
 
+    if (!phoneNumber[0].service_provider_sid) {
+      const [r] = await Account.retrieve(phoneNumber[0].account_sid);
+      service_provider_sid = r.service_provider_sid;
+    } else {
+      service_provider_sid = phoneNumber[0].service_provider_sid;
+    }
+
+    if (phoneNumber && phoneNumber.length && service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
   // TODO: if we are assigning to an account, verify it exists
 
   // TODO: if we are assigning to an application, verify it is associated to the same account
@@ -74,7 +94,9 @@ decorate(router, PhoneNumber, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
+    const results = req.user.hasServiceProviderAuth ?
+      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid) :
+      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -85,9 +107,22 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parsePhoneNumberSid(req);
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await PhoneNumber.retrieve(req.params.sid, account_sid);
+    const results = await PhoneNumber.retrieve(sid, account_sid);
     if (results.length === 0) return res.status(404).end();
+
+    if (req.user.hasServiceProviderAuth && results.length === 1) {
+      const account_sid = results[0].account_sid;
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]);
+      if (r.length === 1 && r[0].service_provider_sid !== req.user.service_provider_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+    if (req.user.hasAccountAuth && results.length > 1) {
+      return res.status(200).json(results.filter((r) => r.phone_number_sid === sid)[0]);
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/recent-calls.js

@@ -2,6 +2,17 @@ const router = require('express').Router();
 const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
+const {getJaegerTrace} = require('../../utils/jaeger-utils');
+const Account = require('../../models/account');
+const {
+  getS3Object,
+  getGoogleStorageObject,
+  getAzureStorageObject,
+  deleteS3Object,
+  deleteGoogleStorageObject,
+  deleteAzureStorageObject
+} = require('../../utils/storage-utils');
+
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
   if (arr) return arr[1];
@@ -18,7 +29,7 @@ router.get('/', async(req, res) => {
     logger.debug({opts: req.query}, 'GET /RecentCalls');
     const account_sid = parseAccountSid(req.originalUrl);
     const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
-    const {page, count, trunk, direction, days, answered, start, end} = req.query || {};
+    const {page, count, trunk, direction, days, answered, start, end, filter} = req.query || {};
     if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
     if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
 
@@ -33,6 +44,7 @@ router.get('/', async(req, res) => {
         answered,
         start: days ? undefined : start,
         end: days ? undefined : end,
+        filter
       });
       res.status(200).json(data);
     }
@@ -47,6 +59,7 @@ router.get('/', async(req, res) => {
         answered,
         start: days ? undefined : start,
         end: days ? undefined : end,
+        filter
       });
       res.status(200).json(data);
     }
@@ -72,12 +85,12 @@ router.get('/:call_id', async(req, res) => {
   }
 });
 
-router.get('/:call_id/pcap', async(req, res) => {
+router.get('/:call_id/:method/pcap', async(req, res) => {
   const {logger} = req.app.locals;
   try {
     const token = await getHomerApiKey(logger);
     if (!token) return res.sendStatus(400, {msg: 'getHomerApiKey: Failed to get Homer API token; check server config'});
-    const stream = await getHomerPcap(logger, token, [req.params.call_id]);
+    const stream = await getHomerPcap(logger, token, [req.params.call_id], req.params.method);
     if (!stream) {
       logger.info(`getHomerApiKey: unable to get sip traces from Homer for ${req.params.call_id}`);
       return res.sendStatus(404);
@@ -93,4 +106,100 @@ router.get('/:call_id/pcap', async(req, res) => {
   }
 });
 
+router.get('/trace/:trace_id', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {trace_id} = req.params;
+  try {
+    const obj = await getJaegerTrace(logger, trace_id);
+    if (!obj) {
+      logger.info(`/RecentCalls: unable to get spans from jaeger for ${trace_id}`);
+      return res.sendStatus(404);
+    }
+    res.status(200).json(obj.result);
+  } catch (err) {
+    logger.error({err}, `/RecentCalls error retrieving jaeger trace ${trace_id}`);
+    res.sendStatus(500);
+  }
+});
+
+router.get('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+    const getOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+    let stream;
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        stream = await getS3Object(logger, getOptions);
+        break;
+      case 'google':
+        stream = await getGoogleStorageObject(logger, getOptions);
+        break;
+      case 'azure':
+        stream = await getAzureStorageObject(logger, getOptions);
+        break;
+      default:
+        logger.error(`There is no handler for fetching record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.set({
+      'Content-Type': `audio/${format || 'mp3'}`
+    });
+    if (stream) {
+      stream.pipe(res);
+    } else {
+      return res.sendStatus(404);
+    }
+  }  catch (err) {
+    logger.error({err}, ` error retrieving recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
+router.delete('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+
+    const deleteOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        await deleteS3Object(logger, deleteOptions);
+        break;
+      case 'google':
+        await deleteGoogleStorageObject(logger, deleteOptions);
+        break;
+      case 'azure':
+        await deleteAzureStorageObject(logger, deleteOptions);
+        break;
+      default:
+        logger.error(`There is no handler for deleting record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.sendStatus(204);
+  }  catch (err) {
+    logger.error({err}, ` error deleting recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
 module.exports = router;

lib/routes/api/register.js

@@ -4,6 +4,7 @@ const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/er
 const {promisePool} = require('../../db');
 const {doGithubAuth, doGoogleAuth, doLocalAuth} = require('../../utils/oauth-utils');
 const {validateEmail} = require('../../utils/email-utils');
+const {cacheClient} = require('../../helpers');
 const { v4: uuid } = require('uuid');
 const short = require('short-uuid');
 const translator = short();
@@ -15,8 +16,9 @@ const insertUserSql = `INSERT into users
 (user_sid, account_sid, name, email, provider, provider_userid, email_validated) 
 values (?, ?, ?, ?, ?, ?, 1)`;
 const insertUserLocalSql = `INSERT into users 
-(user_sid, account_sid, name, email, email_activation_code, email_validated, provider, hashed_password) 
-values (?, ?, ?, ?, ?, 0, 'local', ?)`;
+(user_sid, account_sid, name, email, email_activation_code, email_validated, provider,
+hashed_password, service_provider_sid) 
+values (?, ?, ?, ?, ?, 0, 'local', ?, ?)`;
 const insertAccountSql = `INSERT into accounts 
 (account_sid, service_provider_sid, name, is_active, webhook_secret, trial_end_date) 
 values (?, ?, ?, ?, ?, CURDATE() + INTERVAL 21 DAY)`;
@@ -35,7 +37,7 @@ const insertSignupHistorySql = `INSERT into signup_history
 values (?, ?)`;
 
 const addLocalUser = async(logger, user_sid, account_sid,
-  name, email, email_activation_code, passwordHash) => {
+  name, email, email_activation_code, passwordHash, service_provider_sid) => {
   const [r] = await promisePool.execute(insertUserLocalSql,
     [
       user_sid,
@@ -43,7 +45,8 @@ const addLocalUser = async(logger, user_sid, account_sid,
       name,
       email,
       email_activation_code,
-      passwordHash
+      passwordHash,
+      service_provider_sid
     ]);
   debug({r}, 'Result from adding user');
 };
@@ -144,7 +147,7 @@ router.post('/', async(req, res) => {
       const user = await doGithubAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from github');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         email_validated: user.email_validated,
         avatar_url: user.avatar_url,
@@ -156,7 +159,7 @@ router.post('/', async(req, res) => {
       const user = await doGoogleAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from google');
       Object.assign(userProfile, {
-        name: user.name || user.email,
+        name: user.email || user.email,
         email: user.email,
         email_validated: user.verified_email,
         picture: user.picture,
@@ -169,7 +172,7 @@ router.post('/', async(req, res) => {
       logger.info({user}, 'retrieved user details for local provider');
       debug({user}, 'retrieved user details for local provider');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         provider: 'local',
         email_activation_code: user.email_activation_code
@@ -279,7 +282,8 @@ router.post('/', async(req, res) => {
         const passwordHash = await generateHashedPassword(req.body.password);
         debug(`hashed password: ${passwordHash}`);
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
-          userProfile.name, userProfile.email, userProfile.email_activation_code, passwordHash);
+          userProfile.name, userProfile.email, userProfile.email_activation_code,
+          passwordHash, req.body.service_provider_sid);
         debug('added local user');
       }
       else {
@@ -292,17 +296,25 @@ router.post('/', async(req, res) => {
       const callStatusSid = uuid();
       const helloWordSid = uuid();
       const dialTimeSid = uuid();
+      const echoSid = uuid();
 
-      /* 3 webhooks */
-      await promisePool.execute(insertWebookSql, [callStatusSid, 'https://public-apps.jambonz.us/call-status', 'POST']);
-      await promisePool.execute(insertWebookSql, [helloWordSid, 'https://public-apps.jambonz.us/hello-world', 'POST']);
-      await promisePool.execute(insertWebookSql, [dialTimeSid, 'https://public-apps.jambonz.us/dial-time', 'POST']);
+      /* 4 webhooks */
+      await promisePool.execute(insertWebookSql,
+        [callStatusSid, 'https://public-apps.jambonz.cloud/call-status', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [helloWordSid, 'https://public-apps.jambonz.cloud/hello-world', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [dialTimeSid, 'https://public-apps.jambonz.cloud/dial-time', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [echoSid, 'https://public-apps.jambonz.cloud/echo', 'POST']);
 
       /* 2 applications */
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'hello world',
         helloWordSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'dial time clock',
         dialTimeSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
+      await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'simple echo test',
+        echoSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
 
       Object.assign(userProfile, {
         pristine: true,
@@ -326,7 +338,7 @@ router.post('/', async(req, res) => {
 
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
           userProfile.name, userProfile.email, userProfile.email_activation_code,
-          passwordHash);
+          passwordHash, req.body.service_provider_sid);
 
         /* note: we deactivate the old user once the new email is validated */
       }
@@ -338,16 +350,21 @@ router.post('/', async(req, res) => {
         /* deactivate the old/replaced user */
         const [r] = await promisePool.execute('DELETE FROM users WHERE user_sid = ?', [user_sid]);
         logger.debug({r}, 'register - removed old user');
+        const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+        await cacheClient.delete(redisKey);
       }
     }
 
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60 ;
     // generate a json web token for this user
     const token = jwt.sign({
       user_sid: userProfile.user_sid,
       account_sid: userProfile.account_sid,
+      service_provider_sid: req.body.service_provider_sid,
+      scope: 'account',
       email: userProfile.email,
       name: userProfile.name
-    }, process.env.JWT_SECRET, { expiresIn: parseInt(process.env.JWT_EXPIRES_IN || 60) * 60 });
+    }, process.env.JWT_SECRET, { expiresIn });
 
     logger.debug({
       user_sid: userProfile.user_sid,
@@ -356,6 +373,13 @@ router.post('/', async(req, res) => {
 
     res.json({jwt: token, ...userProfile});
 
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', userProfile.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
   } catch (err) {
     debug(err, 'Error');
     sysError(logger, res, err);

lib/routes/api/sbcs.js

@@ -2,25 +2,46 @@ const router = require('express').Router();
 const Sbc = require('../../models/sbc');
 const decorate = require('./decorate');
 const sysError = require('../error');
-//const {DbErrorBadRequest} = require('../../utils/errors');
-//const {promisePool} = require('../../db');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
 
-decorate(router, Sbc, ['add', 'delete']);
+const validate = (req, res) => {
+  if (req.user.hasScope('admin')) return;
+  res.status(403).json({
+    status: 'fail',
+    message: 'insufficient privileges'
+  });
+};
+
+const preconditions = {
+  'add': validate,
+  'delete': validate
+};
+
+decorate(router, Sbc, ['add', 'delete'], preconditions);
 
 /* list */
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const service_provider_sid = req.query.service_provider_sid;
-    /*
+    let service_provider_sid = req.query.service_provider_sid;
+
     if (req.user.hasAccountAuth) {
       const [r] = await promisePool.query('SELECT * from accounts WHERE account_sid = ?', req.user.account_sid);
-      if (0 === r.length) throw new Error('invalid account_sid');
+      if (0 === r.length) throw new DbErrorBadRequest('invalid account_sid');
+
       service_provider_sid = r[0].service_provider_sid;
     }
-    if (!service_provider_sid) throw new DbErrorBadRequest('missing service_provider_sid in query');
-    */
-    const results = await Sbc.retrieveAll(service_provider_sid);
+
+    if (req.user.hasServiceProviderAuth) {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
+    /** generally, we have a global set of SBCs that all accounts use.
+     * However, we can have a set of SBCs that are specific for use by a service provider.
+     */
+    let results = await Sbc.retrieveAll(service_provider_sid);
+    if (results.length === 0) results = await Sbc.retrieveAll();
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/service-providers.js

@@ -1,6 +1,6 @@
 const router = require('express').Router();
 const {promisePool} = require('../../db');
-const {DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
+const {DbErrorForbidden} = require('../../utils/errors');
 const Webhook = require('../../models/webhook');
 const ServiceProvider = require('../../models/service-provider');
 const Account = require('../../models/account');
@@ -8,7 +8,11 @@ const VoipCarrier = require('../../models/voip-carrier');
 const Application = require('../../models/application');
 const PhoneNumber = require('../../models/phone-number');
 const ApiKey = require('../../models/api-key');
-const {hasServiceProviderPermissions, parseServiceProviderSid} = require('./utils');
+const {
+  hasServiceProviderPermissions,
+  parseServiceProviderSid,
+  parseVoipCarrierSid,
+} = require('./utils');
 const sysError = require('../error');
 const decorate = require('./decorate');
 const preconditions = {
@@ -35,50 +39,59 @@ function validateAdd(req) {
 }
 
 async function validateRetrieve(req) {
-  const service_provider_sid = parseServiceProviderSid(req);
-  if (req.user.hasScope('admin')) return ;
-  if (req.user.hasScope('service_provider')) {
-    if (service_provider_sid === req.user.service_provider_sid) return ;
+  try {
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider') || req.user.hasScope('account')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  } catch (error) {
+    throw error;
   }
-  if (req.user.hasScope('account')) {
-    /* allow account users to retrieve service provider data from parent SP */
-    const sid = req.user.account_sid;
-    const [r] = await promisePool.execute('SELECT service_provider_sid from accounts WHERE account_sid = ?', [sid]);
-    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) return;
-  }
-  throw new DbErrorForbidden('insufficient permissions to update service provider');
 }
 
 function validateUpdate(req) {
-  if (req.user.hasScope('admin')) return ;
-  if (req.user.hasScope('service_provider')) {
+  try {
     const service_provider_sid = parseServiceProviderSid(req);
-    if (service_provider_sid === req.user.service_provider_sid) return ;
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions to update service provider');
+  } catch (error) {
+    throw error;
   }
-  throw new DbErrorForbidden('insufficient permissions to update service provider');
 }
 
-/* can not delete a service provider if it has any active accounts */
+/* can not delete a service provider if it has any active accounts or users*/
 async function noActiveAccountsOrUsers(req, sid) {
   if (!req.user.hasAdminAuth) {
     throw new DbErrorForbidden('only admin users can delete a service provider');
   }
   const activeAccounts = await ServiceProvider.getForeignKeyReferences('accounts.service_provider_sid', sid);
   const activeUsers = await ServiceProvider.getForeignKeyReferences('users.service_provider_sid', sid);
-  if (activeAccounts > 0 && activeUsers > 0) throw new DbErrorUnprocessableRequest(
-    'cannot delete service provider with active accounts or users'
-  );
+  if (activeAccounts > 0 && activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
 
-  if (activeAccounts > 0) throw new DbErrorUnprocessableRequest('cannot delete service provider with active accounts');
-  if (activeUsers > 0) throw new DbErrorUnprocessableRequest(
-    'cannot delete service provider with active service provider level users'
-  );
+  if (activeAccounts > 0) throw new DbErrorForbidden('insufficient privileges');
+  if (activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
 
   /* ok we can delete -- no active accounts.  remove carriers and speech credentials */
   await promisePool.execute('DELETE from speech_credentials WHERE service_provider_sid = ?', [sid]);
   await promisePool.query(sqlDeleteSipGateways, [sid]);
   await promisePool.query(sqlDeleteSmppGateways, [sid]);
   await promisePool.query('DELETE from voip_carriers WHERE service_provider_sid = ?', [sid]);
+  await promisePool.query('DELETE from api_keys WHERE service_provider_sid = ?', [sid]);
 }
 
 decorate(router, ServiceProvider, ['delete'], preconditions);
@@ -102,6 +115,7 @@ router.get('/:sid/Accounts', async(req, res) => {
     sysError(logger, res, err);
   }
 });
+
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
@@ -161,7 +175,8 @@ router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     validateUpdate(req);
-    const rowsAffected = await VoipCarrier.update(req.params.voip_carrier_sid, req.body);
+    const sid = parseVoipCarrierSid(req);
+    const rowsAffected = await VoipCarrier.update(sid, req.body);
     if (rowsAffected === 0) {
       return res.sendStatus(404);
     }
@@ -191,7 +206,6 @@ router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     validateAdd(req);
-
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
     for (const prop of ['registration_hook']) {
@@ -212,7 +226,6 @@ router.post('/', async(req, res) => {
 /* list */
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
-
   try {
     const results = await ServiceProvider.retrieveAll();
     logger.debug({results, user: req.user}, 'ServiceProvider.retrieveAll');
@@ -231,7 +244,9 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await ServiceProvider.retrieve(req.params.sid);
+    await validateRetrieve(req);
+    const sid = parseServiceProviderSid(req);
+    const results = await ServiceProvider.retrieve(sid);
     if (results.length === 0) return res.status(404).end();
     return res.status(200).json(results[0]);
   }
@@ -242,10 +257,10 @@ router.get('/:sid', async(req, res) => {
 
 /* update */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
     validateUpdate(req);
+    const sid = parseServiceProviderSid(req);
 
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
@@ -255,15 +270,14 @@ router.put('/:sid', async(req, res) => {
           const sid = obj[prop]['webhook_sid'];
           delete obj[prop]['webhook_sid'];
           await Webhook.update(sid, obj[prop]);
-        }
-        else {
+        } else {
           const sid = await Webhook.make(obj[prop]);
           obj[`${prop}_sid`] = sid;
         }
-      }
-      else {
+      } else {
         obj[`${prop}_sid`] = null;
       }
+
       delete obj[prop];
     }
 
@@ -271,6 +285,7 @@ router.put('/:sid', async(req, res) => {
     if (rowsAffected === 0) {
       return res.status(404).end();
     }
+
     res.status(204).end();
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/signin.js

@@ -3,10 +3,17 @@ const router = require('express').Router();
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {promisePool} = require('../../db');
 const {verifyPassword} = require('../../utils/password-utils');
+const {cacheClient} = require('../../helpers');
 const jwt = require('jsonwebtoken');
 const sysError = require('../error');
+const retrievePermissionsSql = `
+SELECT p.name 
+FROM permissions p, user_permissions up 
+WHERE up.permission_sid = p.permission_sid 
+AND up.user_sid = ? 
+`;
 
-const validateRequest = async(req) => {
+const validateRequest = (req) => {
   const {email, password} = req.body || {};
 
   /* check required properties are there */
@@ -52,6 +59,7 @@ router.post('/', async(req, res) => {
       email: user.email,
       phone: user.phone,
       account_sid: user.account_sid,
+      service_provider_sid: a[0].service_provider_sid,
       force_change: !!user.force_change,
       provider: user.provider,
       provider_userid: user.provider_userid,
@@ -64,11 +72,22 @@ router.post('/', async(req, res) => {
       pristine: false
     });
 
+    const [p] = await promisePool.query(retrievePermissionsSql, user.user_sid);
+    const permissions = p.map((x) => x.name);
+
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60;
     // generate a json web token for this session
-    const token = jwt.sign({
+    const payload = {
+      scope: 'account',
+      permissions,
       user_sid: userProfile.user_sid,
-      account_sid: userProfile.account_sid
-    }, process.env.JWT_SECRET, { expiresIn: parseInt(process.env.JWT_EXPIRES_IN || 60) * 60 });
+      account_sid: userProfile.account_sid,
+      service_provider_sid: userProfile.service_provider_sid
+    };
+    const token = jwt.sign(payload,
+      process.env.JWT_SECRET,
+      { expiresIn }
+    );
 
     logger.debug({
       user_sid: userProfile.user_sid,
@@ -76,6 +95,14 @@ router.post('/', async(req, res) => {
     }, 'generated jwt');
 
     res.json({jwt: token, ...userProfile});
+
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', userProfile.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/sip-gateways.js

@@ -1,11 +1,47 @@
 const router = require('express').Router();
 const SipGateway = require('../../models/sip-gateway');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
+//const {parseSipGatewaySid} = require('./utils');
 const decorate = require('./decorate');
 const sysError = require('../error');
 
+const checkUserScope = async(req, voip_carrier_sid) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+  if (req.user.hasAccountAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+
+    if ((!carrier.service_provider_sid || carrier.service_provider_sid === req.user.service_provider_sid) &&
+      (!carrier.account_sid || carrier.account_sid === req.user.account_sid)) {
+
+      if (req.method !== 'GET' && !carrier.account_sid) {
+        throw new DbErrorForbidden('insufficient privileges');
+      }
+
+      return;
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) {
+      throw new DbErrorBadRequest('invalid voip_carrier_sid');
+    }
+
+    if (carrier.service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+  }
+  throw new DbErrorForbidden('insufficient privileges');
+};
+
 const validate = async(req, sid) => {
-  const {lookupCarrierBySid, lookupSipGatewayBySid} = req.app.locals;
+  const {lookupSipGatewayBySid} = req.app.locals;
+  const {netmask} = req.body;
   let voip_carrier_sid;
 
   if (sid) {
@@ -17,13 +53,13 @@ const validate = async(req, sid) => {
     voip_carrier_sid = req.body.voip_carrier_sid;
     if (!voip_carrier_sid) throw new DbErrorBadRequest('missing voip_carrier_sid');
   }
-  if (req.hasAccountAuth) {
-    const carrier = await lookupCarrierBySid(voip_carrier_sid);
-    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
-    if (carrier.account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('user can not add gateway for voip_carrier belonging to other account');
-    }
+  if (netmask &&
+    process.env.JAMBONZ_MIN_GATEWAY_NETMASK &&
+    parseInt(netmask) < process.env.JAMBONZ_MIN_GATEWAY_NETMASK) {
+    throw new DbErrorBadRequest(
+      `netmask required to have value equal or greater than ${process.env.JAMBONZ_MIN_GATEWAY_NETMASK}`);
   }
+  await checkUserScope(req, voip_carrier_sid);
 };
 
 const preconditions = {
@@ -39,6 +75,7 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   const voip_carrier_sid = req.query.voip_carrier_sid;
   try {
+    await checkUserScope(req, voip_carrier_sid);
     if (!voip_carrier_sid) {
       logger.info('GET /SipGateways missing voip_carrier_sid param');
       return res.status(400).json({message: 'missing voip_carrier_sid query param'});

lib/routes/api/sip-realm.js

@@ -31,6 +31,7 @@ router.post('/:sip_realm', async(req, res) => {
       const [sbcs] = await promisePool.query('SELECT ipv4 from sbc_addresses');
       if (sbcs.length === 0) throw new Error('no SBC addresses provisioned in the database!');
       const ips = sbcs.map((s) => s.ipv4);
+      const uniqueIps = [...new Set(ips)];
 
       /* retrieve existing dns records */
       const [old_recs] = await promisePool.query('SELECT record_id from dns_records WHERE account_sid = ?',
@@ -48,7 +49,7 @@ router.post('/:sip_realm', async(req, res) => {
       }
 
       /* add the dns records */
-      const records = await createDnsRecords(logger, domain, subdomain, ips);
+      const records = await createDnsRecords(logger, domain, subdomain, uniqueIps);
       if (!records) throw new Error(`failure updating dns records for ${sip_realm}`);
       const values = records.map((r) => {
         return `('${uuid()}', '${account_sid}', '${r.type}', ${r.id})`;

lib/routes/api/smpp-gateways.js

@@ -1,11 +1,38 @@
 const router = require('express').Router();
 const SmppGateway = require('../../models/smpp-gateway');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
 const decorate = require('./decorate');
 const sysError = require('../error');
 
+const checkUserScope = async(req, voip_carrier_sid) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+
+  if (req.user.hasAccountAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+
+    if ((!carrier.service_provider_sid || carrier.service_provider_sid === req.user.service_provider_sid) &&
+      (!carrier.account_sid || carrier.account_sid === req.user.account_sid)) {
+      return;
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+    if (carrier.service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+  }
+  throw new DbErrorForbidden('insufficient privileges');
+};
+
 const validate = async(req, sid) => {
-  const {lookupCarrierBySid, lookupSmppGatewayBySid} = req.app.locals;
+  const {lookupSmppGatewayBySid} = req.app.locals;
   let voip_carrier_sid;
 
   if (sid) {
@@ -17,13 +44,8 @@ const validate = async(req, sid) => {
     voip_carrier_sid = req.body.voip_carrier_sid;
     if (!voip_carrier_sid) throw new DbErrorBadRequest('missing voip_carrier_sid');
   }
-  if (req.hasAccountAuth) {
-    const carrier = await lookupCarrierBySid(voip_carrier_sid);
-    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
-    if (carrier.account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('user can not add gateway for voip_carrier belonging to other account');
-    }
-  }
+
+  await checkUserScope(req, voip_carrier_sid);
 };
 
 const preconditions = {
@@ -39,6 +61,7 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   const voip_carrier_sid = req.query.voip_carrier_sid;
   try {
+    await checkUserScope(req, voip_carrier_sid);
     if (!voip_carrier_sid) {
       logger.info('GET /SmppGateways missing voip_carrier_sid param');
       return res.status(400).json({message: 'missing voip_carrier_sid query param'});

lib/routes/api/sms-inbound.js

@@ -14,20 +14,14 @@ const getFsUrl = async(logger, retrieveSet, setName, provider) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/messaging/${provider}`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/messaging/${provider}`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
-};
-
 const doSendResponse = async(res, respondFn, body) => {
   if (typeof respondFn === 'number') res.sendStatus(respondFn);
   else if (typeof respondFn !== 'function') res.sendStatus(200);
@@ -44,7 +38,7 @@ router.post('/:provider', async(req, res) => {
     lookupAppByPhoneNumber,
     logger
   } = req.app.locals;
-  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:active-fs`;
+  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:fs-service-url`;
   logger.debug({path: req.path, body: req.body}, 'incomingSMS from carrier');
 
   // search for provider module

lib/routes/api/speech-credentials.js

@@ -4,8 +4,15 @@ const Account = require('../../models/account');
 const SpeechCredential = require('../../models/speech-credential');
 const sysError = require('../error');
 const {decrypt, encrypt} = require('../../utils/encrypt-decrypt');
-const {parseAccountSid, parseServiceProviderSid} = require('./utils');
-const {DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {parseAccountSid, parseServiceProviderSid, parseSpeechCredentialSid} = require('./utils');
+const {decryptCredential, testWhisper, testDeepgramTTS,
+  getLanguagesAndVoicesForVendor,
+  testPlayHT,
+  testRimelabs,
+  testVerbioTts,
+  testVerbioStt,
+  testSpeechmaticsStt} = require('../../utils/speech-utils');
+const {DbErrorUnprocessableRequest, DbErrorForbidden, DbErrorBadRequest} = require('../../utils/errors');
 const {
   testGoogleTts,
   testGoogleStt,
@@ -19,18 +26,86 @@ const {
   testDeepgramStt,
   testSonioxStt,
   testIbmTts,
-  testIbmStt
+  testIbmStt,
+  testElevenlabs,
+  testAssemblyStt
 } = require('../../utils/speech-utils');
+const {promisePool} = require('../../db');
 
-const obscureKey = (key) => {
-  const key_spoiler_length = 6;
-  const key_spoiler_char = 'X';
+const validateAdd = async(req) => {
+  const account_sid = parseAccountSid(req);
+  const service_provider_sid = parseServiceProviderSid(req);
 
-  if (key.length <= key_spoiler_length) {
-    return key;
+  if (service_provider_sid) {
+    if (req.user.hasServiceProviderAuth && service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+    if (req.user.hasAccountAuth && service_provider_sid !== req.user.service_provider_sid &&
+      req.body.account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
   }
 
-  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
+  if (account_sid) {
+    if (req.user.hasAccountAuth && account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+    );
+
+    if (req.user.hasServiceProviderAuth && r[0].service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+  return;
+};
+
+const validateRetrieveUpdateDelete = async(req, speech_credentials) => {
+  if (req.user.hasServiceProviderAuth && speech_credentials[0].service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+
+  if (req.user.hasAccountAuth && speech_credentials[0].account_sid !== req.user.account_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+  return;
+};
+
+const validateRetrieveList = async(req) => {
+  const service_provider_sid = parseServiceProviderSid(req);
+
+  if (service_provider_sid) {
+    if ((req.user.hasServiceProviderAuth || req.user.hasAccountAuth) &&
+     service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+  return;
+};
+
+const validateTest = async(req, speech_credentials) => {
+  if (req.user.hasAdminAuth) {
+    return;
+  }
+
+  if (!req.user.hasAdminAuth && speech_credentials.service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+
+  if (speech_credentials.service_provider_sid === req.user.service_provider_sid) {
+    if (req.user.hasServiceProviderAuth) {
+      return;
+    }
+
+    if (req.user.hasAccountAuth && (!speech_credentials.account_sid ||
+       speech_credentials.account_sid === req.user.account_sid)) {
+      return;
+    }
+
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
 };
 
 const encryptCredential = (obj) => {
@@ -41,19 +116,38 @@ const encryptCredential = (obj) => {
     secret_access_key,
     aws_region,
     api_key,
+    role_arn,
     region,
     client_id,
+    client_secret,
     secret,
+    nuance_tts_uri,
+    nuance_stt_uri,
+    speechmatics_stt_uri,
+    deepgram_stt_uri,
+    deepgram_stt_use_tls,
+    deepgram_tts_uri,
     use_custom_tts,
     custom_tts_endpoint,
+    custom_tts_endpoint_url,
     use_custom_stt,
     custom_stt_endpoint,
+    custom_stt_endpoint_url,
     tts_api_key,
     tts_region,
     stt_api_key,
     stt_region,
     riva_server_uri,
-    instance_id
+    instance_id,
+    custom_stt_url,
+    custom_tts_url,
+    auth_token = '',
+    cobalt_server_uri,
+    model_id,
+    user_id,
+    voice_engine,
+    engine_version,
+    options
   } = obj;
 
   switch (vendor) {
@@ -69,22 +163,33 @@ const encryptCredential = (obj) => {
       return encrypt(service_key);
 
     case 'aws':
-      assert(access_key_id, 'invalid aws speech credential: access_key_id is required');
-      assert(secret_access_key, 'invalid aws speech credential: secret_access_key is required');
-      assert(aws_region, 'invalid aws speech credential: aws_region is required');
-      const awsData = JSON.stringify({aws_region, access_key_id, secret_access_key});
+      // AWS polly can work for 3 types of credentials:
+      // 1/ access_key_id and secret_access_key
+      // 2/ RoleArn Assume role
+      // 3/ RoleArn assigned to instance profile where will run this application
+      const awsData = JSON.stringify(
+        {
+          aws_region,
+          ...(access_key_id && {access_key_id}),
+          ...(secret_access_key && {secret_access_key}),
+          ...(role_arn && {role_arn}),
+        });
       return encrypt(awsData);
 
     case 'microsoft':
-      assert(region, 'invalid azure speech credential: region is required');
-      assert(api_key, 'invalid azure speech credential: api_key is required');
+      if (!custom_tts_endpoint_url && !custom_stt_endpoint_url) {
+        assert(region, 'invalid azure speech credential: region is required');
+        assert(api_key, 'invalid azure speech credential: api_key is required');
+      }
       const azureData = JSON.stringify({
-        region,
-        api_key,
+        ...(region && {region}),
+        ...(api_key && {api_key}),
         use_custom_tts,
         custom_tts_endpoint,
+        custom_tts_endpoint_url,
         use_custom_stt,
-        custom_stt_endpoint
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
       });
       return encrypt(azureData);
 
@@ -94,14 +199,18 @@ const encryptCredential = (obj) => {
       return encrypt(wsData);
 
     case 'nuance':
-      assert(client_id, 'invalid nuance speech credential: client_id is required');
-      assert(secret, 'invalid nuance speech credential: secret is required');
-      const nuanceData = JSON.stringify({client_id, secret});
+      const checked = (client_id && secret) || (nuance_tts_uri || nuance_stt_uri);
+      assert(checked, 'invalid nuance speech credential: either entered client id and\
+        secret or entered a nuance_tts_uri or nuance_stt_uri');
+      const nuanceData = JSON.stringify({client_id, secret, nuance_tts_uri, nuance_stt_uri});
       return encrypt(nuanceData);
 
     case 'deepgram':
-      assert(api_key, 'invalid deepgram speech credential: api_key is required');
-      const deepgramData = JSON.stringify({api_key});
+      // API key is optional if onprem
+      if (!deepgram_stt_uri || !deepgram_tts_uri) {
+        assert(api_key, 'invalid deepgram speech credential: api_key is required');
+      }
+      const deepgramData = JSON.stringify({api_key, deepgram_stt_uri, deepgram_stt_use_tls, deepgram_tts_uri});
       return encrypt(deepgramData);
 
     case 'ibm':
@@ -118,33 +227,101 @@ const encryptCredential = (obj) => {
       const sonioxData = JSON.stringify({api_key});
       return encrypt(sonioxData);
 
+    case 'cobalt':
+      assert(cobalt_server_uri, 'invalid cobalt speech credential: cobalt_server_uri is required');
+      const cobaltData = JSON.stringify({cobalt_server_uri});
+      return encrypt(cobaltData);
+
+    case 'elevenlabs':
+      assert(api_key, 'invalid elevenLabs speech credential: api_key is required');
+      assert(model_id, 'invalid elevenLabs speech credential: model_id is required');
+      const elevenlabsData = JSON.stringify({api_key, model_id, options});
+      return encrypt(elevenlabsData);
+
+    case 'speechmatics':
+      assert(api_key, 'invalid speechmatics speech credential: api_key is required');
+      assert(speechmatics_stt_uri, 'invalid speechmatics speech credential: speechmatics_stt_uri is required');
+      const speechmaticsData = JSON.stringify({api_key, speechmatics_stt_uri, options});
+      return encrypt(speechmaticsData);
+
+    case 'playht':
+      assert(api_key, 'invalid playht speech credential: api_key is required');
+      assert(user_id, 'invalid playht speech credential: user_id is required');
+      assert(voice_engine, 'invalid voice_engine speech credential: voice_engine is required');
+      const playhtData = JSON.stringify({api_key, user_id, voice_engine, options});
+      return encrypt(playhtData);
+
+    case 'rimelabs':
+      assert(api_key, 'invalid rimelabs speech credential: api_key is required');
+      assert(model_id, 'invalid rimelabs speech credential: model_id is required');
+      const rimelabsData = JSON.stringify({api_key, model_id, options});
+      return encrypt(rimelabsData);
+
+    case 'assemblyai':
+      assert(api_key, 'invalid assemblyai speech credential: api_key is required');
+      const assemblyaiData = JSON.stringify({api_key});
+      return encrypt(assemblyaiData);
+
+    case 'whisper':
+      assert(api_key, 'invalid whisper speech credential: api_key is required');
+      assert(model_id, 'invalid whisper speech credential: model_id is required');
+      const whisperData = JSON.stringify({api_key, model_id});
+      return encrypt(whisperData);
+
+    case 'verbio':
+      assert(engine_version, 'invalid verbio speech credential: client_id is required');
+      assert(client_id, 'invalid verbio speech credential: client_id is required');
+      assert(client_secret, 'invalid verbio speech credential: secret is required');
+      const verbioData = JSON.stringify({client_id, client_secret, engine_version});
+      return encrypt(verbioData);
+
     default:
-      assert(false, `invalid or missing vendor: ${vendor}`);
+      if (vendor.startsWith('custom:')) {
+        const customData = JSON.stringify({auth_token, custom_stt_url, custom_tts_url});
+        return encrypt(customData);
+      }
+      else assert(false, `invalid or missing vendor: ${vendor}`);
   }
 };
 
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
-  const {
-    use_for_stt,
-    use_for_tts,
-    vendor,
-  } = req.body;
-  const account_sid = req.user.account_sid || req.body.account_sid;
-  const service_provider_sid = req.user.service_provider_sid ||
-  req.body.service_provider_sid || parseServiceProviderSid(req);
-  if (!account_sid) {
-    if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
-      logger.error('POST /SpeechCredentials invalid credentials');
-      return res.sendStatus(403);
-    }
-  }
+
   try {
+    const {
+      use_for_stt,
+      use_for_tts,
+      vendor,
+      label
+    } = req.body;
+    const account_sid = req.user.account_sid || req.body.account_sid;
+    const service_provider_sid = req.user.service_provider_sid ||
+    req.body.service_provider_sid || parseServiceProviderSid(req);
+
+    await validateAdd(req);
+
+    if (!account_sid) {
+      if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
+        logger.error('POST /SpeechCredentials invalid credentials');
+        return res.sendStatus(403);
+      }
+    }
+
+    // Check if vendor and label is already used for account or SP
+    if (label) {
+      const existingSpeech = await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+        service_provider_sid, account_sid, vendor, label);
+      if (existingSpeech.length > 0) {
+        throw new DbErrorUnprocessableRequest(`Label ${label} is already in use for another speech credential`);
+      }
+    }
+
     const encrypted_credential = encryptCredential(req.body);
     const uuid = await SpeechCredential.make({
       account_sid,
       service_provider_sid,
       vendor,
+      label,
       use_for_tts,
       use_for_stt,
       credential: encrypted_credential
@@ -159,10 +336,14 @@ router.post('/', async(req, res) => {
  * retrieve all speech credentials for an account
  */
 router.get('/', async(req, res) => {
-  const account_sid = parseAccountSid(req) || req.user.account_sid;
-  const service_provider_sid = parseServiceProviderSid(req) || req.user.service_provider_sid;
   const logger = req.app.locals.logger;
+
   try {
+    const account_sid = parseAccountSid(req) ? parseAccountSid(req) : req.user.account_sid;
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    await validateRetrieveList(req);
+
     const credsAccount = account_sid ? await SpeechCredential.retrieveAll(account_sid) : [];
     const credsSP = service_provider_sid ?
       await SpeechCredential.retrieveAllForSP(service_provider_sid) :
@@ -176,59 +357,16 @@ router.get('/', async(req, res) => {
 
     res.status(200).json(creds.map((c) => {
       const {credential, ...obj} = c;
-      if ('google' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        const key_header = '-----BEGIN PRIVATE KEY-----\n';
-        const obscured = {
-          ...o,
-          private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
-        };
-        obj.service_key = obscured;
-      }
-      else if ('aws' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.access_key_id = o.access_key_id;
-        obj.secret_access_key = obscureKey(o.secret_access_key);
-        obj.aws_region = o.aws_region;
-        logger.info({obj, o}, 'retrieving aws speech credential');
-      }
-      else if ('microsoft' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-        obj.region = o.region;
-        obj.use_custom_tts = o.use_custom_tts;
-        obj.custom_tts_endpoint = o.custom_tts_endpoint;
-        obj.use_custom_stt = o.use_custom_stt;
-        obj.custom_stt_endpoint = o.custom_stt_endpoint;
-        logger.info({obj, o}, 'retrieving azure speech credential');
-      }
-      else if ('wellsaid' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if ('nuance' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.client_id = o.client_id;
-        obj.secret = obscureKey(o.secret);
-      }
-      else if ('deepgram' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
-      }
-      else if ('ibm' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.tts_api_key = obscureKey(o.tts_api_key);
-        obj.tts_region = o.tts_region;
-        obj.stt_api_key = obscureKey(o.stt_api_key);
-        obj.stt_region = o.stt_region;
-        obj.instance_id = o.instance_id;
-      } else if ('nvidia' == obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.riva_server_uri = o.riva_server_uri;
-      }
-      else if ('soniox' === obj.vendor) {
-        const o = JSON.parse(decrypt(credential));
-        obj.api_key = obscureKey(o.api_key);
+
+      decryptCredential(obj, credential, logger);
+
+      if (req.user.hasAccountAuth && obj.account_sid === null) {
+        delete obj.api_key;
+        delete obj.secret_access_key;
+        delete obj.secret;
+        delete obj.auth_token;
+        delete obj.stt_api_key;
+        delete obj.tts_api_key;
       }
       return obj;
     }));
@@ -241,60 +379,26 @@ router.get('/', async(req, res) => {
  * retrieve a specific speech credential
  */
 router.get('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
     const cred = await SpeechCredential.retrieve(sid);
     if (0 === cred.length) return res.sendStatus(404);
+
+    await validateRetrieveUpdateDelete(req, cred);
+
     const {credential, ...obj} = cred[0];
-    if ('google' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      const key_header = '-----BEGIN PRIVATE KEY-----\n';
-      const obscured = {
-        ...o,
-        private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
-      };
-      obj.service_key = JSON.stringify(obscured);
-    }
-    else if ('aws' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.access_key_id = o.access_key_id;
-      obj.secret_access_key = obscureKey(o.secret_access_key);
-      obj.aws_region = o.aws_region;
-    }
-    else if ('microsoft' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-      obj.region = o.region;
-      obj.use_custom_tts = o.use_custom_tts;
-      obj.custom_tts_endpoint = o.custom_tts_endpoint;
-      obj.use_custom_stt = o.use_custom_stt;
-      obj.custom_stt_endpoint = o.custom_stt_endpoint;
-    }
-    else if ('wellsaid' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if ('nuance' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.client_id = o.client_id;
-      obj.secret = obscureKey(o.secret);
-    }
-    else if ('deepgram' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.api_key = obscureKey(o.api_key);
-    }
-    else if ('ibm' === obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.tts_api_key = obscureKey(o.tts_api_key);
-      obj.tts_region = o.tts_region;
-      obj.stt_api_key = obscureKey(o.stt_api_key);
-      obj.stt_region = o.stt_region;
-      obj.instance_id = o.instance_id;
-    } else if ('nvidia' == obj.vendor) {
-      const o = JSON.parse(decrypt(credential));
-      obj.riva_server_uri = o.riva_server_uri;
+    decryptCredential(obj, credential, logger);
+
+    if (req.user.hasAccountAuth && obj.account_sid === null) {
+      delete obj.api_key;
+      delete obj.secret_access_key;
+      delete obj.secret;
+      delete obj.auth_token;
+      delete obj.stt_api_key;
+      delete obj.tts_api_key;
     }
+
     res.status(200).json(obj);
   } catch (err) {
     sysError(logger, res, err);
@@ -305,9 +409,11 @@ router.get('/:sid', async(req, res) => {
  * delete a speech credential
  */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
+    const cred = await SpeechCredential.retrieve(sid);
+    await validateRetrieveUpdateDelete(req, cred);
     const count = await SpeechCredential.remove(sid);
     if (0 === count) return res.sendStatus(404);
     res.sendStatus(204);
@@ -321,10 +427,11 @@ router.delete('/:sid', async(req, res) => {
  * update a speech credential -- we only allow use_for_tts and use_for_stt to be updated
  */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
-    const {use_for_tts, use_for_stt, region, aws_region, stt_region, tts_region, riva_server_uri} = req.body;
+    const sid = parseSpeechCredentialSid(req);
+    const {use_for_tts, use_for_stt, region, aws_region, stt_region, tts_region,
+      riva_server_uri, nuance_tts_uri, nuance_stt_uri} = req.body;
     if (typeof use_for_tts === 'undefined' && typeof use_for_stt === 'undefined') {
       throw new DbErrorUnprocessableRequest('use_for_tts and use_for_stt are the only updateable fields');
     }
@@ -339,14 +446,29 @@ router.put('/:sid', async(req, res) => {
     /* update the credential if provided */
     try {
       const cred = await SpeechCredential.retrieve(sid);
+
+      await validateRetrieveUpdateDelete(req, cred);
+
       if (1 === cred.length) {
         const {credential, vendor} = cred[0];
         const o = JSON.parse(decrypt(credential));
         const {
           use_custom_tts,
           custom_tts_endpoint,
+          custom_tts_endpoint_url,
           use_custom_stt,
-          custom_stt_endpoint
+          custom_stt_endpoint,
+          custom_stt_endpoint_url,
+          custom_stt_url,
+          custom_tts_url,
+          cobalt_server_uri,
+          model_id,
+          voice_engine,
+          options,
+          deepgram_stt_uri,
+          deepgram_stt_use_tls,
+          deepgram_tts_uri,
+          engine_version
         } = req.body;
 
         const newCred = {
@@ -356,11 +478,25 @@ router.put('/:sid', async(req, res) => {
           aws_region,
           use_custom_tts,
           custom_tts_endpoint,
+          custom_tts_endpoint_url,
           use_custom_stt,
           custom_stt_endpoint,
+          custom_stt_endpoint_url,
           stt_region,
           tts_region,
-          riva_server_uri
+          riva_server_uri,
+          nuance_stt_uri,
+          nuance_tts_uri,
+          custom_stt_url,
+          custom_tts_url,
+          cobalt_server_uri,
+          model_id,
+          voice_engine,
+          options,
+          deepgram_stt_uri,
+          deepgram_stt_use_tls,
+          deepgram_tts_uri,
+          engine_version
         };
         logger.info({o, newCred}, 'updating speech credential with this new credential');
         obj.credential = encryptCredential(newCred);
@@ -389,12 +525,15 @@ router.put('/:sid', async(req, res) => {
  * Test a credential
  */
 router.get('/:sid/test', async(req, res) => {
-  const sid = req.params.sid;
-  const logger = req.app.locals.logger;
+  const {logger, synthAudio, getVerbioAccessToken} = req.app.locals;
   try {
+    const sid = parseSpeechCredentialSid(req);
     const creds = await SpeechCredential.retrieve(sid);
+
     if (!creds || 0 === creds.length) return res.sendStatus(404);
 
+    await validateTest(req, creds[0]);
+
     const cred = creds[0];
     const credential = JSON.parse(decrypt(cred.credential));
     const results = {
@@ -412,7 +551,8 @@ router.get('/:sid/test', async(req, res) => {
 
       if (cred.use_for_tts) {
         try {
-          await testGoogleTts(logger, credential);
+          const {getTtsVoices} = req.app.locals;
+          await testGoogleTts(logger, getTtsVoices, credential);
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
         } catch (err) {
@@ -433,11 +573,13 @@ router.get('/:sid/test', async(req, res) => {
       }
     }
     else if (cred.vendor === 'aws') {
+      const {getTtsVoices, getAwsAuthToken} = req.app.locals;
       if (cred.use_for_tts) {
         try {
-          await testAwsTts(logger, {
+          await testAwsTts(logger, getTtsVoices, {
             accessKeyId: credential.access_key_id,
             secretAccessKey: credential.secret_access_key,
+            roleArn: credential.role_arn,
             region: credential.aws_region || process.env.AWS_REGION
           });
           results.tts.status = 'ok';
@@ -449,9 +591,10 @@ router.get('/:sid/test', async(req, res) => {
       }
       if (cred.use_for_stt) {
         try {
-          await testAwsStt(logger, {
+          await testAwsStt(logger, getAwsAuthToken, {
             accessKeyId: credential.access_key_id,
             secretAccessKey: credential.secret_access_key,
+            roleArn: credential.role_arn,
             region: credential.aws_region || process.env.AWS_REGION
           });
           results.stt.status = 'ok';
@@ -468,18 +611,22 @@ router.get('/:sid/test', async(req, res) => {
         region,
         use_custom_tts,
         custom_tts_endpoint,
+        custom_tts_endpoint_url,
         use_custom_stt,
-        custom_stt_endpoint
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
       } = credential;
       if (cred.use_for_tts) {
         try {
-          await testMicrosoftTts(logger, {
+          await testMicrosoftTts(logger, synthAudio, {
             api_key,
             region,
             use_custom_tts,
             custom_tts_endpoint,
+            custom_tts_endpoint_url,
             use_custom_stt,
-            custom_stt_endpoint
+            custom_stt_endpoint,
+            custom_stt_endpoint_url
           });
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
@@ -490,7 +637,7 @@ router.get('/:sid/test', async(req, res) => {
       }
       if (cred.use_for_stt) {
         try {
-          await testMicrosoftStt(logger, {api_key, region});
+          await testMicrosoftStt(logger, {api_key, region, use_custom_stt, custom_stt_endpoint_url});
           results.stt.status = 'ok';
           SpeechCredential.sttTestResult(sid, true);
         } catch (err) {
@@ -517,13 +664,16 @@ router.get('/:sid/test', async(req, res) => {
 
       const {
         client_id,
-        secret
+        secret,
+        nuance_tts_uri,
+        nuance_stt_uri
       } = credential;
       if (cred.use_for_tts) {
         try {
           await testNuanceTts(logger, getTtsVoices, {
             client_id,
-            secret
+            secret,
+            nuance_tts_uri
           });
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
@@ -538,7 +688,7 @@ router.get('/:sid/test', async(req, res) => {
       }
       if (cred.use_for_stt) {
         try {
-          await testNuanceStt(logger, {client_id, secret});
+          await testNuanceStt(logger, {client_id, secret, nuance_stt_uri});
           results.stt.status = 'ok';
           SpeechCredential.sttTestResult(sid, true);
         } catch (err) {
@@ -546,10 +696,19 @@ router.get('/:sid/test', async(req, res) => {
           SpeechCredential.sttTestResult(sid, false);
         }
       }
-    }
-    else if (cred.vendor === 'deepgram') {
+    } else if (cred.vendor === 'deepgram') {
       const {api_key} = credential;
-      if (cred.use_for_stt) {
+      if (cred.use_for_tts) {
+        try {
+          await testDeepgramTTS(logger, synthAudio, credential);
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+      if (cred.use_for_stt && api_key) {
         try {
           await testDeepgramStt(logger, {api_key});
           results.stt.status = 'ok';
@@ -592,7 +751,8 @@ router.get('/:sid/test', async(req, res) => {
           SpeechCredential.sttTestResult(sid, false);
         }
       }
-    } else if (cred.vendor === 'soniox') {
+    }
+    else if (cred.vendor === 'soniox') {
       const {api_key} = credential;
       if (cred.use_for_stt) {
         try {
@@ -604,9 +764,135 @@ router.get('/:sid/test', async(req, res) => {
           SpeechCredential.sttTestResult(sid, false);
         }
       }
+    } else if (cred.vendor === 'elevenlabs') {
+      const {api_key, model_id} = credential;
+      if (cred.use_for_tts) {
+        try {
+          await testElevenlabs(logger, {api_key, model_id});
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'speechmatics') {
+      const {api_key} = credential;
+      if (cred.use_for_stt) {
+        try {
+          await testSpeechmaticsStt(logger, {api_key});
+          results.stt.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'playht') {
+      if (cred.use_for_tts) {
+        try {
+          await testPlayHT(logger, synthAudio, credential);
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          let reason = err.message;
+          // if error is from bent, let get the body
+          try {
+            reason = await err.text();
+          } catch {}
+          results.tts = {status: 'fail', reason};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'rimelabs') {
+      if (cred.use_for_tts) {
+        try {
+          await testRimelabs(logger, synthAudio, credential);
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'assemblyai') {
+      const {api_key} = credential;
+      if (cred.use_for_stt) {
+        try {
+          await testAssemblyStt(logger, {api_key});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'whisper') {
+      if (cred.use_for_tts) {
+        try {
+          await testWhisper(logger, synthAudio, credential);
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+    } else if (cred.vendor === 'verbio') {
+      if (cred.use_for_tts) {
+        try {
+          await testVerbioTts(logger, synthAudio, credential);
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+      if (cred.use_for_stt) {
+        try {
+          await testVerbioStt(logger, getVerbioAccessToken, credential);
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
     }
 
     res.status(200).json(results);
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/**
+ * Fetch speech voices and languages
+ */
+
+router.get('/speech/supportedLanguagesAndVoices', async(req, res) => {
+  const {logger, getTtsVoices} = req.app.locals;
+  try {
+    const {vendor, label, create_new} = req.query;
+    if (!vendor) {
+      throw new DbErrorBadRequest('vendor is required');
+    }
+    const account_sid = req.user.account_sid || req.body.account_sid;
+    const service_provider_sid = req.user.service_provider_sid ||
+      req.body.service_provider_sid || parseServiceProviderSid(req);
+
+    const credentials = create_new ? null : await SpeechCredential.getSpeechCredentialsByVendorAndLabel(
+      service_provider_sid, account_sid, vendor, label);
+    const tmp = credentials && credentials.length > 0 ? credentials[0] : null;
+    const cred = tmp ? JSON.parse(decrypt(tmp.credential)) : null;
+    try {
+      const data = await getLanguagesAndVoicesForVendor(logger, vendor, cred, getTtsVoices);
+      res.status(200).json(data);
+    } catch (err) {
+      throw new DbErrorUnprocessableRequest(err.message);
+    }
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/system-information.js

@@ -0,0 +1,14 @@
+const router = require('express').Router();
+const SystemInformation = require('../../models/system-information');
+
+router.post('/', async(req, res) => {
+  const sysInfo = await SystemInformation.add(req.body);
+  res.status(201).json(sysInfo);
+});
+
+router.get('/', async(req, res) => {
+  const [sysInfo] = await SystemInformation.retrieveAll();
+  res.status(200).json(sysInfo || {});
+});
+
+module.exports = router;

lib/routes/api/tts-cache.js

@@ -0,0 +1,137 @@
+const router = require('express').Router();
+const {
+  parseAccountSid
+} = require('./utils');
+const SpeechCredential = require('../../models/speech-credential');
+const fs = require('fs');
+const { v4: uuidv4 } = require('uuid');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const Account = require('../../models/account');
+const sysError = require('../error');
+const { getSpeechCredential, decryptCredential } = require('../../utils/speech-utils');
+const PCMToMP3Encoder = require('../../record/encoder');
+const { pipeline } = require('stream');
+
+router.delete('/', async(req, res) => {
+  const {purgeTtsCache} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  if (account_sid) {
+    await purgeTtsCache({account_sid});
+  } else {
+    await purgeTtsCache();
+  }
+  res.sendStatus(204);
+});
+
+router.get('/', async(req, res) => {
+  const {getTtsSize} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  let size = 0;
+  if (account_sid) {
+    size = await getTtsSize(`tts:${account_sid}:*`);
+  } else {
+    size = await getTtsSize();
+  }
+  res.status(200).json({size});
+});
+
+router.post('/Synthesize', async(req, res) => {
+  const {logger, synthAudio} = req.app.locals;
+  try {
+    const accountSid = parseAccountSid(req);
+    const body = req.body;
+    const encodingMp3 = req.body.encodingMp3 || false;
+    if (!body.speech_credential_sid || !body.text || !body.language || !body.voice) {
+      throw new DbErrorBadRequest('speech_credential_sid, text, language, voice are all required');
+    }
+
+    const result = await Account.retrieve(accountSid);
+    if (!result || result.length === 0 || !result[0].is_active) {
+      throw new DbErrorBadRequest(`Account not found for sid ${accountSid}`);
+    }
+    const credentials = await SpeechCredential.retrieve(body.speech_credential_sid);
+    if (!credentials || credentials.length === 0) {
+      throw new
+      DbErrorBadRequest(`There is no available speech credential for ${body.speech_credential_sid}`);
+    }
+    const {credential, ...obj} = credentials[0];
+
+    decryptCredential(obj, credential, logger, false);
+    const cred = getSpeechCredential(obj, logger);
+
+    const { text, language, engine = 'standard' } = body;
+    const salt = uuidv4();
+    /* parse Nuance voices into name and model */
+    let voice = body.voice;
+    let model;
+    if (cred.vendor === 'nuance' && voice) {
+      const arr = /([A-Za-z-]*)\s+-\s+(enhanced|standard)/.exec(voice);
+      if (arr) {
+        voice = arr[1];
+        model = arr[2];
+      }
+    }  else if (cred.vendor === 'deepgram') {
+      model = voice;
+    }
+    const stats = {
+      histogram: () => {},
+      increment: () => {},
+    };
+    const { filePath } = await synthAudio(stats, {
+      account_sid: accountSid,
+      text,
+      vendor: cred.vendor,
+      language,
+      voice,
+      engine,
+      model,
+      salt,
+      credentials: cred,
+      disableTtsCache: false,
+      disableTtsStreaming: true
+    });
+
+    let contentType = 'audio/mpeg';
+
+    let readStream = fs.createReadStream(filePath);
+    if (['nuance', 'nvidia'].includes(cred.vendor) ||
+      (
+        process.env.JAMBONES_TTS_TRIM_SILENCE &&
+        ['microsoft', 'azure'].includes(cred.vendor)
+      )
+    ) {
+      if (encodingMp3) {
+        readStream = readStream
+          .pipe(new PCMToMP3Encoder({
+            channels: 1,
+            sampleRate: 8000,
+            bitRate: 128
+          }, logger));
+      } else {
+        contentType = 'application/octet-stream';
+      }
+    }
+    res.writeHead(200, {
+      'Content-Type': contentType,
+    });
+
+    pipeline(readStream, res, (err) => {
+      if (err) {
+        logger.error('ttscache/Synthesize failed:', err);
+        if (!res.headersSent) {
+          res.status(500).end('Server error');
+        }
+      }
+
+      fs.unlink(filePath, (unlinkErr) => {
+        if (unlinkErr) throw unlinkErr;
+        logger.info(`${filePath} was deleted`);
+      });
+    });
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/users.js

@@ -1,11 +1,11 @@
 const router = require('express').Router();
 const User = require('../../models/user');
-const jwt = require('jsonwebtoken');
-const request = require('request');
-const {DbErrorBadRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, BadRequestError, DbErrorForbidden} = require('../../utils/errors');
 const {generateHashedPassword, verifyPassword} = require('../../utils/password-utils');
 const {promisePool} = require('../../db');
+const {validatePasswordSettings, parseUserSid} = require('./utils');
 const {decrypt} = require('../../utils/encrypt-decrypt');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 const retrieveMyDetails = `SELECT * 
 FROM users user
@@ -28,7 +28,8 @@ AND account_subscriptions.pending=0`;
 const updateSql = 'UPDATE users set hashed_password = ?, force_change = false WHERE user_sid = ?';
 const retrieveStaticIps = 'SELECT * FROM account_static_ips WHERE account_sid = ?';
 
-const validateRequest = async(user_sid, payload) => {
+const validateRequest = async(user_sid, req) => {
+  const payload = req.body;
   const {
     old_password,
     new_password,
@@ -37,15 +38,53 @@ const validateRequest = async(user_sid, payload) => {
     email,
     email_activation_code,
     force_change,
-    is_active} = payload;
+    is_active
+  } = payload;
 
   const [r] = await promisePool.query(retrieveSql, user_sid);
-  if (r.length === 0) return null;
+  if (r.length === 0) {
+    throw new DbErrorBadRequest('Invalid request: user_sid does not exist');
+  }
   const user = r[0];
 
+  /* it is not allowed for anyone to promote a user to a higher level of authority */
+  if (null === payload.account_sid || null === payload.service_provider_sid) {
+    throw new DbErrorBadRequest('Invalid request: user may not be promoted');
+  }
+
+  if (req.user.hasAccountAuth) {
+    /* account user may not change modify account_sid or service_provider_sid */
+    if ('account_sid' in payload && payload.account_sid !== user.account_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another account');
+    }
+    if ('service_provider_sid' in payload && payload.service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another service provider');
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    if ('service_provider_sid' in payload && payload.service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another service provider');
+    }
+  }
+  if ('account_sid' in payload) {
+    const [r] = await promisePool.query('SELECT * FROM accounts WHERE account_sid = ?', payload.account_sid);
+    if (r.length === 0) throw new DbErrorBadRequest('Invalid request: account_sid does not exist');
+    const {service_provider_sid} = r[0];
+    if (service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be moved to another service provider');
+    }
+  }
+
+  if (initial_password) {
+    await validatePasswordSettings(initial_password);
+  }
+
   if ((old_password && !new_password) || (new_password && !old_password)) {
     throw new DbErrorBadRequest('new_password and old_password both required');
   }
+  if (new_password) {
+    await validatePasswordSettings(new_password);
+  }
   if (new_password && name) throw new DbErrorBadRequest('can not change name and password simultaneously');
   if (new_password && user.provider !== 'local') {
     throw new DbErrorBadRequest('can not change password when using oauth2');
@@ -60,25 +99,62 @@ const validateRequest = async(user_sid, payload) => {
   return user;
 };
 
+const getActiveAdminUsers = (users) => {
+  return users.filter((e) => !e.account_sid && !e.service_provider_sid && e.is_active);
+};
+
+const ensureUserActionIsAllowed = (req, user) => {
+  if (req.user.hasAdminAuth) {
+    return;
+  }
+
+  if (req.user.hasServiceProviderAuth && req.user.service_provider_sid === user.service_provider_sid) {
+    return;
+  }
+
+  if (req.user.hasAccountAuth && req.user.account_sid === user.account_sid) {
+    return;
+  }
+
+  throw new DbErrorForbidden('insufficient permissions');
+};
+
+const ensureUserDeletionIsAllowed = (req, activeAdminUsers, user) => {
+  try {
+    if (req.user.hasAdminAuth && activeAdminUsers.length === 1 && activeAdminUsers[0].user_sid === user[0].user_sid) {
+      throw new BadRequestError('cannot delete this admin user - there are no other active admin users');
+    }
+
+    ensureUserActionIsAllowed(req, user[0]);
+    return;
+  } catch (error) {
+    throw error;
+  }
+};
+
+const ensureUserRetrievalIsAllowed = (req, user) => {
+  try {
+    ensureUserActionIsAllowed(req, user);
+    return;
+  } catch (error) {
+    throw error;
+  }
+};
+
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
-  const token = req.user.jwt;
-  const decodedJwt = jwt.verify(token, process.env.JWT_SECRET);
 
   let usersList;
   try {
     let results;
-    if (decodedJwt.scope === 'admin') {
+    if (req.user.hasAdminAuth) {
       results = await User.retrieveAll();
     }
-    else if (decodedJwt.scope === 'account') {
-      results = await User.retrieveAllForAccount(decodedJwt.account_sid, true);
+    else if (req.user.hasAccountAuth) {
+      results = await User.retrieveAllForAccount(req.user.account_sid, true);
     }
-    else if (decodedJwt.scope === 'service_provider') {
-      results = await User.retrieveAllForServiceProvider(decodedJwt.service_provider_sid, true);
-    }
-    else {
-      throw new DbErrorBadRequest(`invalid scope: ${decodedJwt.scope}`);
+    else if (req.user.hasServiceProviderAuth) {
+      results = await User.retrieveAllForServiceProvider(req.user.service_provider_sid, true);
     }
 
     if (results.length === 0) throw new Error('failure retrieving users list');
@@ -222,24 +298,20 @@ router.get('/me', async(req, res) => {
 
 router.get('/:user_sid', async(req, res) => {
   const logger = req.app.locals.logger;
-  const token = req.user.jwt;
-  const decodedJwt = jwt.verify(token, process.env.JWT_SECRET);
-  const {user_sid} = req.params;
 
   try {
+    const user_sid = parseUserSid(req);
     const [user] = await User.retrieve(user_sid);
-    // eslint-disable-next-line no-unused-vars
-    const {hashed_password, ...rest} = user;
-    if (!user) throw new Error('failure retrieving user');
 
-    if (decodedJwt.scope === 'admin' ||
-    decodedJwt.scope === 'account' && decodedJwt.account_sid === user.account_sid ||
-    decodedJwt.scope === 'service_provider' && decodedJwt.service_provider_sid === user.service_provider_sid) {
-      res.status(200).json(rest);
-    } else {
-      res.sendStatus(403);
+    if (!user) {
+      throw new Error('failure retrieving user');
     }
 
+    ensureUserRetrievalIsAllowed(req, user);
+
+    // eslint-disable-next-line no-unused-vars
+    const { hashed_password, ...rest } = user;
+    return res.status(200).json(rest);
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -249,8 +321,7 @@ router.put('/:user_sid', async(req, res) => {
   const logger = req.app.locals.logger;
   const {user_sid} = req.params;
   const user = await User.retrieve(user_sid);
-  const token = req.user.jwt;
-  const decodedJwt = jwt.verify(token, process.env.JWT_SECRET);
+  const {hasAccountAuth, hasServiceProviderAuth, hasAdminAuth} = req.user;
   const {
     old_password,
     new_password,
@@ -266,15 +337,15 @@ router.put('/:user_sid', async(req, res) => {
 
   //if (req.user.user_sid && req.user.user_sid !== user_sid) return res.sendStatus(403);
 
-  if (decodedJwt.scope !== 'admin' &&
-  !(decodedJwt.scope === 'account' && decodedJwt.account_sid === user[0].account_sid) &&
-  !(decodedJwt.scope === 'service_provider' && decodedJwt.service_provider_sid === user[0].service_provider_sid) &&
+  if (!hasAdminAuth &&
+  !(hasAccountAuth && user[0] && req.user.account_sid === user[0].account_sid) &&
+  !(hasServiceProviderAuth && user[0] && req.user.service_provider_sid === user[0].service_provider_sid) &&
   (req.user.user_sid && req.user.user_sid !== user_sid)) {
     return res.sendStatus(403);
   }
 
   try {
-    const user = await validateRequest(user_sid, req.body);
+    const user = await validateRequest(user_sid, req);
     if (!user) return res.sendStatus(404);
 
     if (new_password) {
@@ -327,6 +398,8 @@ router.put('/:user_sid', async(req, res) => {
         'UPDATE users SET account_sid = ? WHERE user_sid = ?',
         [account_sid, user_sid]);
       if (0 === r.changedRows) throw new Error('database update failed');
+      const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+      await cacheClient.delete(redisKey);
     }
 
     if (service_provider_sid || service_provider_sid === null) {
@@ -334,6 +407,8 @@ router.put('/:user_sid', async(req, res) => {
         'UPDATE users SET service_provider_sid = ? WHERE user_sid = ?',
         [service_provider_sid, user_sid]);
       if (0 === r.changedRows) throw new Error('database update failed');
+      const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+      await cacheClient.delete(redisKey);
     }
 
     if (email) {
@@ -367,48 +442,46 @@ router.post('/', async(req, res) => {
     hashed_password: passwordHash,
   };
   const allUsers = await User.retrieveAll();
-  const token = req.user.jwt;
-  const decodedJwt = jwt.verify(token, process.env.JWT_SECRET);
   delete payload.initial_password;
 
   try {
+    if (req.body.initial_password) {
+      await validatePasswordSettings(req.body.initial_password);
+    }
     const email = allUsers.find((e) => e.email === payload.email);
     const name = allUsers.find((e) => e.name === payload.name);
 
     if (name) {
       logger.debug({payload}, 'user with this username already exists');
-      return res.status(422).json({msg: 'user with this username already exists'});
+      return res.status(422).json({msg: 'invalid username or email'});
     }
 
     if (email) {
       logger.debug({payload}, 'user with this email already exists');
-      return res.status(422).json({msg: 'user with this email already exists'});
+      return res.status(422).json({msg: 'invalid username or email'});
     }
 
-    if (decodedJwt.scope === 'admin') {
+    if (req.user.hasAdminAuth) {
       logger.debug({payload}, 'POST /users');
       const uuid = await User.make(payload);
       res.status(201).json({user_sid: uuid});
     }
-    else if (decodedJwt.scope === 'account') {
+    else if (req.user.hasAccountAuth) {
       logger.debug({payload}, 'POST /users');
       const uuid = await User.make({
         ...payload,
-        account_sid: decodedJwt.account_sid,
+        account_sid: req.user.account_sid,
       });
       res.status(201).json({user_sid: uuid});
     }
-    else if (decodedJwt.scope === 'service_provider') {
+    else if (req.user.hasServiceProviderAuth) {
       logger.debug({payload}, 'POST /users');
       const uuid = await User.make({
         ...payload,
-        service_provider_sid: decodedJwt.service_provider_sid,
+        service_provider_sid: req.user.service_provider_sid,
       });
       res.status(201).json({user_sid: uuid});
     }
-    else {
-      throw new DbErrorBadRequest(`invalid scope: ${decodedJwt.scope}`);
-    }
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -416,44 +489,24 @@ router.post('/', async(req, res) => {
 
 router.delete('/:user_sid', async(req, res) => {
   const logger = req.app.locals.logger;
-  const {user_sid} = req.params;
-  const token = req.user.jwt;
-  const decodedJwt = jwt.verify(token, process.env.JWT_SECRET);
-  const allUsers = await User.retrieveAll();
-  const activeAdminUsers = allUsers.filter((e) => !e.account_sid && !e.service_provider_sid && e.is_active);
-  const user = await User.retrieve(user_sid);
 
   try {
-    if (decodedJwt.scope === 'admin' && !user.account_sid && !user.service_provider_sid &&
-     activeAdminUsers.length === 1) {
-      throw new Error('cannot delete this admin user - there are no other active admin users');
-    }
+    const user_sid = parseUserSid(req);
+    const allUsers = await User.retrieveAll();
+    const activeAdminUsers = getActiveAdminUsers(allUsers);
+    const user = allUsers.filter((user) => user.user_sid === user_sid);
 
-    if (decodedJwt.scope === 'admin' ||
-    (decodedJwt.scope === 'account' && decodedJwt.account_sid === user[0].account_sid) ||
-    (decodedJwt.scope === 'service_provider' && decodedJwt.service_provider_sid === user[0].service_provider_sid)) {
-      await User.remove(user_sid);
-      //logout user after self-delete
-      if (decodedJwt.user_sid === user_sid) {
-        request({
-          url:'http://localhost:3000/v1/logout',
-          method: 'POST',
-        }, (err) => {
-          if (err) {
-            logger.error(err, 'could not log out user');
-            return res.sendStatus(500);
-          }
-          logger.debug({user}, 'user deleted and logged out');
-        });
-      }
-      return res.sendStatus(204);
-    } else {
-      throw new DbErrorBadRequest(`invalid scope: ${decodedJwt.scope}`);
-    }
+    ensureUserDeletionIsAllowed(req, activeAdminUsers, user);
+    await User.remove(user_sid);
+
+    /* invalidate the jwt of the deleted user */
+    const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+    await cacheClient.delete(redisKey);
+
+    return res.sendStatus(204);
   } catch (err) {
     sysError(logger, res, err);
   }
 });
 
-
 module.exports = router;

lib/routes/api/utils.js

@@ -1,9 +1,10 @@
-const { v4: uuid } = require('uuid');
+const { v4: uuid, validate } = require('uuid');
 const bent = require('bent');
 const Account = require('../../models/account');
 const {promisePool} = require('../../db');
 const {cancelSubscription, detachPaymentMethod} = require('../../utils/stripe-utils');
 const freePlans = require('../../utils/free_plans');
+const { BadRequestError, DbErrorBadRequest } = require('../../utils/errors');
 const insertAccountSubscriptionSql = `INSERT INTO account_subscriptions 
 (account_subscription_sid, account_sid) 
 values (?, ?)`;
@@ -137,39 +138,179 @@ const createTestAlerts = async(writeAlerts, AlertType, account_sid) => {
 
 };
 
+const validateSid = (model, req) => {
+  const arr = new RegExp(`${model}\/([^\/]*)`).exec(req.originalUrl);
+
+  if (arr) {
+    const sid = arr[1];
+    const sid_validation = validate(sid);
+    if (!sid_validation) {
+      throw new BadRequestError(`invalid ${model}Sid format`);
+    }
+
+    return arr[1];
+  }
+
+  return;
+};
+
 const parseServiceProviderSid = (req) => {
-  const arr = /ServiceProviders\/([^\/]*)/.exec(req.originalUrl);
-  if (arr) return arr[1];
+  try {
+    return validateSid('ServiceProviders', req);
+  } catch (error) {
+    throw error;
+  }
 };
 
 const parseAccountSid = (req) => {
-  const arr = /Accounts\/([^\/]*)/.exec(req.originalUrl);
-  if (arr) return arr[1];
+  try {
+    return validateSid('Accounts', req);
+  } catch (error) {
+    throw error;
+  }
 };
 
-const hasAccountPermissions = (req, res, next) => {
-  if (req.user.hasScope('admin')) return next();
-  if (req.user.hasScope('service_provider')) return next();
-  if (req.user.hasScope('account')) {
-    const account_sid = parseAccountSid(req);
-    if (account_sid === req.user.account_sid) return next();
+const parseApplicationSid = (req) => {
+  try {
+    return validateSid('Applications', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseCallSid = (req) => {
+  try {
+    return validateSid('Calls', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parsePhoneNumberSid = (req) => {
+  try {
+    return validateSid('PhoneNumbers', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseSpeechCredentialSid = (req) => {
+  try {
+    return validateSid('SpeechCredentials', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseVoipCarrierSid = (req) => {
+  try {
+    return validateSid('VoipCarriers', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseWebhookSid = (req) => {
+  try {
+    return validateSid('Webhooks', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseSipGatewaySid = (req) => {
+  try {
+    return validateSid('SipGateways', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseUserSid = (req) => {
+  try {
+    return validateSid('Users', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseLcrSid = (req) => {
+  try {
+    return validateSid('Lcrs', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const hasAccountPermissions = async(req, res, next) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return next();
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const service_provider_sid = parseServiceProviderSid(req);
+      const account_sid = parseAccountSid(req);
+      if (service_provider_sid) {
+        if (service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+      if (account_sid) {
+        const [r] = await Account.retrieve(account_sid);
+        if (r && r.service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+    }
+
+    if (req.user.hasScope('account')) {
+      const account_sid = parseAccountSid(req);
+      const service_provider_sid = parseServiceProviderSid(req);
+      const [r] = await Account.retrieve(account_sid);
+
+      if (account_sid) {
+        if (r && r.account_sid === req.user.account_sid) {
+          return next();
+        }
+      }
+
+      if (service_provider_sid) {
+        if (r && r.service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+    }
+
+    res.status(403).json({
+      status: 'fail',
+      message: 'insufficient privileges'
+    });
+  } catch (error) {
+    throw error;
   }
-  res.status(403).json({
-    status: 'fail',
-    message: 'insufficient privileges'
-  });
 };
 
 const hasServiceProviderPermissions = (req, res, next) => {
-  if (req.user.hasScope('admin')) return next();
-  if (req.user.hasScope('service_provider')) {
-    const service_provider_sid = parseServiceProviderSid(req);
-    if (service_provider_sid === req.user.service_provider_sid) return next();
+  try {
+    if (req.user.hasScope('admin')) {
+      return next();
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const service_provider_sid = parseServiceProviderSid(req);
+      if (service_provider_sid === req.user.service_provider_sid) {
+        return next();
+      }
+    }
+
+    res.status(403).json({
+      status: 'fail',
+      message: 'insufficient privileges'
+    });
+  } catch (error) {
+    throw error;
   }
-  res.status(403).json({
-    status: 'fail',
-    message: 'insufficient privileges'
-  });
 };
 
 const checkLimits = async(req, res, next) => {
@@ -274,15 +415,50 @@ const disableSubspace = async(opts) => {
   return;
 };
 
+const validatePasswordSettings = async(password) => {
+  const sql = 'SELECT * from password_settings';
+  const [rows] = await promisePool.execute(sql);
+  const specialChars = /[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]+/;
+  const numbers = /[0-9]+/;
+  if (rows.length === 0) {
+    if (password.length < 8 || password.length > 20) {
+      throw new DbErrorBadRequest('password length must be between 8 and 20');
+    }
+  } else {
+    if (rows[0].min_password_length && password.length < rows[0].min_password_length) {
+      throw new DbErrorBadRequest(`password must be at least ${rows[0].min_password_length} characters long`);
+    }
+
+    if (rows[0].require_digit === 1 && !numbers.test(password)) {
+      throw new DbErrorBadRequest('password must contain at least one digit');
+    }
+
+    if (rows[0].require_special_character === 1 && !specialChars.test(password)) {
+      throw new DbErrorBadRequest('password must contain at least one special character');
+    }
+  }
+  return;
+};
+
 module.exports = {
   setupFreeTrial,
   createTestCdrs,
   createTestAlerts,
   parseAccountSid,
+  parseApplicationSid,
+  parseCallSid,
+  parsePhoneNumberSid,
   parseServiceProviderSid,
+  parseSpeechCredentialSid,
+  parseVoipCarrierSid,
+  parseWebhookSid,
+  parseSipGatewaySid,
+  parseUserSid,
+  parseLcrSid,
   hasAccountPermissions,
   hasServiceProviderPermissions,
   checkLimits,
   enableSubspace,
-  disableSubspace
+  disableSubspace,
+  validatePasswordSettings
 };

lib/routes/api/voip-carriers.js

@@ -4,6 +4,7 @@ const VoipCarrier = require('../../models/voip-carrier');
 const {promisePool} = require('../../db');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const { parseVoipCarrierSid } = require('./utils');
 
 const validate = async(req) => {
   const {lookupAppBySid, lookupAccountBySid} = req.app.locals;
@@ -73,7 +74,14 @@ decorate(router, VoipCarrier, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await VoipCarrier.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
+    const results = req.user.hasAdminAuth ?
+      await VoipCarrier.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null) :
+      await VoipCarrier.retrieveAllForSP(req.user.service_provider_sid);
+
+    if (req.user.hasScope('account')) {
+      return res.status(200).json(results.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid));
+    }
+
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -84,9 +92,24 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parseVoipCarrierSid(req);
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await VoipCarrier.retrieve(req.params.sid, account_sid);
+    const results = await VoipCarrier.retrieve(sid, account_sid);
     if (results.length === 0) return res.status(404).end();
+    const ret = results[0];
+    ret.register_status = JSON.parse(ret.register_status || '{}');
+
+    if (req.user.hasServiceProviderAuth && results.length === 1) {
+      if (results.length === 1 && results[0].service_provider_sid !== req.user.service_provider_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+    if (req.user.hasAccountAuth && results.length === 1) {
+      if (results.length === 1 && results[0].account_sid !== req.user.account_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/webhooks.js

@@ -2,15 +2,37 @@ const router = require('express').Router();
 const Webhook = require('../../models/webhook');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const {DbErrorForbidden} = require('../../utils/errors');
+const { parseWebhookSid } = require('./utils');
+const {promisePool} = require('../../db');
 
 decorate(router, Webhook, ['add']);
 
 /* retrieve */
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
+
   try {
-    const results = await Webhook.retrieve(req.params.sid);
+    const sid = parseWebhookSid(req);
+    const results = await Webhook.retrieve(sid);
+
     if (results.length === 0) return res.status(404).end();
+
+    if (req.user.hasAccountAuth) {
+      /* can only update carriers for the user's account */
+      if (results[0].account_sid !== req.user.account_sid) {
+        throw new DbErrorForbidden('insufficient privileges');
+      }
+    }
+    if (req.user.hasServiceProviderAuth) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [results[0].account_sid]
+      );
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+      throw new DbErrorForbidden('insufficient permissions');
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/error.js

@@ -1,6 +1,15 @@
-const {DbErrorBadRequest, DbErrorUnprocessableRequest, DbErrorForbidden} = require('../utils/errors');
+const {
+  BadRequestError,
+  DbErrorBadRequest,
+  DbErrorUnprocessableRequest,
+  DbErrorForbidden
+} = require('../utils/errors');
 
 function sysError(logger, res, err) {
+  if (err instanceof BadRequestError) {
+    logger.info(err, err.message);
+    return res.status(400).json({msg: 'Bad request'});
+  }
   if (err instanceof DbErrorBadRequest) {
     logger.info(err, 'invalid client request');
     return res.status(400).json({msg: err.message});

lib/routes/stripe/webhook.js

@@ -61,8 +61,7 @@ router.post('/', express.raw({type: 'application/json'}), async(req, res) => {
   }
 
   /* process event */
-  logger.info(`received webhook: ${evt.type}`);
-  if (evt.type.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
+  if (evt?.type?.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
   else {
     logger.debug(evt, 'unhandled stripe webook');
   }

lib/utils/dns-utils.js

@@ -89,7 +89,12 @@ const createDnsRecords = async(logger, domain, name, value, ttl = 3600) => {
       const str = await res.text();
       return JSON.parse(str);
     }
-    logger.error({res}, 'Error creating records');
+    let body;
+    try {
+      body = await res.json();
+    } catch (err) {
+    }
+    logger.error({headers: res.headers, body}, `Error creating records, status ${res.statusCode}`);
   } catch (err) {
     logger.error({err}, 'Error retrieving domains');
   }

lib/utils/email-utils.js

@@ -1,6 +1,7 @@
 const formData = require('form-data');
 const Mailgun = require('mailgun.js');
 const mailgun = new Mailgun(formData);
+const bent = require('bent');
 const validateEmail = (email) => {
   // eslint-disable-next-line max-len
   const re = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
@@ -8,23 +9,67 @@ const validateEmail = (email) => {
 };
 
 const emailSimpleText = async(logger, to, subject, text) => {
-  const mg = mailgun.client({
-    username: 'api',
-    key: process.env.MAILGUN_API_KEY
-  });
-  if (!process.env.MAILGUN_API_KEY) throw new Error('MAILGUN_API_KEY env variable is not defined!');
-  if (!process.env.MAILGUN_DOMAIN) throw new Error('MAILGUN_DOMAIN env variable is not defined!');
+  const from = 'jambonz Support <support@jambonz.org>';
+  if (process.env.CUSTOM_EMAIL_VENDOR_URL) {
+    await sendEmailByCustomVendor(logger, from, to, subject, text);
+  } else {
+    await sendEmailByMailgun(logger, from, to, subject, text);
+  }
 
+};
+
+const sendEmailByCustomVendor = async(logger, from, to, subject, text) => {
   try {
-    const res = await mg.messages.create(process.env.MAILGUN_DOMAIN, {
-      from: 'jambonz Support <support@jambonz.org>',
+    const post = bent('POST', {
+      'Content-Type': 'application/json',
+      ...((process.env.CUSTOM_EMAIL_VENDOR_USERNAME && process.env.CUSTOM_EMAIL_VENDOR_PASSWORD) &&
+        ({
+          'Authorization':`Basic ${Buffer.from(
+            `${process.env.CUSTOM_EMAIL_VENDOR_USERNAME}:${process.env.CUSTOM_EMAIL_VENDOR_PASSWORD}`
+          ).toString('base64')}`
+        }))
+    });
+
+    const res = await post(process.env.CUSTOM_EMAIL_VENDOR_URL, {
+      from,
       to,
       subject,
       text
     });
-    logger.debug({res}, 'sent email');
+    logger.debug({
+      res
+    }, 'sent email to custom vendor.');
   } catch (err) {
-    logger.info({err}, 'Error sending email');
+    logger.info({
+      err
+    }, 'Error sending email From Custom email vendor');
+  }
+};
+
+const sendEmailByMailgun = async(logger, from, to, subject, text) => {
+  if (!process.env.MAILGUN_API_KEY) throw new Error('MAILGUN_API_KEY env variable is not defined!');
+  if (!process.env.MAILGUN_DOMAIN) throw new Error('MAILGUN_DOMAIN env variable is not defined!');
+
+  const mg = mailgun.client({
+    username: 'api',
+    key: process.env.MAILGUN_API_KEY,
+    ...(process.env.MAILGUN_URL && {url: process.env.MAILGUN_URL})
+  });
+
+  try {
+    const res = await mg.messages.create(process.env.MAILGUN_DOMAIN, {
+      from,
+      to,
+      subject,
+      text
+    });
+    logger.debug({
+      res
+    }, 'sent email');
+  } catch (err) {
+    logger.info({
+      err
+    }, 'Error sending email From mailgun');
   }
 };
 

lib/utils/encrypt-decrypt.js

@@ -2,9 +2,9 @@ const crypto = require('crypto');
 const algorithm = process.env.LEGACY_CRYPTO ? 'aes-256-ctr' : 'aes-256-cbc';
 const iv = crypto.randomBytes(16);
 const secretKey = crypto.createHash('sha256')
-  .update(String(process.env.JWT_SECRET))
+  .update(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET)
   .digest('base64')
-  .substr(0, 32);
+  .substring(0, 32);
 
 const encrypt = (text) => {
   const cipher = crypto.createCipheriv(algorithm, secretKey, iv);
@@ -23,7 +23,18 @@ const decrypt = (data) => {
   return decrpyted.toString();
 };
 
+const obscureKey = (key, key_spoiler_length = 6) => {
+  const key_spoiler_char = 'X';
+
+  if (!key || key.length <= key_spoiler_length) {
+    return key;
+  }
+
+  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
+};
+
 module.exports = {
   encrypt,
-  decrypt
+  decrypt,
+  obscureKey
 };

lib/utils/errors.js

@@ -1,3 +1,9 @@
+class BadRequestError extends Error {
+  constructor(msg) {
+    super(msg);
+  }
+}
+
 class DbError extends Error {
   constructor(msg) {
     super(msg);
@@ -23,6 +29,7 @@ class DbErrorForbidden extends DbError {
 }
 
 module.exports = {
+  BadRequestError,
   DbError,
   DbErrorBadRequest,
   DbErrorUnprocessableRequest,

lib/utils/free_plans.json

@@ -2,7 +2,7 @@
   "trial": [
     {
       "category": "voice_call_session",
-      "quantity": 20
+      "quantity": 5
     },
     {
       "category": "device",

lib/utils/homer-utils.js

@@ -39,11 +39,17 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
     const obj = await postJSON('/api/v3/call/transaction', {
       param: {
         transaction: {
-          call: true
+          call: true,
+          registration: true,
+          rest: false
         },
+        orlogic: true,
         search: {
           '1_call': {
             callid: [callId]
+          },
+          '1_registration': {
+            callid: [callId]
           }
         },
       },
@@ -58,7 +64,7 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
   }
 };
 
-const getHomerPcap = async(logger, apiKey, callIds) => {
+const getHomerPcap = async(logger, apiKey, callIds, method) => {
   if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
     logger.debug('getHomerPcap: Homer integration not installed');
   }
@@ -67,12 +73,23 @@ const getHomerPcap = async(logger, apiKey, callIds) => {
     const stream = await postPcap('/api/v3/export/call/messages/pcap', {
       param: {
         transaction: {
-          call: true
+          call: method === 'invite',
+          registration: method === 'register',
+          rest: false
         },
+        orlogic: true,
         search: {
-          '1_call': {
-            callid: callIds
-          }
+          ...(method === 'invite' && {
+            '1_call': {
+              callid: callIds
+            }
+          })
+          ,
+          ...(method === 'register' && {
+            '1_registration': {
+              callid: callIds
+            }
+          })
         },
       },
       timestamp: {

lib/utils/jaeger-utils.js

@@ -0,0 +1,19 @@
+const bent = require('bent');
+const getJSON = bent(process.env.JAEGER_BASE_URL || 'http://127.0.0.1', 'GET', 'json', 200);
+
+const getJaegerTrace = async(logger, traceId) => {
+  if (!process.env.JAEGER_BASE_URL) {
+    logger.debug('getJaegerTrace: jaeger integration not installed');
+    return null;
+  }
+  try {
+    return await getJSON(`/api/v3/traces/${traceId}`);
+  } catch (err) {
+    const url = `${process.env.JAEGER_BASE_URL}/api/traces/${traceId}`;
+    logger.error({err, traceId}, `getJaegerTrace: Error retrieving spans from ${url}`);
+  }
+};
+
+module.exports = {
+  getJaegerTrace
+};

lib/utils/jambonz-sample.text

@@ -0,0 +1 @@
+Hello From Jambonz. This file was created because Record all call bucket credential test.

lib/utils/password-utils.js

@@ -1,18 +1,19 @@
 const crypto = require('crypto');
-const { argon2i } = require('argon2-ffi');
+const argon2 = require('argon2');
 const util = require('util');
 
+const { argon2i } = argon2;
+
 const getRandomBytes = util.promisify(crypto.randomBytes);
 
 const generateHashedPassword = async(password) => {
   const salt = await getRandomBytes(32);
-  const passwordHash =  await argon2i.hash(password, salt);
+  const passwordHash = await argon2.hash(password, { type: argon2i, salt });
   return passwordHash;
 };
 
-const verifyPassword = async(passwordHash, password) => {
-  const isCorrect = await argon2i.verify(passwordHash, password);
-  return isCorrect;
+const verifyPassword = (passwordHash, password) => {
+  return argon2.verify(passwordHash, password);
 };
 
 const hashString = (s) => crypto.createHash('md5').update(s).digest('hex');

lib/utils/speech-data/stt-assemblyai.js

@@ -0,0 +1,22 @@
+module.exports = [
+  { name: 'Global English', value: 'en' },
+  { name: 'Australian English', value: 'en_au' },
+  { name: 'British English', value: 'en_uk' },
+  { name: 'US English', value: 'en_us' },
+  { name: 'Spanish', value: 'es' },
+  { name: 'French', value: 'fr' },
+  { name: 'German', value: 'de' },
+  { name: 'Italian', value: 'it' },
+  { name: 'Portuguese', value: 'pt' },
+  { name: 'Dutch', value: 'nl' },
+  { name: 'Hindi', value: 'hi' },
+  { name: 'Japanese', value: 'ja' },
+  { name: 'Chinese', value: 'zh' },
+  { name: 'Finnish', value: 'fi' },
+  { name: 'Korean', value: 'ko' },
+  { name: 'Polish', value: 'pl' },
+  { name: 'Russian', value: 'ru' },
+  { name: 'Turkish', value: 'tr' },
+  { name: 'Ukrainian', value: 'uk' },
+  { name: 'Vietnamese', value: 'vi' },
+];

lib/utils/speech-data/stt-aws.js

@@ -0,0 +1,10 @@
+module.exports = [
+  { name: 'Australian English', value: 'en-AU' },
+  { name: 'British English', value: 'en-GB' },
+  { name: 'US English', value: 'en-US' },
+  { name: 'French', value: 'fr-FR' },
+  { name: 'Canadian French', value: 'fr-CA' },
+  { name: 'German', value: 'de-DE' },
+  { name: 'Italian', value: 'it-IT' },
+  { name: 'US Spanish', value: 'es-US' },
+];

lib/utils/speech-data/stt-cobalt.js

@@ -0,0 +1,26 @@
+module.exports = [
+  {
+    name: 'English US',
+    value: 'en_US-8khz',
+  },
+  {
+    name: 'English UK',
+    value: 'en_UK-8khz',
+  },
+  {
+    name: 'Spanish',
+    value: 'es_xx-8khz',
+  },
+  {
+    name: 'French',
+    value: 'fr_fr-8khz',
+  },
+  {
+    name: 'Russian',
+    value: 'ru_ru-8khz',
+  },
+  {
+    name: 'Portuguese',
+    value: 'pt_br-8khz',
+  },
+];

lib/utils/speech-data/stt-deepgram.js

@@ -0,0 +1,138 @@
+module.exports = [
+  {
+    name: 'Chinese - general',
+    value: 'zh',
+  },
+  {
+    name: 'Chinese (China)',
+    value: 'zh-CN',
+  },
+  {
+    name: 'Chinese (Taiwan)',
+    value: 'zh-TW',
+  },
+  {
+    name: 'Dutch - general',
+    value: 'nl',
+  },
+  {
+    name: 'English - general',
+    value: 'en',
+  },
+  {
+    name: 'English (Australia)',
+    value: 'en-AU',
+  },
+  {
+    name: 'English (United Kingdom)',
+    value: 'en-GB',
+  },
+  {
+    name: 'English (India)',
+    value: 'en-IN',
+  },
+  {
+    name: 'English (New Zealand)',
+    value: 'en-NZ',
+  },
+  {
+    name: 'English (United States)',
+    value: 'en-US',
+  },
+  {
+    name: 'French - general',
+    value: 'fr',
+  },
+  {
+    name: 'French (Canada)',
+    value: 'fr-CA',
+  },
+  {
+    name: 'German - general',
+    value: 'de',
+  },
+  {
+    name: 'Hindi - general',
+    value: 'hi',
+  },
+  {
+    name: 'Hindi (Roman Script)',
+    value: 'hi-Latin',
+  },
+  {
+    name: 'Indonesian - general',
+    value: 'in',
+  },
+  {
+    name: 'Italian - general',
+    value: 'it',
+  },
+  {
+    name: 'Japanese - general',
+    value: 'ja',
+  },
+  {
+    name: 'Korean - general',
+    value: 'ko',
+  },
+  {
+    name: 'Norwegian - general',
+    value: 'no',
+  },
+  {
+    name: 'Polish - general',
+    value: 'pl',
+  },
+  {
+    name: 'Portuguese - general',
+    value: 'pt',
+  },
+  {
+    name: 'Portuguese (Brazil)',
+    value: 'pt-BR',
+  },
+  {
+    name: 'Portuguese (Portugal)',
+    value: 'pt-PT',
+  },
+  {
+    name: 'Russian - general',
+    value: 'ru',
+  },
+  {
+    name: 'Spanish - general',
+    value: 'es',
+  },
+  {
+    name: 'Spanish (Latin America)',
+    value: 'es-419',
+  },
+  {
+    name: 'Swedish - general',
+    value: 'sv',
+  },
+  {
+    name: 'Turkish - general',
+    value: 'tr',
+  },
+  {
+    name: 'Ukrainian - general',
+    value: 'uk',
+  },
+  {
+    name: 'Flemish - general',
+    value: 'nl-BE',
+  },
+  {
+    name: 'Danish - general',
+    value: 'da',
+  },
+  {
+    name: 'Tamil - general',
+    value: 'ta',
+  },
+  {
+    name: 'Tamasheq - general',
+    value: 'taq',
+  },
+];

lib/utils/speech-data/stt-google.js

@@ -0,0 +1,130 @@
+module.exports = [
+  { name: 'Afrikaans (South Africa)', value: 'af-ZA' },
+  { name: 'Albanian (Albania)', value: 'sq-AL' },
+  { name: 'Amharic (Ethiopia)', value: 'am-ET' },
+  { name: 'Arabic (Algeria)', value: 'ar-DZ' },
+  { name: 'Arabic (Bahrain)', value: 'ar-BH' },
+  { name: 'Arabic (Egypt)', value: 'ar-EG' },
+  { name: 'Arabic (Iraq)', value: 'ar-IQ' },
+  { name: 'Arabic (Israel)', value: 'ar-IL' },
+  { name: 'Arabic (Jordan)', value: 'ar-JO' },
+  { name: 'Arabic (Kuwait)', value: 'ar-KW' },
+  { name: 'Arabic (Lebanon)', value: 'ar-LB' },
+  { name: 'Arabic (Morocco)', value: 'ar-MA' },
+  { name: 'Arabic (Oman)', value: 'ar-OM' },
+  { name: 'Arabic (Qatar)', value: 'ar-QA' },
+  { name: 'Arabic (Saudi Arabia)', value: 'ar-SA' },
+  { name: 'Arabic (State of Palestine)', value: 'ar-PS' },
+  { name: 'Arabic (Tunisia)', value: 'ar-TN' },
+  { name: 'Arabic (United Arab Emirates)', value: 'ar-AE' },
+  { name: 'Armenian (Armenia)', value: 'hy-AM' },
+  { name: 'Azerbaijani (Azerbaijan)', value: 'az-AZ' },
+  { name: 'Basque (Spain)', value: 'eu-ES' },
+  { name: 'Bengali (Bangladesh)', value: 'bn-BD' },
+  { name: 'Bengali (India)', value: 'bn-IN' },
+  { name: 'Bulgarian (Bulgaria)', value: 'bg-BG' },
+  { name: 'Burmese (Myanmar)', value: 'my-MM' },
+  { name: 'Catalan (Spain)', value: 'ca-ES' },
+  { name: 'Chinese, Cantonese (Traditional, Hong Kong)', value: 'yue-Hant-HK' },
+  { name: 'Chinese, Mandarin (Simplified, China)', value: 'zh' },
+  { name: 'Chinese, Mandarin (Simplified, Hong Kong)', value: 'zh-HK' },
+  { name: 'Chinese, Mandarin (Simplified, Taiwan)', value: 'zh-TW' },
+  { name: 'Croatian (Croatia)', value: 'hr-HR' },
+  { name: 'Czech (Czech Republic)', value: 'cs-CZ' },
+  { name: 'Danish (Denmark)', value: 'da-DK' },
+  { name: 'Dutch (Belgium)', value: 'nl-BE' },
+  { name: 'Dutch (Netherlands)', value: 'nl-NL' },
+  { name: 'English (Australia)', value: 'en-AU' },
+  { name: 'English (Canada)', value: 'en-CA' },
+  { name: 'English (Ghana)', value: 'en-GH' },
+  { name: 'English (India)', value: 'en-IN' },
+  { name: 'English (Ireland)', value: 'en-IE' },
+  { name: 'English (Kenya)', value: 'en-KE' },
+  { name: 'English (New Zealand)', value: 'en-NZ' },
+  { name: 'English (Nigeria)', value: 'en-NG' },
+  { name: 'English (Philippines)', value: 'en-PH' },
+  { name: 'English (Singapore)', value: 'en-SG' },
+  { name: 'English (South Africa)', value: 'en-ZA' },
+  { name: 'English (Tanzania)', value: 'en-TZ' },
+  { name: 'English (United Kingdom)', value: 'en-GB' },
+  { name: 'English (United States)', value: 'en-US' },
+  { name: 'Estonian (Estonia)', value: 'et-EE' },
+  { name: 'Filipino (Philippines)', value: 'fil-PH' },
+  { name: 'Finnish (Finland)', value: 'fi-FI' },
+  { name: 'French (Canada)', value: 'fr-CA' },
+  { name: 'French (France)', value: 'fr-FR' },
+  { name: 'Galician (Spain)', value: 'gl-ES' },
+  { name: 'Georgian (Georgia)', value: 'ka-GE' },
+  { name: 'German (Germany)', value: 'de-DE' },
+  { name: 'Greek (Greece)', value: 'el-GR' },
+  { name: 'Gujarati (India)', value: 'gu-IN' },
+  { name: 'Hebrew (Israel)', value: 'he-IL' },
+  { name: 'Hindi (India)', value: 'hi-IN' },
+  { name: 'Hungarian (Hungary)', value: 'hu-HU' },
+  { name: 'Icelandic (Iceland)', value: 'is-IS' },
+  { name: 'Indonesian (Indonesia)', value: 'id-ID' },
+  { name: 'Italian (Italy)', value: 'it-IT' },
+  { name: 'Japanese (Japan)', value: 'ja-JP' },
+  { name: 'Javanese (Indonesia)', value: 'jv-ID' },
+  { name: 'Kannada (India)', value: 'kn-IN' },
+  { name: 'Khmer (Cambodia)', value: 'km-KH' },
+  { name: 'Korean (South Korea)', value: 'ko-KR' },
+  { name: 'Lao (Laos)', value: 'lo-LA' },
+  { name: 'Latvian (Latvia)', value: 'lv-LV' },
+  { name: 'Lithuanian (Lithuania)', value: 'lt-LT' },
+  { name: 'Macedonian (North Macedonia)', value: 'mk-MK' },
+  { name: 'Malay (Malaysia)', value: 'ms-MY' },
+  { name: 'Malayalam (India)', value: 'ml-IN' },
+  { name: 'Marathi (India)', value: 'mr-IN' },
+  { name: 'Mongolian (Mongolia)', value: 'mn-MN' },
+  { name: 'Nepali (Nepal)', value: 'ne-NP' },
+  { name: 'Norwegian Bokmål (Norway)', value: 'nb-NO' },
+  { name: 'Persian (Iran)', value: 'fa-IR' },
+  { name: 'Polish (Poland)', value: 'pl-PL' },
+  { name: 'Portuguese (Brazil)', value: 'pt-BR' },
+  { name: 'Portuguese (Portugal)', value: 'pt-PT' },
+  { name: 'Punjabi (Gurmukhi, India)', value: 'pa-guru-IN' },
+  { name: 'Romanian (Romania)', value: 'ro-RO' },
+  { name: 'Russian (Russia)', value: 'ru-RU' },
+  { name: 'Serbian (Serbia)', value: 'sr-RS' },
+  { name: 'Sinhala (Sri Lanka)', value: 'si-LK' },
+  { name: 'Slovak (Slovakia)', value: 'sk-SK' },
+  { name: 'Slovenian (Slovenia)', value: 'sl-SI' },
+  { name: 'Spanish (Argentina)', value: 'es-AR' },
+  { name: 'Spanish (Bolivia)', value: 'es-BO' },
+  { name: 'Spanish (Chile)', value: 'es-CL' },
+  { name: 'Spanish (Colombia)', value: 'es-CO' },
+  { name: 'Spanish (Costa Rica)', value: 'es-CR' },
+  { name: 'Spanish (Dominican Republic)', value: 'es-DO' },
+  { name: 'Spanish (Ecuador)', value: 'es-EC' },
+  { name: 'Spanish (El Salvador)', value: 'es-SV' },
+  { name: 'Spanish (Guatemala)', value: 'es-GT' },
+  { name: 'Spanish (Honduras)', value: 'es-HN' },
+  { name: 'Spanish (Mexico)', value: 'es-MX' },
+  { name: 'Spanish (Nicaragua)', value: 'es-NI' },
+  { name: 'Spanish (Panama)', value: 'es-PA' },
+  { name: 'Spanish (Paraguay)', value: 'es-PY' },
+  { name: 'Spanish (Peru)', value: 'es-PE' },
+  { name: 'Spanish (Puerto Rico)', value: 'es-PR' },
+  { name: 'Spanish (Spain)', value: 'es-ES' },
+  { name: 'Spanish (United States)', value: 'es-US' },
+  { name: 'Spanish (Uruguay)', value: 'es-UY' },
+  { name: 'Spanish (Venezuela)', value: 'es-VE' },
+  { name: 'Sundanese (Indonesia)', value: 'su-ID' },
+  { name: 'Swahili (Kenya)', value: 'sw-KE' },
+  { name: 'Swahili (Tanzania)', value: 'sw-TZ' },
+  { name: 'Swedish (Sweden)', value: 'sv-SE' },
+  { name: 'Tamil (India)', value: 'ta-IN' },
+  { name: 'Tamil (Malaysia)', value: 'ta-MY' },
+  { name: 'Tamil (Singapore)', value: 'ta-SG' },
+  { name: 'Tamil (Sri Lanka)', value: 'ta-LK' },
+  { name: 'Telugu (India)', value: 'te-IN' },
+  { name: 'Thai (Thailand)', value: 'th-TH' },
+  { name: 'Turkish (Turkey)', value: 'tr-TR' },
+  { name: 'Ukrainian (Ukraine)', value: 'uk-UA' },
+  { name: 'Urdu (India)', value: 'ur-IN' },
+  { name: 'Urdu (Pakistan)', value: 'ur-PK' },
+  { name: 'Uzbek (Uzbekistan)', value: 'uz-UZ' },
+  { name: 'Vietnamese (Vietnam)', value: 'vi-VN' },
+  { name: 'Zulu (South Africa)', value: 'zu-ZA' },
+];

lib/utils/speech-data/stt-ibm.js

@@ -0,0 +1,82 @@
+module.exports = [
+  {
+    name: 'Arabic (Modern Standard)',
+    value: 'ar-MS_Telephony',
+  },
+  {
+    name: 'Chinese (Mandarin)',
+    value: 'zh-CN_Telephony',
+  },
+  {
+    name: 'Czech ',
+    value: 'cs-CZ_Telephony',
+  },
+  {
+    name: 'Dutch (Belgian)',
+    value: 'nl-BE_Telephony',
+  },
+  {
+    name: 'Dutch (Netherlands)',
+    value: 'nl-NL_Telephony',
+  },
+  {
+    name: 'English (all supported dialects)',
+    value: 'en-WW_Medical_Telephony',
+  },
+  {
+    name: 'English (Australian)',
+    value: 'en-AU_Telephony',
+  },
+  {
+    name: 'English (Indian)',
+    value: 'en-IN_Telephony',
+  },
+  {
+    name: 'English (United Kingdom)',
+    value: 'en-GB_Telephony',
+  },
+  {
+    name: 'English (United States)',
+    value: 'en-US_Telephony',
+  },
+  {
+    name: 'French (Canadian)',
+    value: 'fr-CA_Telephony',
+  },
+  {
+    name: 'French (France)',
+    value: 'fr-FR_Telephony',
+  },
+  {
+    name: 'German',
+    value: 'de-DE_Telephony',
+  },
+  {
+    name: 'Hindi (Indian)',
+    value: 'hi-IN_Telephony',
+  },
+  {
+    name: 'Italian',
+    value: 'it-IT_Telephony',
+  },
+  {
+    name: 'Korean',
+    value: 'ko-KR_Telephony',
+  },
+  {
+    name: 'Portuguese (Brazilian)',
+    value: 'pt-BR_Telephony',
+  },
+  {
+    name: 'Spanish (Mexican)',
+    value: 'es-LA_Telephony',
+  },
+  {
+    name: 'Spanish (Castilian)',
+    value: 'es-ES_Telephony',
+  },
+  {
+    name: 'Swedish ',
+    value: 'sv-SE_Telephony',
+  },
+];

lib/utils/speech-data/stt-microsoft.js

@@ -0,0 +1,490 @@
+module.exports = [
+  {
+    name: 'Afrikaans (South Africa)',
+    value: 'af-ZA',
+  },
+  {
+    name: 'Amharic (Ethiopia)',
+    value: 'am-ET',
+  },
+  {
+    name: 'Arabic (Algeria)',
+    value: 'ar-DZ',
+  },
+  {
+    name: 'Arabic (Bahrain)',
+    value: 'ar-BH',
+  },
+  {
+    name: 'Arabic (Egypt)',
+    value: 'ar-EG',
+  },
+  {
+    name: 'Arabic (Iraq)',
+    value: 'ar-IQ',
+  },
+  {
+    name: 'Arabic (Israel)',
+    value: 'ar-IL',
+  },
+  {
+    name: 'Arabic (Jordan)',
+    value: 'ar-JO',
+  },
+  {
+    name: 'Arabic (Kuwait)',
+    value: 'ar-KW',
+  },
+  {
+    name: 'Arabic (Lebanon)',
+    value: 'ar-LB',
+  },
+  {
+    name: 'Arabic (Libya)',
+    value: 'ar-LY',
+  },
+  {
+    name: 'Arabic (Morocco)',
+    value: 'ar-MA',
+  },
+  {
+    name: 'Arabic (Oman)',
+    value: 'ar-OM',
+  },
+  {
+    name: 'Arabic (Qatar)',
+    value: 'ar-QA',
+  },
+  {
+    name: 'Arabic (Saudi Arabia)',
+    value: 'ar-SA',
+  },
+  {
+    name: 'Arabic (Palestinian Authority)',
+    value: 'ar-PS',
+  },
+  {
+    name: 'Arabic (Syria)',
+    value: 'ar-SY',
+  },
+  {
+    name: 'Arabic (Tunisia)',
+    value: 'ar-TN',
+  },
+  {
+    name: 'Arabic (United Arab Emirates)',
+    value: 'ar-AE',
+  },
+  {
+    name: 'Arabic (Yemen)',
+    value: 'ar-YE',
+  },
+  {
+    name: 'Bulgarian (Bulgaria)',
+    value: 'bg-BG',
+  },
+  {
+    name: 'Bengali (India)',
+    value: 'bn-IN',
+  },
+  {
+    name: 'Catalan (Spain)',
+    value: 'ca-ES',
+  },
+  {
+    name: 'Chinese (Cantonese, Traditional)',
+    value: 'zh-HK',
+  },
+  {
+    name: 'Chinese (Mandarin, Simplified)',
+    value: 'zh-CN',
+  },
+  {
+    name: 'Chinese (Taiwanese Mandarin)',
+    value: 'zh-TW',
+  },
+  {
+    name: 'Croatian (Croatia)',
+    value: 'hr-HR',
+  },
+  {
+    name: 'Czech (Czech)',
+    value: 'cs-CZ',
+  },
+  {
+    name: 'Danish (Denmark)',
+    value: 'da-DK',
+  },
+  {
+    name: 'Dutch (Netherlands)',
+    value: 'nl-NL',
+  },
+  {
+    name: 'Dutch (Belgium)',
+    value: 'nl-BE',
+  },
+  {
+    name: 'English (Australia)',
+    value: 'en-AU',
+  },
+  {
+    name: 'English (Canada)',
+    value: 'en-CA',
+  },
+  {
+    name: 'English (Ghana)',
+    value: 'en-GH',
+  },
+  {
+    name: 'English (Hong Kong)',
+    value: 'en-HK',
+  },
+  {
+    name: 'English (India)',
+    value: 'en-IN',
+  },
+  {
+    name: 'English (Ireland)',
+    value: 'en-IE',
+  },
+  {
+    name: 'English (Kenya)',
+    value: 'en-KE',
+  },
+  {
+    name: 'English (New Zealand)',
+    value: 'en-NZ',
+  },
+  {
+    name: 'English (Nigeria)',
+    value: 'en-NG',
+  },
+  {
+    name: 'English (Philippines)',
+    value: 'en-PH',
+  },
+  {
+    name: 'English (Singapore)',
+    value: 'en-SG',
+  },
+  {
+    name: 'English (South Africa)',
+    value: 'en-ZA',
+  },
+  {
+    name: 'English (Tanzania)',
+    value: 'en-TZ',
+  },
+  {
+    name: 'English (United Kingdom)',
+    value: 'en-GB',
+  },
+  {
+    name: 'English (United States)',
+    value: 'en-US',
+  },
+  {
+    name: 'Estonian(Estonia)',
+    value: 'et-EE',
+  },
+  {
+    name: 'Filipino (Philippines)',
+    value: 'fil-PH',
+  },
+  {
+    name: 'Finnish (Finland)',
+    value: 'fi-FI',
+  },
+  {
+    name: 'French (Belgium)',
+    value: 'fr-BE',
+  },
+  {
+    name: 'French (Canada)',
+    value: 'fr-CA',
+  },
+  {
+    name: 'French (France)',
+    value: 'fr-FR',
+  },
+  {
+    name: 'French (Switzerland)',
+    value: 'fr-CH',
+  },
+  {
+    name: 'German (Austria)',
+    value: 'de-AT',
+  },
+  {
+    name: 'German (Switzerland)',
+    value: 'de-CH',
+  },
+  {
+    name: 'German (Germany)',
+    value: 'de-DE',
+  },
+  {
+    name: 'Greek (Greece)',
+    value: 'el-GR',
+  },
+  {
+    name: 'Gujarati (Indian)',
+    value: 'gu-IN',
+  },
+  {
+    name: 'Hebrew (Israel)',
+    value: 'he-IL',
+  },
+  {
+    name: 'Hindi (India)',
+    value: 'hi-IN',
+  },
+  {
+    name: 'Hungarian (Hungary)',
+    value: 'hu-HU',
+  },
+  {
+    name: 'Indonesian (Indonesia)',
+    value: 'id-ID',
+  },
+  {
+    name: 'Icelandic (Iceland)',
+    value: 'is-IS',
+  },
+  {
+    name: 'Irish (Ireland)',
+    value: 'ga-IE',
+  },
+  {
+    name: 'Italian (Italy)',
+    value: 'it-IT',
+  },
+  {
+    name: 'Japanese (Japan)',
+    value: 'ja-JP',
+  },
+  {
+    name: 'Javanese (Indonesia)',
+    value: 'jv-ID',
+  },
+  {
+    name: 'Kannada (India)',
+    value: 'kn-IN',
+  },
+  {
+    name: 'Khmer (Cambodia)',
+    value: 'km-KH',
+  },
+  {
+    name: 'Korean (Korea)',
+    value: 'ko-KR',
+  },
+  {
+    name: 'Latvian (Latvia)',
+    value: 'lv-LV',
+  },
+  {
+    name: 'Lao (Laos)',
+    value: 'lo-LA',
+  },
+  {
+    name: 'Lithuanian (Lithuania)',
+    value: 'lt-LT',
+  },
+  {
+    name: 'Malay (Malaysia)',
+    value: 'ms-MY',
+  },
+  {
+    name: 'Macedonian (North Macedonia)',
+    value: 'mk-MK',
+  },
+  {
+    name: 'Maltese (Malta)',
+    value: 'mt-MT',
+  },
+  {
+    name: 'Marathi (India)',
+    value: 'mr-IN',
+  },
+  {
+    name: 'Burmese (Myanmar)',
+    value: 'my-MM',
+  },
+  {
+    name: 'Norwegian (Bokmål, Norway)',
+    value: 'nb-NO',
+  },
+  {
+    name: 'Persian (Iran)',
+    value: 'fa-IR',
+  },
+  {
+    name: 'Polish (Poland)',
+    value: 'pl-PL',
+  },
+  {
+    name: 'Portuguese (Brazil)',
+    value: 'pt-BR',
+  },
+  {
+    name: 'Portuguese (Portugal)',
+    value: 'pt-PT',
+  },
+  {
+    name: 'Romanian (Romania)',
+    value: 'ro-RO',
+  },
+  {
+    name: 'Russian (Russia)',
+    value: 'ru-RU',
+  },
+  {
+    name: 'Slovak (Slovakia)',
+    value: 'sk-SK',
+  },
+  {
+    name: 'Slovenian (Slovenia)',
+    value: 'sl-SI',
+  },
+  {
+    name: 'Spanish (Argentina)',
+    value: 'es-AR',
+  },
+  {
+    name: 'Spanish (Bolivia)',
+    value: 'es-BO',
+  },
+  {
+    name: 'Spanish (Chile)',
+    value: 'es-CL',
+  },
+  {
+    name: 'Spanish (Colombia)',
+    value: 'es-CO',
+  },
+  {
+    name: 'Spanish (Costa Rica)',
+    value: 'es-CR',
+  },
+  {
+    name: 'Spanish (Cuba)',
+    value: 'es-CU',
+  },
+  {
+    name: 'Spanish (Dominican Republic)',
+    value: 'es-DO',
+  },
+  {
+    name: 'Spanish (Ecuador)',
+    value: 'es-EC',
+  },
+  {
+    name: 'Spanish (El Salvador)',
+    value: 'es-SV',
+  },
+  {
+    name: 'Spanish (Equatorial Guinea)',
+    value: 'es-GQ',
+  },
+  {
+    name: 'Spanish (Guatemala)',
+    value: 'es-GT',
+  },
+  {
+    name: 'Spanish (Honduras)',
+    value: 'es-HN',
+  },
+  {
+    name: 'Spanish (Mexico)',
+    value: 'es-MX',
+  },
+  {
+    name: 'Spanish (Nicaragua)',
+    value: 'es-NI',
+  },
+  {
+    name: 'Spanish (Panama)',
+    value: 'es-PA',
+  },
+  {
+    name: 'Spanish (Paraguay)',
+    value: 'es-PY',
+  },
+  {
+    name: 'Spanish (Peru)',
+    value: 'es-PE',
+  },
+  {
+    name: 'Spanish (Puerto Rico)',
+    value: 'es-PR',
+  },
+  {
+    name: 'Spanish (Spain)',
+    value: 'es-ES',
+  },
+  {
+    name: 'Spanish (Uruguay)',
+    value: 'es-UY',
+  },
+  {
+    name: 'Spanish (USA)',
+    value: 'es-US',
+  },
+  {
+    name: 'Spanish (Venezuela)',
+    value: 'es-VE',
+  },
+  {
+    name: 'Swahili (Kenya)',
+    value: 'sw-KE',
+  },
+  {
+    name: 'Swahili (Tanzania)',
+    value: 'sw-TZ',
+  },
+  {
+    name: 'Sinhala (Sri Lanka)',
+    value: 'si-LK',
+  },
+  {
+    name: 'Swedish (Sweden)',
+    value: 'sv-SE',
+  },
+  {
+    name: 'Serbian (Serbia)',
+    value: 'sr-RS',
+  },
+  {
+    name: 'Tamil (India)',
+    value: 'ta-IN',
+  },
+  {
+    name: 'Telugu (India)',
+    value: 'te-IN',
+  },
+  {
+    name: 'Thai (Thailand)',
+    value: 'th-TH',
+  },
+  {
+    name: 'Turkish (Turkey)',
+    value: 'tr-TR',
+  },
+  {
+    name: 'Ukrainian (Ukraine)',
+    value: 'uk-UA',
+  },
+  {
+    name: 'Uzbek (Uzbekistan)',
+    value: 'uz-UZ',
+  },
+  {
+    name: 'Zulu (South Africa)',
+    value: 'zu-ZA',
+  },
+  {
+    name: 'Vietnamese (Vietnam)',
+    value: 'vi-VN',
+  },
+];

lib/utils/speech-data/stt-nuance.js

@@ -0,0 +1,207 @@
+module.exports = [
+  {
+    name: 'Arabic (Worldwide)',
+    value: 'ar-WW',
+    valueMix: 'ara-XWW',
+  },
+  {
+    name: 'Catalan (Spain)',
+    value: 'ca-ES',
+    valueMix: 'cat-ESP',
+  },
+  {
+    name: 'Croatian (Croatia)',
+    value: 'hr-HR',
+    valueMix: 'hrv-HRV',
+  },
+  {
+    name: 'Czech (Czech Republic)',
+    value: 'cs-CZ',
+    valueMix: 'ces-CZE',
+  },
+  {
+    name: 'Danish (Denmark)',
+    value: 'da-DK',
+    valueMix: 'dan-DNK',
+  },
+  {
+    name: 'Dutch (Netherlands)',
+    value: 'nl-NL',
+    valueMix: 'nld-NLD',
+  },
+  {
+    name: 'English (Australia)',
+    value: 'en-AU',
+    valueMix: 'eng-AUS',
+  },
+  {
+    name: 'English (United States)',
+    value: 'en-US',
+    valueMix: 'eng-USA',
+  },
+  {
+    name: 'English (India)',
+    value: 'en-IN',
+    valueMix: 'eng-IND',
+  },
+  {
+    name: 'English (United Kingdom)',
+    value: 'en-GB',
+    valueMix: 'eng-GBR',
+  },
+  {
+    name: 'Finnish (Finland)',
+    value: 'fi-FI',
+    valueMix: 'fin-FIN',
+  },
+  {
+    name: 'French (Canada)',
+    value: 'fr-CA',
+    valueMix: 'fra-CAN',
+  },
+  {
+    name: 'French (France)',
+    value: 'fr-FR',
+    valueMix: 'fra-FRA',
+  },
+  {
+    name: 'German (Germany)',
+    value: 'de-DE',
+    valueMix: 'deu-DEU',
+  },
+  {
+    name: 'Greek (Greece)',
+    value: 'el-GR',
+    valueMix: 'ell-GRC',
+  },
+  {
+    name: 'Hebrew (Israel)',
+    value: 'he-IL',
+    valueMix: 'heb-ISR',
+  },
+  {
+    name: 'Hindi (India)',
+    value: 'hi-IN',
+    valueMix: 'hin-IND',
+  },
+  {
+    name: 'Hungarian (Hungary)',
+    value: 'hu-HU',
+    valueMix: 'hun-HUN',
+  },
+  {
+    name: 'Indonesian (Indonesia)',
+    value: 'id-ID',
+    valueMix: 'ind-IDN',
+  },
+  {
+    name: 'Italian (Italy)',
+    value: 'it-IT',
+    valueMix: 'ita-ITA',
+  },
+  {
+    name: 'Japanese (Japan)',
+    value: 'ja-JP',
+    valueMix: 'jpn-JPN',
+  },
+  {
+    name: 'Korean (South Korea)',
+    value: 'ko-KR',
+    valueMix: 'kor-KOR',
+  },
+  {
+    name: 'Malay (Malaysia)',
+    value: 'ms-MY',
+    valueMix: 'zlm-MYS',
+  },
+  {
+    name: 'Norwegian (Norway)',
+    value: 'no-NO',
+    valueMix: 'nor-NOR',
+  },
+  {
+    name: 'Polish (Poland)',
+    value: 'pl-PL',
+    valueMix: 'pol-POL',
+  },
+  {
+    name: 'Portuguese (Brazil)',
+    value: 'pt-BR',
+    valueMix: 'por-BRA',
+  },
+  {
+    name: 'Portuguese (Portugal)',
+    value: 'pt-PT',
+    valueMix: 'por-PRT',
+  },
+  {
+    name: 'Romanian (Romania)',
+    value: 'ro-RO',
+    valueMix: 'ron-ROU',
+  },
+  {
+    name: 'Russian (Russia)',
+    value: 'ru-RU',
+    valueMix: 'rus-RUS',
+  },
+  {
+    name: 'Shanghainese (China)',
+    value: 'zh-WU',
+    valueMix: 'wuu-CHN',
+  },
+  {
+    name: 'Mandarin (China)',
+    value: 'zh-CN',
+    valueMix: 'cmn-CHN',
+  },
+  {
+    name: 'Slovak (Slovakia)',
+    value: 'sk-SK',
+    valueMix: 'slk-SVK',
+  },
+  {
+    name: 'Spanish (Spain)',
+    value: 'es-ES',
+    valueMix: 'spa-ESP',
+  },
+  {
+    name: 'Spanish (Latin America)',
+    value: 'es-US',
+    valueMix: 'spa-XLA',
+  },
+  {
+    name: 'Swedish (Sweden)',
+    value: 'sv-SE',
+    valueMix: 'swe-SWE',
+  },
+  {
+    name: 'Thai (Thailand)',
+    value: 'th-TH',
+    valueMix: 'tha-THA',
+  },
+  {
+    name: 'Cantonese (Hong Kong)',
+    value: 'cn-HK',
+    valueMix: 'yue-CHS',
+  },
+  {
+    name: 'Mandarin (Taiwan)',
+    value: 'zh-TW',
+    valueMix: 'cmn-TWN',
+  },
+  {
+    name: 'Turkish (Turkey)',
+    value: 'tr-TR',
+    valueMix: 'tur-TUR',
+  },
+  {
+    name: 'Ukrainian (Ukraine)',
+    value: 'uk-UA',
+    valueMix: 'ukr-UKR',
+  },
+  {
+    name: 'Vietnamese (Vietnam)',
+    value: 'vi-VN',
+    valueMix: 'vie-VNM',
+  },
+];

lib/utils/speech-data/stt-nvidia.js

@@ -0,0 +1,58 @@
+module.exports = [
+  {
+    name: 'Arabic',
+    value: 'ar-AR',
+  },
+  {
+    name: 'English',
+    value: 'en-US',
+  },
+  {
+    name: 'English - GB',
+    value: 'en-GB',
+  },
+  {
+    name: 'Spanish - US',
+    value: 'es-US',
+  },
+  {
+    name: 'Spanish',
+    value: 'es-ES',
+  },
+  {
+    name: 'German',
+    value: 'de-DE',
+  },
+  {
+    name: 'French',
+    value: 'fr-FR',
+  },
+  {
+    name: 'Hindi',
+    value: 'hi-IN',
+  },
+  {
+    name: 'Russian',
+    value: 'ru-RU',
+  },
+  {
+    name: 'Korean',
+    value: 'ko-KR',
+  },
+  {
+    name: 'Brazilian-Portuguese',
+    value: 'pt-BR',
+  },
+  {
+    name: 'Japanese',
+    value: 'ja-JP',
+  },
+  {
+    name: 'Italian',
+    value: 'it-IT',
+  },
+  {
+    name: 'Mandarin',
+    value: 'zh-CN',
+  },
+];

lib/utils/speech-data/stt-soniox.js

@@ -0,0 +1,6 @@
+module.exports = [
+  {
+    name: 'English (United States)',
+    value: 'en-US',
+  },
+];

lib/utils/speech-data/stt-speechmatics.js

@@ -0,0 +1,218 @@
+module.exports = [
+  {
+    name: 'Automatic',
+    value: 'auto',
+  },
+  {
+    name: 'Arabic',
+    value: 'ar',
+  },
+  {
+    name: 'Bashkir',
+    value: 'ba',
+  },
+  {
+    name: 'Basque',
+    value: 'eu',
+  },
+  {
+    name: 'Belarusian',
+    value: 'be',
+  },
+  {
+    name: 'Bulgarian',
+    value: 'bg',
+  },
+  {
+    name: 'Cantonese',
+    value: 'yue',
+  },
+  {
+    name: 'Catalan',
+    value: 'ca',
+  },
+  {
+    name: 'Croatian',
+    value: 'hr',
+  },
+  {
+    name: 'Czech',
+    value: 'cs',
+  },
+  {
+    name: 'Danish',
+    value: 'da',
+  },
+  {
+    name: 'Dutch',
+    value: 'nl',
+  },
+  {
+    name: 'English',
+    value: 'en',
+  },
+  {
+    name: 'Esperanto',
+    value: 'eo',
+  },
+  {
+    name: 'Estonian',
+    value: 'et',
+  },
+  {
+    name: 'Finnish',
+    value: 'fi',
+  },
+  {
+    name: 'French',
+    value: 'fr',
+  },
+  {
+    name: 'Galician',
+    value: 'gl',
+  },
+  {
+    name: 'German',
+    value: 'de',
+  },
+  {
+    name: 'Greek',
+    value: 'el',
+  },
+  {
+    name: 'Hebrew',
+    value: 'he',
+  },
+  {
+    name: 'Hindi',
+    value: 'hi',
+  },
+  {
+    name: 'Hungarian',
+    value: 'hu',
+  },
+  {
+    name: 'Irish',
+    value: 'ga',
+  },
+  {
+    name: 'Interlingua',
+    value: 'ia',
+  },
+  {
+    name: 'Italian',
+    value: 'it',
+  },
+  {
+    name: 'Indonesian',
+    value: 'id',
+  },
+  {
+    name: 'Japanese',
+    value: 'ja',
+  },
+  {
+    name: 'Korean',
+    value: 'ko',
+  },
+  {
+    name: 'Latvian',
+    value: 'lv',
+  },
+  {
+    name: 'Lithuanian',
+    value: 'lt',
+  },
+  {
+    name: 'Maltese',
+    value: 'mt',
+  },
+  {
+    name: 'Malay',
+    value: 'ms',
+  },
+  {
+    name: 'Mandarin',
+    value: 'cmn',
+  },
+  {
+    name: 'Marathi',
+    value: 'mr',
+  },
+  {
+    name: 'Mongolian',
+    value: 'mn',
+  },
+  {
+    name: 'Norwegian',
+    value: 'no',
+  },
+  {
+    name: 'Persian',
+    value: 'fa',
+  },
+  {
+    name: 'Polish',
+    value: 'pl',
+  },
+  {
+    name: 'Portuguese',
+    value: 'pt',
+  },
+  {
+    name: 'Romanian',
+    value: 'ro',
+  },
+  {
+    name: 'Russian',
+    value: 'ru',
+  },
+  {
+    name: 'Slovakian',
+    value: 'sk',
+  },
+  {
+    name: 'Slovenian',
+    value: 'sl',
+  },
+  {
+    name: 'Spanish',
+    value: 'es',
+  },
+  {
+    name: 'Spanish & English bilingual',
+    value: 'es',
+  },
+  {
+    name: 'Swedish',
+    value: 'sv',
+  },
+  {
+    name: 'Tamil',
+    value: 'ta',
+  },
+  {
+    name: 'Thai',
+    value: 'th',
+  },
+  {
+    name: 'Turkish',
+    value: 'tr',
+  },
+  {
+    name: 'Uyghur',
+    value: 'ug',
+  },
+  {
+    name: 'Ukrainian',
+    value: 'uk',
+  },
+  {
+    name: 'Vietnamese',
+    value: 'vi',
+  },
+  {
+    name: 'Welsh',
+    value: 'cy',
+  },
+];