fix

* slight change to pino logger construction

* remove console.log in test

* added test logging back in

* test

.github/workflows/ci.yml

@@ -1,17 +1,15 @@
 name: CI
 
-on:
-  push:
-  workflow_dispatch:
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: lts/*
       - run: npm install
       - run: npm run jslint
       - run: npm test

.github/workflows/docker-publish-dbcreate.yml

@@ -2,16 +2,10 @@ name: Docker
 
 on:
   push:
-    # Publish `main` as Docker `latest` image.
     branches:
       - main
-
-    # Publish `v1.2.3` tags as releases.
     tags:
-      - v*
-
-env:
-  IMAGE_NAME: db-create
+      - '*'
 
 jobs:
   push:
@@ -20,32 +14,42 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build . --file Dockerfile.db-create --tag $IMAGE_NAME
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Push image
+      - name: prepare tag 
+        id: prepare_tag
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+            IMAGE_ID=jambonz/db-create
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
 
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
 
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "main" ] && VERSION=latest
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
 
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.db-create
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.github/workflows/docker-publish.yml

@@ -2,16 +2,10 @@ name: Docker
 
 on:
   push:
-    # Publish `main` as Docker `latest` image.
     branches:
       - main
-
-    # Publish `v1.2.3` tags as releases.
     tags:
-      - v*
-
-env:
-  IMAGE_NAME: api-server
+      - '*'
 
 jobs:
   push:
@@ -20,32 +14,41 @@ jobs:
     if: github.event_name == 'push'
 
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout code
+        uses: actions/checkout@v3
 
-      - name: Build image
-        run: docker build . --file Dockerfile --tag $IMAGE_NAME
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Push image
+      - name: prepare tag 
+        id: prepare_tag
         run: |
-          IMAGE_ID=ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME
+            IMAGE_ID=jambonz/api-server
 
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
 
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
 
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
 
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "main" ] && VERSION=latest
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
 
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
 
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.gitignore

@@ -1,6 +1,7 @@
 # Logs
 logs
 *.log
+run-tests.sh
 
 # Runtime data
 pids

Dockerfile

@@ -1,10 +1,23 @@
-FROM node:17
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
+
+RUN apk --update --no-cache add --virtual .builds-deps build-base python3
+
 WORKDIR /opt/app/
-COPY package.json ./
-RUN npm install
-RUN npm prune
-COPY . /opt/app
+
+FROM base as build
+
+COPY package.json package-lock.json ./
+
+RUN npm ci
+
+COPY . .
+
+FROM base
+
+COPY --from=build /opt/app /opt/app/
+
 ARG NODE_ENV
+
 ENV NODE_ENV $NODE_ENV
 
-CMD [ "npm", "start" ]
+CMD [ "node", "app.js" ]

Dockerfile.db-create

@@ -1,10 +1,23 @@
-FROM node:17
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
+
+RUN apk --update --no-cache add --virtual .builds-deps build-base python3
+
 WORKDIR /opt/app/
-COPY package.json ./
-RUN npm install
-RUN npm prune
-COPY . /opt/app
+
+FROM base as build
+
+COPY package.json package-lock.json ./
+
+RUN npm ci
+
+COPY . .
+
+FROM base
+
+COPY --from=build /opt/app /opt/app/
+
 ARG NODE_ENV
+
 ENV NODE_ENV $NODE_ENV
 
 CMD [ "npm", "run", "upgrade-db" ]

README.md

@@ -1,29 +1,44 @@
 # jambonz-api-server ![Build Status](https://github.com/jambonz/jambonz-api-server/workflows/CI/badge.svg)
 
-Jambones REST API server.
+Jambones REST API server of the jambones platform.
 
 ## Configuration
 
-This process requires the following environment variables to be set.
+Configuration is provided via environment variables:
 
-```
-JAMBONES_MYSQL_HOST
-JAMBONES_MYSQL_USER
-JAMBONES_MYSQL_PASSWORD
-JAMBONES_MYSQL_DATABASE
-JAMBONES_MYSQL_CONNECTION_LIMIT   # defaults to 10
-JAMBONES_REDIS_HOST
-JAMBONES_REDIS_PORT
-JAMBONES_LOGLEVEL                 # defaults to info
-JAMBONES_API_VERSION              # defaults to v1
-HTTP_PORT                         # defaults to 3000
-```
+| variable | meaning | required?|
+|----------|----------|---------|
+|JWT_SECRET| secret for signing JWT token |yes|
+|JWT_EXPIRES_IN| expiration time for JWT token(in minutes) |no|
+|ENCRYPTION_SECRET| secret for credential encryption(JWT_SECRET is deprecated) |yes|
+|HTTP_PORT| tcp port to listen on for API requests from jambonz-api-server |no|
+|JAMBONES_LOGLEVEL| log level for application, 'info' or 'debug' |no|
+|JAMBONES_MYSQL_HOST| mysql host |yes|
+|JAMBONES_MYSQL_USER| mysql username |yes|
+|JAMBONES_MYSQL_PASSWORD|  mysql password |yes|
+|JAMBONES_MYSQL_DATABASE| mysql data |yes|
+|JAMBONES_MYSQL_PORT| mysql port |no|
+|JAMBONES_MYSQL_CONNECTION_LIMIT| mysql connection limit |no|
+|JAMBONES_REDIS_HOST| redis host |yes|
+|JAMBONES_REDIS_PORT| redis port |no|
+|RATE_LIMIT_WINDOWS_MINS| rate limit window |no|
+|RATE_LIMIT_MAX_PER_WINDOW| number of requests per window |no|
+|JAMBONES_TRUST_PROXY| trust proxies, must be a number |no|
+|JAMBONES_API_VERSION| api version |no|
+|JAMBONES_TIME_SERIES_HOST| influxdb host |yes|
+|JAMBONES_CLUSTER_ID| cluster id |no|
+|HOMER_BASE_URL| HOMER URL |no|
+|HOMER_USERNAME| HOMER username |no|
+|HOMER_PASSWORD| HOMER password |no|
+|K8S| service running as kubernetes service |no|
+|K8S_FEATURE_SERVER_SERVICE_NAME| feature server name(required for K8S) |no|
+|K8S_FEATURE_SERVER_SERVICE_PORT| feature server port(required for K8S) |no|
 
 #### Database dependency
 A mysql database is used to store long-lived objects such as Accounts, Applications, etc. To create the database schema, use or review the scripts in the 'db' folder, particularly:
-- [create_db.sql](db/create_db.sql), which creates the database and associated user (you may want to edit the username and password),
 - [jambones-sql.sql](db/jambones-sql.sql), which creates the schema,
-- [create-admin-token.sql](db/create-admin-token.sql), which creates an admin-level auth token that can be used for testing/exercising the API.
+- [seed-production-database-open-source.sql](db/seed-production-database-open-source.sql), which seeds the database with initial dataset(accounts, permissions, api keys, applications etc).
+- [create-admin-user.sql](db/create-admin-user.sql), which creates admin user with password set to "admin". The password will be forced to change after the first login.
 
 > Note: due to the dependency on the npmjs [mysql](https://www.npmjs.com/package/mysql) package, the mysql database must be configured to use sql [native authentication](https://medium.com/@crmcmullen/how-to-run-mysql-8-0-with-native-password-authentication-502de5bac661).
 

app.js

@@ -1,15 +1,9 @@
 const assert = require('assert');
-const opts = Object.assign({
-  timestamp: () => {
-    return `, "time": "${new Date().toISOString()}"`;
-  }
-}, {
-  level: process.env.JAMBONES_LOGLEVEL || 'info'
-});
-const logger = require('pino')(opts);
+const logger = require('./lib/logger');
 const express = require('express');
 const app = express();
 const helmet = require('helmet');
+const nocache = require('nocache');
 const rateLimit = require('express-rate-limit');
 const cors = require('cors');
 const passport = require('passport');
@@ -19,22 +13,46 @@ assert.ok(process.env.JAMBONES_MYSQL_HOST &&
   process.env.JAMBONES_MYSQL_USER &&
   process.env.JAMBONES_MYSQL_PASSWORD &&
   process.env.JAMBONES_MYSQL_DATABASE, 'missing JAMBONES_MYSQL_XXX env vars');
-assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+if (process.env.JAMBONES_REDIS_SENTINELS) {
+  assert.ok(process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+    'missing JAMBONES_REDIS_SENTINEL_MASTER_NAME env var, JAMBONES_REDIS_SENTINEL_PASSWORD env var is optional');
+} else {
+  assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+}
 assert.ok(process.env.JAMBONES_TIME_SERIES_HOST, 'missing JAMBONES_TIME_SERIES_HOST env var');
-const {queryCdrs, queryAlerts, writeCdrs, writeAlerts, AlertType} = require('@jambonz/time-series')(
+assert.ok(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET, 'missing ENCRYPTION_SECRET env var');
+assert.ok(process.env.JWT_SECRET, 'missing JWT_SECRET env var');
+
+const {
+  queryCdrs,
+  queryCdrsSP,
+  queryAlerts,
+  queryAlertsSP,
+  writeCdrs,
+  writeAlerts,
+  AlertType
+} = require('@jambonz/time-series')(
   logger, process.env.JAMBONES_TIME_SERIES_HOST
 );
 const {
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
   retrieveKey,
-  deleteKey
-} = require('@jambonz/realtimedb-helpers')({
-  host: process.env.JAMBONES_REDIS_HOST || 'localhost',
+  deleteKey,
+  incrKey,
+  JAMBONES_REDIS_SENTINELS
+} = require('./lib/helpers/realtimedb-helpers');
+const {
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache
+} = require('@jambonz/speech-utils')(JAMBONES_REDIS_SENTINELS || {
+  host: process.env.JAMBONES_REDIS_HOST,
   port: process.env.JAMBONES_REDIS_PORT || 6379
 }, logger);
 const {
@@ -55,6 +73,8 @@ const {
 }, logger);
 const PORT = process.env.HTTP_PORT || 3000;
 const authStrategy = require('./lib/auth')(logger, retrieveKey);
+const {delayLoginMiddleware} = require('./lib/middleware');
+const Websocket = require('ws');
 
 passport.use(authStrategy);
 
@@ -65,11 +85,16 @@ app.locals = {
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
+  incrKey,
   retrieveKey,
   deleteKey,
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache,
   lookupAppBySid,
   lookupAccountBySid,
   lookupAccountByPhoneNumber,
@@ -78,7 +103,9 @@ app.locals = {
   lookupSipGatewayBySid,
   lookupSmppGatewayBySid,
   queryCdrs,
+  queryCdrsSP,
   queryAlerts,
+  queryAlertsSP,
   writeCdrs,
   writeAlerts,
   AlertType
@@ -98,12 +125,31 @@ const limiter = rateLimit({
   legacyHeaders: false, // Disable the `X-RateLimit-*` headers
 });
 
+// Setup websocket for recording audio
+const recordWsServer = require('./lib/record');
+const wsServer = new Websocket.Server({ noServer: true });
+wsServer.setMaxListeners(0);
+wsServer.on('connection', recordWsServer.bind(null, logger));
+
+if (process.env.JAMBONES_TRUST_PROXY) {
+  const proxyCount = parseInt(process.env.JAMBONES_TRUST_PROXY);
+  if (!isNaN(proxyCount) && proxyCount > 0) {
+    logger.info(`setting trust proxy to ${proxyCount} and mounting endpoint /ip`);
+    app.set('trust proxy', proxyCount);
+    app.get('/ip', (req, res) => {
+      logger.info({headers: req.headers}, 'received GET /ip');
+      res.send(req.ip);
+    });
+  }
+}
 app.use(limiter);
 app.use(helmet());
 app.use(helmet.hidePoweredBy());
+app.use(nocache());
 app.use(passport.initialize());
 app.use(cors());
 app.use(express.urlencoded({extended: true}));
+app.use(delayLoginMiddleware);
 app.use(unless(['/stripe'], express.json()));
 app.use('/v1', unless(
   [
@@ -125,7 +171,41 @@ app.use((err, req, res, next) => {
   });
 });
 logger.info(`listening for HTTP traffic on port ${PORT}`);
-app.listen(PORT);
+const server = app.listen(PORT);
+
+
+const isValidWsKey = (hdr) => {
+  const username = process.env.JAMBONZ_RECORD_WS_USERNAME;
+  const password = process.env.JAMBONZ_RECORD_WS_PASSWORD;
+  const token = Buffer.from(`${username}:${password}`).toString('base64');
+  const arr = /^Basic (.*)$/.exec(hdr);
+  return arr[1] === token;
+};
+
+server.on('upgrade', (request, socket, head) => {
+  logger.debug({
+    url: request.url,
+    headers: request.headers,
+  }, 'received upgrade request');
+
+  /* verify the path starts with /transcribe */
+  if (!request.url.includes('/record/')) {
+    logger.info(`unhandled path: ${request.url}`);
+    return socket.write('HTTP/1.1 404 Not Found \r\n\r\n', () => socket.destroy());
+  }
+
+  /* verify the api key */
+  if (!isValidWsKey(request.headers['authorization'])) {
+    logger.info(`invalid auth header: ${request.headers['authorization']}`);
+    return socket.write('HTTP/1.1 403 Forbidden \r\n\r\n', () => socket.destroy());
+  }
+
+  /* complete the upgrade */
+  wsServer.handleUpgrade(request, socket, head, (ws) => {
+    logger.info(`upgraded to websocket, url: ${request.url}`);
+    wsServer.emit('connection', ws, request.url);
+  });
+});
 
 // purge old calls from active call set every 10 mins
 async function purge() {

db/add-predefined-carriers.sql

@@ -45,8 +45,6 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);

db/create-admin-user.sql

@@ -0,0 +1,10 @@
+/* hashed password is "admin" */
+insert into users (user_sid, name, email, hashed_password, force_change, provider, email_validated)
+values ('12c80508-edf9-4b22-8d09-55abd02648eb', 'admin', 'joe@foo.bar', '$argon2i$v=19$m=65536,t=3,p=4$c2FsdHNhbHRzYWx0c2FsdA$x5OO6gXFXS25oqUU2JvbYqrSgRxBujNUJBq6xv9EgjM', 1, 'local', 1);
+
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('8919e0dc-4d69-4de5-be56-a121598d9093', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc342a-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('d6fdf064-0a65-4b17-8b10-5500e956a159', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3a10-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('f68185dd-0486-4767-a77d-a0b84c1b236e' ,'12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3c5e-546a-11ed-bdc3-0242ac120002');

db/jambones-sql.sql

@@ -1,9 +1,10 @@
 /* SQLEditor (MySQL (2))*/
-
 SET FOREIGN_KEY_CHECKS=0;
 
 DROP TABLE IF EXISTS account_static_ips;
 
+DROP TABLE IF EXISTS account_limits;
+
 DROP TABLE IF EXISTS account_products;
 
 DROP TABLE IF EXISTS account_subscriptions;
@@ -12,12 +13,22 @@ DROP TABLE IF EXISTS beta_invite_codes;
 
 DROP TABLE IF EXISTS call_routes;
 
+DROP TABLE IF EXISTS clients;
+
 DROP TABLE IF EXISTS dns_records;
 
+DROP TABLE IF EXISTS lcr;
+
 DROP TABLE IF EXISTS lcr_carrier_set_entry;
 
 DROP TABLE IF EXISTS lcr_routes;
 
+DROP TABLE IF EXISTS password_settings;
+
+DROP TABLE IF EXISTS user_permissions;
+
+DROP TABLE IF EXISTS permissions;
+
 DROP TABLE IF EXISTS predefined_sip_gateways;
 
 DROP TABLE IF EXISTS predefined_smpp_gateways;
@@ -36,12 +47,16 @@ DROP TABLE IF EXISTS sbc_addresses;
 
 DROP TABLE IF EXISTS ms_teams_tenants;
 
+DROP TABLE IF EXISTS service_provider_limits;
+
 DROP TABLE IF EXISTS signup_history;
 
 DROP TABLE IF EXISTS smpp_addresses;
 
 DROP TABLE IF EXISTS speech_credentials;
 
+DROP TABLE IF EXISTS system_information;
+
 DROP TABLE IF EXISTS users;
 
 DROP TABLE IF EXISTS smpp_gateways;
@@ -69,6 +84,15 @@ private_ipv4 VARBINARY(16) NOT NULL UNIQUE ,
 PRIMARY KEY (account_static_ip_sid)
 );
 
+CREATE TABLE account_limits
+(
+account_limits_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+category ENUM('api_rate','voice_call_session', 'device','voice_call_minutes','voice_call_session_license', 'voice_call_minutes_license') NOT NULL,
+quantity INTEGER NOT NULL,
+PRIMARY KEY (account_limits_sid)
+);
+
 CREATE TABLE account_subscriptions
 (
 account_subscription_sid CHAR(36) NOT NULL UNIQUE ,
@@ -105,6 +129,16 @@ application_sid CHAR(36) NOT NULL,
 PRIMARY KEY (call_route_sid)
 ) COMMENT='a regex-based pattern match for call routing';
 
+CREATE TABLE clients
+(
+client_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+is_active BOOLEAN NOT NULL DEFAULT 1,
+username VARCHAR(64),
+password VARCHAR(1024),
+PRIMARY KEY (client_sid)
+);
+
 CREATE TABLE dns_records
 (
 dns_record_sid CHAR(36) NOT NULL UNIQUE ,
@@ -117,11 +151,38 @@ PRIMARY KEY (dns_record_sid)
 CREATE TABLE lcr_routes
 (
 lcr_route_sid CHAR(36),
+lcr_sid CHAR(36) NOT NULL,
 regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
 description VARCHAR(1024),
-priority INTEGER NOT NULL UNIQUE  COMMENT 'lower priority routes are attempted first',
+priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
 PRIMARY KEY (lcr_route_sid)
-) COMMENT='Least cost routing table';
+) COMMENT='An ordered list of  digit patterns in an LCR table.  The patterns are tested in sequence until one matches';
+
+CREATE TABLE lcr
+(
+lcr_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+is_active BOOLEAN NOT NULL DEFAULT 1,
+default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+service_provider_sid CHAR(36),
+account_sid CHAR(36),
+PRIMARY KEY (lcr_sid)
+) COMMENT='An LCR (least cost routing) table that is used by a service provider or account to make decisions about routing outbound calls when multiple carriers are available.';
+
+CREATE TABLE password_settings
+(
+min_password_length INTEGER NOT NULL DEFAULT 8,
+require_digit BOOLEAN NOT NULL DEFAULT false,
+require_special_character BOOLEAN NOT NULL DEFAULT false
+);
+
+CREATE TABLE permissions
+(
+permission_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(32) NOT NULL UNIQUE ,
+description VARCHAR(255),
+PRIMARY KEY (permission_sid)
+);
 
 CREATE TABLE predefined_carriers
 (
@@ -214,7 +275,10 @@ CREATE TABLE sbc_addresses
 sbc_address_sid CHAR(36) NOT NULL UNIQUE ,
 ipv4 VARCHAR(255) NOT NULL,
 port INTEGER NOT NULL DEFAULT 5060,
+tls_port INTEGER,
+wss_port INTEGER,
 service_provider_sid CHAR(36),
+last_updated DATETIME,
 PRIMARY KEY (sbc_address_sid)
 );
 
@@ -228,6 +292,15 @@ tenant_fqdn VARCHAR(255) NOT NULL UNIQUE ,
 PRIMARY KEY (ms_teams_tenant_sid)
 ) COMMENT='A Microsoft Teams customer tenant';
 
+CREATE TABLE service_provider_limits
+(
+service_provider_limits_sid CHAR(36) NOT NULL UNIQUE ,
+service_provider_sid CHAR(36) NOT NULL,
+category ENUM('api_rate','voice_call_session', 'device','voice_call_minutes','voice_call_session_license', 'voice_call_minutes_license') NOT NULL,
+quantity INTEGER NOT NULL,
+PRIMARY KEY (service_provider_limits_sid)
+);
+
 CREATE TABLE signup_history
 (
 email VARCHAR(255) NOT NULL,
@@ -264,6 +337,13 @@ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
 PRIMARY KEY (speech_credential_sid)
 );
 
+CREATE TABLE system_information
+(
+domain_name VARCHAR(255),
+sip_domain_name VARCHAR(255),
+monitoring_domain_name VARCHAR(255)
+);
+
 CREATE TABLE users
 (
 user_sid CHAR(36) NOT NULL UNIQUE ,
@@ -283,6 +363,7 @@ email_activation_code VARCHAR(16),
 email_validated BOOLEAN NOT NULL DEFAULT false,
 phone_validated BOOLEAN NOT NULL DEFAULT false,
 email_content_opt_out BOOLEAN NOT NULL DEFAULT false,
+is_active BOOLEAN NOT NULL DEFAULT true,
 PRIMARY KEY (user_sid)
 );
 
@@ -310,9 +391,21 @@ smpp_password VARCHAR(64),
 smpp_enquire_link_interval INTEGER DEFAULT 0,
 smpp_inbound_system_id VARCHAR(255),
 smpp_inbound_password VARCHAR(64),
+register_from_user VARCHAR(128),
+register_from_domain VARCHAR(255),
+register_public_ip_in_contact BOOLEAN NOT NULL DEFAULT false,
+register_status VARCHAR(4096),
 PRIMARY KEY (voip_carrier_sid)
 ) COMMENT='A Carrier or customer PBX that can send or receive calls';
 
+CREATE TABLE user_permissions
+(
+user_permissions_sid CHAR(36) NOT NULL UNIQUE ,
+user_sid CHAR(36) NOT NULL,
+permission_sid CHAR(36) NOT NULL,
+PRIMARY KEY (user_permissions_sid)
+);
+
 CREATE TABLE smpp_gateways
 (
 smpp_gateway_sid CHAR(36) NOT NULL UNIQUE ,
@@ -330,7 +423,7 @@ PRIMARY KEY (smpp_gateway_sid)
 CREATE TABLE phone_numbers
 (
 phone_number_sid CHAR(36) UNIQUE ,
-number VARCHAR(32) NOT NULL UNIQUE ,
+number VARCHAR(132) NOT NULL,
 voip_carrier_sid CHAR(36),
 account_sid CHAR(36),
 application_sid CHAR(36),
@@ -348,6 +441,7 @@ inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound ca
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
 
@@ -365,7 +459,7 @@ CREATE TABLE webhooks
 (
 webhook_sid CHAR(36) NOT NULL UNIQUE ,
 url VARCHAR(1024) NOT NULL,
-method ENUM("GET","POST","WS") NOT NULL DEFAULT 'POST',
+method ENUM("GET","POST") NOT NULL DEFAULT 'POST',
 username VARCHAR(255),
 password VARCHAR(255),
 PRIMARY KEY (webhook_sid)
@@ -380,12 +474,14 @@ account_sid CHAR(36) COMMENT 'account that this application belongs to (if null,
 call_hook_sid CHAR(36) COMMENT 'webhook to call for inbound calls ',
 call_status_hook_sid CHAR(36) COMMENT 'webhook to call for call status events',
 messaging_hook_sid CHAR(36) COMMENT 'webhook to call for inbound SMS/MMS ',
+app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
 speech_synthesis_voice VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
 ) COMMENT='A defined set of behaviors to be applied to phone calls ';
 
@@ -422,6 +518,10 @@ subspace_client_id VARCHAR(255),
 subspace_client_secret VARCHAR(255),
 subspace_sip_teleport_id VARCHAR(255),
 subspace_sip_teleport_destinations VARCHAR(255),
+siprec_hook_sid CHAR(36),
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
+record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
+bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
 PRIMARY KEY (account_sid)
 ) COMMENT='An enterprise that uses the platform for comm services';
 
@@ -429,19 +529,34 @@ CREATE INDEX account_static_ip_sid_idx ON account_static_ips (account_static_ip_
 CREATE INDEX account_sid_idx ON account_static_ips (account_sid);
 ALTER TABLE account_static_ips ADD FOREIGN KEY account_sid_idxfk (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX account_sid_idx ON account_limits (account_sid);
+ALTER TABLE account_limits ADD FOREIGN KEY account_sid_idxfk_1 (account_sid) REFERENCES accounts (account_sid) ON DELETE CASCADE;
+
 CREATE INDEX account_subscription_sid_idx ON account_subscriptions (account_subscription_sid);
 CREATE INDEX account_sid_idx ON account_subscriptions (account_sid);
-ALTER TABLE account_subscriptions ADD FOREIGN KEY account_sid_idxfk_1 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE account_subscriptions ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX invite_code_idx ON beta_invite_codes (invite_code);
 CREATE INDEX call_route_sid_idx ON call_routes (call_route_sid);
-ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE call_routes ADD FOREIGN KEY application_sid_idxfk (application_sid) REFERENCES applications (application_sid);
 
-CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
-ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERENCES accounts (account_sid);
+CREATE INDEX client_sid_idx ON clients (client_sid);
+ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
+ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid);
+ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid);
+
+CREATE INDEX lcr_sid_idx ON lcr (lcr_sid);
+ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid);
+
+CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid);
+CREATE INDEX account_sid_idx ON lcr (account_sid);
+CREATE INDEX permission_sid_idx ON permissions (permission_sid);
 CREATE INDEX predefined_carrier_sid_idx ON predefined_carriers (predefined_carrier_sid);
 CREATE INDEX predefined_sip_gateway_sid_idx ON predefined_sip_gateways (predefined_sip_gateway_sid);
 CREATE INDEX predefined_carrier_sid_idx ON predefined_sip_gateways (predefined_carrier_sid);
@@ -460,14 +575,14 @@ ALTER TABLE account_products ADD FOREIGN KEY product_sid_idxfk (product_sid) REF
 
 CREATE INDEX account_offer_sid_idx ON account_offers (account_offer_sid);
 CREATE INDEX account_sid_idx ON account_offers (account_sid);
-ALTER TABLE account_offers ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE account_offers ADD FOREIGN KEY account_sid_idxfk_5 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX product_sid_idx ON account_offers (product_sid);
 ALTER TABLE account_offers ADD FOREIGN KEY product_sid_idxfk_1 (product_sid) REFERENCES products (product_sid);
 
 CREATE INDEX api_key_sid_idx ON api_keys (api_key_sid);
 CREATE INDEX account_sid_idx ON api_keys (account_sid);
-ALTER TABLE api_keys ADD FOREIGN KEY account_sid_idxfk_5 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE api_keys ADD FOREIGN KEY account_sid_idxfk_6 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX service_provider_sid_idx ON api_keys (service_provider_sid);
 ALTER TABLE api_keys ADD FOREIGN KEY service_provider_sid_idxfk (service_provider_sid) REFERENCES service_providers (service_provider_sid);
@@ -481,59 +596,70 @@ ALTER TABLE sbc_addresses ADD FOREIGN KEY service_provider_sid_idxfk_1 (service_
 CREATE INDEX ms_teams_tenant_sid_idx ON ms_teams_tenants (ms_teams_tenant_sid);
 ALTER TABLE ms_teams_tenants ADD FOREIGN KEY service_provider_sid_idxfk_2 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-ALTER TABLE ms_teams_tenants ADD FOREIGN KEY account_sid_idxfk_6 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE ms_teams_tenants ADD FOREIGN KEY account_sid_idxfk_7 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE ms_teams_tenants ADD FOREIGN KEY application_sid_idxfk_1 (application_sid) REFERENCES applications (application_sid);
 
 CREATE INDEX tenant_fqdn_idx ON ms_teams_tenants (tenant_fqdn);
+CREATE INDEX service_provider_sid_idx ON service_provider_limits (service_provider_sid);
+ALTER TABLE service_provider_limits ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) REFERENCES service_providers (service_provider_sid) ON DELETE CASCADE;
+
 CREATE INDEX email_idx ON signup_history (email);
 CREATE INDEX smpp_address_sid_idx ON smpp_addresses (smpp_address_sid);
 CREATE INDEX service_provider_sid_idx ON smpp_addresses (service_provider_sid);
-ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE UNIQUE INDEX speech_credentials_idx_1 ON speech_credentials (vendor,account_sid);
 
 CREATE INDEX speech_credential_sid_idx ON speech_credentials (speech_credential_sid);
 CREATE INDEX service_provider_sid_idx ON speech_credentials (service_provider_sid);
-ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
-ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_7 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX user_sid_idx ON users (user_sid);
 CREATE INDEX email_idx ON users (email);
 CREATE INDEX phone_idx ON users (phone);
 CREATE INDEX account_sid_idx ON users (account_sid);
-ALTER TABLE users ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE users ADD FOREIGN KEY account_sid_idxfk_9 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX service_provider_sid_idx ON users (service_provider_sid);
-ALTER TABLE users ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE users ADD FOREIGN KEY service_provider_sid_idxfk_6 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX email_activation_code_idx ON users (email_activation_code);
 CREATE INDEX voip_carrier_sid_idx ON voip_carriers (voip_carrier_sid);
 CREATE INDEX account_sid_idx ON voip_carriers (account_sid);
-ALTER TABLE voip_carriers ADD FOREIGN KEY account_sid_idxfk_9 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE voip_carriers ADD FOREIGN KEY account_sid_idxfk_10 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX service_provider_sid_idx ON voip_carriers (service_provider_sid);
-ALTER TABLE voip_carriers ADD FOREIGN KEY service_provider_sid_idxfk_6 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE voip_carriers ADD FOREIGN KEY service_provider_sid_idxfk_7 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 ALTER TABLE voip_carriers ADD FOREIGN KEY application_sid_idxfk_2 (application_sid) REFERENCES applications (application_sid);
 
+CREATE INDEX user_permissions_sid_idx ON user_permissions (user_permissions_sid);
+CREATE INDEX user_sid_idx ON user_permissions (user_sid);
+ALTER TABLE user_permissions ADD FOREIGN KEY user_sid_idxfk (user_sid) REFERENCES users (user_sid) ON DELETE CASCADE;
+
+ALTER TABLE user_permissions ADD FOREIGN KEY permission_sid_idxfk (permission_sid) REFERENCES permissions (permission_sid);
+
 CREATE INDEX smpp_gateway_sid_idx ON smpp_gateways (smpp_gateway_sid);
 CREATE INDEX voip_carrier_sid_idx ON smpp_gateways (voip_carrier_sid);
 ALTER TABLE smpp_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
+CREATE UNIQUE INDEX phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid);
+
 CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
 CREATE INDEX number_idx ON phone_numbers (number);
 CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);
 ALTER TABLE phone_numbers ADD FOREIGN KEY voip_carrier_sid_idxfk_1 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
-ALTER TABLE phone_numbers ADD FOREIGN KEY account_sid_idxfk_10 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE phone_numbers ADD FOREIGN KEY account_sid_idxfk_11 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE phone_numbers ADD FOREIGN KEY application_sid_idxfk_3 (application_sid) REFERENCES applications (application_sid);
 
 CREATE INDEX service_provider_sid_idx ON phone_numbers (service_provider_sid);
-ALTER TABLE phone_numbers ADD FOREIGN KEY service_provider_sid_idxfk_7 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE phone_numbers ADD FOREIGN KEY service_provider_sid_idxfk_8 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX sip_gateway_idx_hostport ON sip_gateways (ipv4,port);
 
@@ -549,10 +675,10 @@ CREATE UNIQUE INDEX applications_idx_name ON applications (account_sid,name);
 
 CREATE INDEX application_sid_idx ON applications (application_sid);
 CREATE INDEX service_provider_sid_idx ON applications (service_provider_sid);
-ALTER TABLE applications ADD FOREIGN KEY service_provider_sid_idxfk_8 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE applications ADD FOREIGN KEY service_provider_sid_idxfk_9 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX account_sid_idx ON applications (account_sid);
-ALTER TABLE applications ADD FOREIGN KEY account_sid_idxfk_11 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE applications ADD FOREIGN KEY account_sid_idxfk_12 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE applications ADD FOREIGN KEY call_hook_sid_idxfk (call_hook_sid) REFERENCES webhooks (webhook_sid);
 
@@ -568,7 +694,7 @@ ALTER TABLE service_providers ADD FOREIGN KEY registration_hook_sid_idxfk (regis
 CREATE INDEX account_sid_idx ON accounts (account_sid);
 CREATE INDEX sip_realm_idx ON accounts (sip_realm);
 CREATE INDEX service_provider_sid_idx ON accounts (service_provider_sid);
-ALTER TABLE accounts ADD FOREIGN KEY service_provider_sid_idxfk_9 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE accounts ADD FOREIGN KEY service_provider_sid_idxfk_10 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 ALTER TABLE accounts ADD FOREIGN KEY registration_hook_sid_idxfk_1 (registration_hook_sid) REFERENCES webhooks (webhook_sid);
 
@@ -576,4 +702,5 @@ ALTER TABLE accounts ADD FOREIGN KEY queue_event_hook_sid_idxfk (queue_event_hoo
 
 ALTER TABLE accounts ADD FOREIGN KEY device_calling_application_sid_idxfk (device_calling_application_sid) REFERENCES applications (application_sid);
 
-SET FOREIGN_KEY_CHECKS=0;
+ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid);
+SET FOREIGN_KEY_CHECKS=1;

db/reset_admin_password.js

@@ -12,6 +12,9 @@ values (?, ?)`;
 const sqlQueryAccount = 'SELECT * from accounts LEFT JOIN api_keys ON api_keys.account_sid = accounts.account_sid';
 const sqlAddAccountToken = `INSERT into api_keys (api_key_sid, token, account_sid) 
 VALUES (?, ?, ?)`;
+const sqlInsertPermissions = `
+INSERT into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+VALUES (?,?,?)`;
 
 const password = process.env.JAMBONES_ADMIN_INITIAL_PASSWORD || 'admin';
 console.log(`reset_admin_password, initial admin password is ${password}`);
@@ -21,6 +24,7 @@ const doIt = async() => {
   const sid = uuidv4();
   await promisePool.execute('DELETE from users where name = "admin"');
   await promisePool.execute('DELETE from api_keys where account_sid is null and service_provider_sid is null');
+
   await promisePool.execute(sqlInsert,
     [
       sid,
@@ -34,6 +38,12 @@ const doIt = async() => {
   );
   await promisePool.execute(sqlInsertAdminToken, [uuidv4(), uuidv4()]);
 
+  /* assign all permissions to the admin user */
+  const [p] = await promisePool.query('SELECT * from permissions');
+  for (const perm of p) {
+    await promisePool.execute(sqlInsertPermissions, [uuidv4(), sid, perm.permission_sid]);
+  }
+
   /* create admin token for single account */
   const [r] = await promisePool.query({sql: sqlQueryAccount, nestTables: true});
   if (1 === r.length && r[0].api_keys.api_key_sid === null) {

db/seed-integration-test.sql

@@ -1,5 +1,12 @@
 SET FOREIGN_KEY_CHECKS=0;
 
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
 insert into sbc_addresses (sbc_address_sid, ipv4, port) 
 values('f6567ae1-bf97-49af-8931-ca014b689995', '52.55.111.178', 5060);
 insert into sbc_addresses (sbc_address_sid, ipv4, port) 
@@ -20,6 +27,14 @@ values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip
 insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
 values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'wh_secret_cJqgtMDPzDhhnjmaJH6Mtk');
 
+-- create account level api key
+insert into api_keys (api_key_sid, token, service_provider_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fa', '38700987-c7a4-4685-a5bb-af378f9734da', '9351f46a-678c-43f5-b8a6-d4eb58d131af');
+
+-- create SP level api key
+insert into api_keys (api_key_sid, token, account_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fs', '38700987-c7a4-4685-a5bb-af378f9734ds', '2708b1b3-2736-40ea-b502-c53d8396247f');
+
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
@@ -72,6 +87,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways
@@ -91,10 +107,8 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
 

db/seed-production-database-open-source.sql

@@ -1,5 +1,12 @@
 SET FOREIGN_KEY_CHECKS=0;
 
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
 -- create one service provider and account
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
@@ -66,6 +73,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways
@@ -85,10 +93,8 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
 SET FOREIGN_KEY_CHECKS=1;

db/seed-production-database.sql

@@ -1,6 +1,15 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
 -- create one service provider
 insert into service_providers (service_provider_sid, name, description, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'sip.jambonz.us', 'jambonz.us service provider', 'sip.jambonz.us');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'sip.jambonz.xyz', 'jambonz.xyz service provider', 'sip.jambonz.xyz');
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
 
@@ -16,9 +25,9 @@ insert into predefined_carriers (predefined_carrier_sid, name, requires_static_i
 requires_register, register_username, register_password, 
 register_sip_realm, tech_prefix, inbound_auth_username, inbound_auth_password, diversion)
 VALUES
+('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
 ('7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', 'Twilio', 0, 1, 0, '<your-twilio-credential-username>', '<your-twilio-credential-password>', NULL, NULL, NULL, NULL, NULL),
 ('032d90d5-39e8-41c0-b807-9c88cffba65c', 'Voxbone', 0, 1, 0, '<your-voxbone-outbound-username>', '<your-voxbone-outbound-password>', NULL, NULL, NULL, NULL, '<your-voxbone-DID>'),
-('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
 ('e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'Simwood', 0, 1, 0, '<your-simwood-auth-trunk-username>',  '<your-simwood-auth-trunk-password>', NULL, NULL, NULL, NULL, NULL);
 
 -- TelecomXchange gateways
@@ -44,6 +53,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways
@@ -63,9 +73,8 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
+SET FOREIGN_KEY_CHECKS=1;

db/upgrade-jambonz-db.js

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+/* eslint-disable max-len */
 const assert = require('assert');
 const mysql = require('mysql2/promise');
 const {readFile} = require('fs/promises');
@@ -22,6 +23,145 @@ const opts = {
   multipleStatements: true
 };
 
+const sql = {
+  '7006': [
+    'ALTER TABLE `accounts` ADD COLUMN `siprec_hook_sid` CHAR(36)',
+    'ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid)'
+  ],
+  '7007': [
+    `CREATE TABLE service_provider_limits 
+    (service_provider_limits_sid CHAR(36) NOT NULL UNIQUE,
+    service_provider_sid CHAR(36) NOT NULL,
+    category ENUM('api_rate','voice_call_session', 'device') NOT NULL,
+    quantity INTEGER NOT NULL,
+    PRIMARY KEY (service_provider_limits_sid)
+    )`,
+    `CREATE TABLE account_limits
+    (
+    account_limits_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    category ENUM('api_rate','voice_call_session', 'device') NOT NULL,
+    quantity INTEGER NOT NULL,
+    PRIMARY KEY (account_limits_sid)
+    )`,
+    'CREATE INDEX service_provider_sid_idx ON service_provider_limits (service_provider_sid)',
+    `ALTER TABLE service_provider_limits 
+    ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) 
+    REFERENCES service_providers (service_provider_sid) 
+    ON DELETE CASCADE`,
+    'CREATE INDEX account_sid_idx ON account_limits (account_sid)',
+    `ALTER TABLE account_limits 
+    ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) 
+    REFERENCES accounts (account_sid) 
+    ON DELETE CASCADE`,
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_from_user` VARCHAR(128)',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_from_domain` VARCHAR(256)',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_public_ip_in_contact` BOOLEAN NOT NULL DEFAULT false'
+  ],
+  '8000': [
+    'ALTER TABLE `applications` ADD COLUMN `app_json` TEXT',
+    'ALTER TABLE voip_carriers CHANGE register_public_domain_in_contact register_public_ip_in_contact BOOLEAN',
+    'alter table phone_numbers modify number varchar(132) NOT NULL UNIQUE',
+    `CREATE TABLE permissions
+    (
+    permission_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(32) NOT NULL UNIQUE ,
+    description VARCHAR(255),
+    PRIMARY KEY (permission_sid)
+    )`,
+    `CREATE TABLE user_permissions
+    (
+    user_permissions_sid CHAR(36) NOT NULL UNIQUE ,
+    user_sid CHAR(36) NOT NULL,
+    permission_sid CHAR(36) NOT NULL,
+    PRIMARY KEY (user_permissions_sid)
+    )`,
+    `CREATE TABLE password_settings
+    (
+    min_password_length INTEGER NOT NULL DEFAULT 8,
+    require_digit BOOLEAN NOT NULL DEFAULT false,
+    require_special_character BOOLEAN NOT NULL DEFAULT false
+    )`,
+    'CREATE INDEX user_permissions_sid_idx ON user_permissions (user_permissions_sid)',
+    'CREATE INDEX user_sid_idx ON user_permissions (user_sid)',
+    'ALTER TABLE user_permissions ADD FOREIGN KEY user_sid_idxfk (user_sid) REFERENCES users (user_sid) ON DELETE CASCADE',
+    'ALTER TABLE user_permissions ADD FOREIGN KEY permission_sid_idxfk (permission_sid) REFERENCES permissions (permission_sid)',
+    'ALTER TABLE `users` ADD COLUMN `is_active` BOOLEAN NOT NULL default true',
+  ],
+  8003: [
+    'SET FOREIGN_KEY_CHECKS=0',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_status` VARCHAR(4096)',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `last_updated` DATETIME',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `tls_port` INTEGER',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `wss_port` INTEGER',
+    `CREATE TABLE system_information
+    (
+    domain_name VARCHAR(255),
+    sip_domain_name VARCHAR(255),
+    monitoring_domain_name VARCHAR(255)
+    )`,
+    'DROP TABLE IF EXISTS `lcr_routes`',
+    'DROP TABLE IF EXISTS `lcr_carrier_set_entry`',
+    `CREATE TABLE lcr_routes
+    (
+    lcr_route_sid CHAR(36),
+    lcr_sid CHAR(36) NOT NULL,
+    regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
+    description VARCHAR(1024),
+    priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
+    PRIMARY KEY (lcr_route_sid)
+    )`,
+    `CREATE TABLE lcr
+    (
+    lcr_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+    service_provider_sid CHAR(36),
+    account_sid CHAR(36),
+    PRIMARY KEY (lcr_sid)
+    )`,
+    `CREATE TABLE lcr_carrier_set_entry
+    (
+    lcr_carrier_set_entry_sid CHAR(36),
+    workload INTEGER NOT NULL DEFAULT 1 COMMENT 'represents a proportion of traffic to send through the associated carrier; can be used for load balancing traffic across carriers with a common priority for a destination',
+    lcr_route_sid CHAR(36) NOT NULL,
+    voip_carrier_sid CHAR(36) NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 0 COMMENT 'lower priority carriers are attempted first',
+    PRIMARY KEY (lcr_carrier_set_entry_sid)
+    )`,
+    'CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid)',
+    'ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid)',
+    'CREATE INDEX lcr_sid_idx ON lcr (lcr_sid)',
+    'ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid)',
+    'CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid)',
+    'CREATE INDEX account_sid_idx ON lcr (account_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY lcr_route_sid_idxfk (lcr_route_sid) REFERENCES lcr_routes (lcr_route_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_3 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid)',
+    'SET FOREIGN_KEY_CHECKS=1',
+  ],
+  8004: [
+    'alter table accounts add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table accounts add column bucket_credential VARCHAR(8192)',
+    'alter table accounts add column record_format VARCHAR(16) NOT NULL DEFAULT \'mp3\'',
+    'alter table applications add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table phone_numbers DROP INDEX number',
+    'create unique index phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid)',
+    `CREATE TABLE clients
+    (
+    client_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    username VARCHAR(64),
+    password VARCHAR(1024),
+    PRIMARY KEY (client_sid)
+    )`,
+    'CREATE INDEX client_sid_idx ON clients (client_sid)',
+    'ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid)',
+    'ALTER TABLE sip_gateways ADD COLUMN protocol ENUM(\'udp\',\'tcp\',\'tls\', \'tls/srtp\') DEFAULT \'udp\''
+  ]
+};
+
 const doIt = async() => {
   let connection;
   try {
@@ -35,10 +175,36 @@ const doIt = async() => {
   try {
     /* does the schema exist at all ? */
     const [r] = await connection.execute('SELECT version from schema_version');
+    let errors = 0;
     if (r.length) {
-      //TODO: check against desired version and perform upgrades
-      logger.info(`current version is ${r[0].version}, no upgrade will be performed`);
+      const {version} = r[0];
+      const arr = /v?(\d+)\.(\d+)\.(\d+)/.exec(version);
+      if (arr) {
+        const upgrades = [];
+        logger.info(`performing schema migration: ${version} => ${desiredVersion}`);
+        const val = (1000 * arr[1]) + (100 * arr[2]) + arr[3];
+        logger.info(`current schema value: ${val}`);
+
+        if (val < 7006) upgrades.push(...sql['7006']);
+        if (val < 7007) upgrades.push(...sql['7007']);
+        if (val < 8000) upgrades.push(...sql['8000']);
+        if (val < 8003) upgrades.push(...sql['8003']);
+        if (val < 8004) upgrades.push(...sql['8004']);
+
+        // perform all upgrades
+        logger.info({upgrades}, 'applying schema upgrades..');
+        for (const upgrade of upgrades) {
+          try {
+            await connection.execute(upgrade);
+          } catch (err) {
+            errors++;
+            logger.info({statement:upgrade, err}, 'Error applying statement');
+          }
+        }
+      }
+      if (errors === 0) await connection.execute(`UPDATE schema_version SET version = '${desiredVersion}'`);
       await connection.end();
+      logger.info(`schema migration to ${desiredVersion} completed with ${errors} errors`);
       return;
     }
   } catch (err) {
@@ -70,5 +236,6 @@ const seedDatabase = async(connection) => {
   await connection.query(sql);
 };
 
+
 doIt();
 

db/webapp-tests.sql

@@ -85,9 +85,6 @@ VALUES
 -- simwood gateways
 insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
 VALUES
-('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '149.91.14.0', 24, 5060, 1, 0),
-('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '154.51.137.96', 27, 5060, 1, 0),
-('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '78.40.245.160', 27, 5060, 1, 0),
 ('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.136.24', 29, 5060, 1, 0),
 ('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.136.28', 28, 5060, 1, 0),
 ('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.48', 28, 5060, 1, 0),
@@ -98,7 +95,7 @@ VALUES
 ('b6ae6240-55ac-4c11-892f-a71b2155ea60', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.0', 26, 5060, 1, 0),
 ('5a976337-164b-408e-8748-d8bfb4bd5d76', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.0', 26, 5060, 1, 0),
 ('ed0434ca-7f26-4624-9523-0419d0d2924d', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.0', 26, 5060, 1, 0),
-('d1a594c2-c14f-4ead-b621-96129bc87886', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.224.0', 24, 5060, 1, 0),
+('6bfb55e5-e248-48dc-a104-4f3eedd7d7de', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.0', 24, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
 

lib/auth/index.js

@@ -1,54 +1,56 @@
 const Strategy = require('passport-http-bearer').Strategy;
 const {getMysqlConnection} = require('../db');
-const {hashString} = require('../utils/password-utils');
 const debug = require('debug')('jambonz:api-server');
+const {cacheClient} = require('../helpers');
 const jwt = require('jsonwebtoken');
 const sql = `
   SELECT *
   FROM api_keys
   WHERE api_keys.token = ?`;
 
-function makeStrategy(logger, retrieveKey) {
+function makeStrategy(logger) {
   return new Strategy(
     async function(token, done) {
-      logger.debug(`validating with token ${token}`);
       jwt.verify(token, process.env.JWT_SECRET, async(err, decoded) => {
         if (err) {
           if (err.name === 'TokenExpiredError') {
             logger.debug('jwt expired');
             return done(null, false);
           }
-          /* its not a jwt obtained through login, check api leys */
+          /* its not a jwt obtained through login, check api keys */
           checkApiTokens(logger, token, done);
         }
         else {
-          /* validated -- make sure it is not on blacklist */
           try {
-            const s = `jwt:${hashString(token)}`;
-            const result = await retrieveKey(s);
-            if (result) {
-              debug(`result from searching for ${s}: ${result}`);
+            const {user_sid} = decoded;
+            /* Valid jwt tokens are stored in redis by hashed user_id */
+            const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+            const result = await cacheClient.get(redisKey);
+            if (result === null) {
+              debug(`result from searching for ${redisKey}: ${result}`);
               logger.info('jwt invalidated after logout');
               return done(null, false);
             }
-          } catch (err) {
+          } catch (error) {
             debug(err);
-            logger.info({err}, 'Error checking blacklist for jwt');
+            logger.info({err}, 'Error checking redis for jwt');
           }
-          const {user_sid, account_sid, email, name} = decoded;
-          //logger.debug({user_sid, account_sid}, 'successfully validated jwt');
-          const scope = ['account'];
+          const { user_sid, service_provider_sid, account_sid, email, name, scope, permissions } = decoded;
+
           const user = {
+            service_provider_sid,
             account_sid,
             user_sid,
             jwt: token,
             email,
             name,
-            hasScope: (s) => s === 'account',
-            hasAdminAuth: false,
-            hasServiceProviderAuth: false,
-            hasAccountAuth: true
+            permissions,
+            hasScope: (s) => s === scope,
+            hasAdminAuth: scope === 'admin',
+            hasServiceProviderAuth: scope === 'service_provider',
+            hasAccountAuth: scope === 'account'
           };
+          logger.debug({user}, 'successfully validated jwt');
           return done(null, user, {scope});
         }
       });
@@ -75,26 +77,30 @@ const checkApiTokens = (logger, token, done) => {
       }
 
       // found api key
-      const scope = [];
+      let scope;
+      //const scope = [];
       if (results[0].account_sid === null && results[0].service_provider_sid === null) {
-        scope.push.apply(scope, ['admin', 'service_provider', 'account']);
+        //scope.push.apply(scope, ['admin', 'service_provider', 'account']);
+        scope = 'admin';
       }
       else if (results[0].service_provider_sid) {
-        scope.push.apply(scope, ['service_provider', 'account']);
+        //scope.push.apply(scope, ['service_provider', 'account']);
+        scope = 'service_provider';
       }
       else {
-        scope.push('account');
+        //scope.push('account');
+        scope = 'account';
       }
 
       const user = {
         account_sid: results[0].account_sid,
         service_provider_sid: results[0].service_provider_sid,
-        hasScope: (s) => scope.includes(s),
-        hasAdminAuth: scope.length === 3,
-        hasServiceProviderAuth: scope.includes('service_provider'),
-        hasAccountAuth: scope.includes('account') && !scope.includes('service_provider')
+        hasScope: (s) => s === scope,
+        hasAdminAuth: scope === 'admin',
+        hasServiceProviderAuth: scope === 'service_provider',
+        hasAccountAuth: scope === 'account'
       };
-      logger.info(user, `successfully validated with scope ${scope}`);
+      logger.debug({user}, `successfully validated with scope ${scope}`);
       return done(null, user, {scope});
     });
   });

lib/helpers/cache-client.js

@@ -0,0 +1,82 @@
+const {
+  addKey: addKeyRedis,
+  deleteKey: deleteKeyRedis,
+  retrieveKey: retrieveKeyRedis,
+} = require('./realtimedb-helpers');
+const { hashString } = require('../utils/password-utils');
+const logger = require('../logger');
+
+class CacheClient {
+  constructor() { }
+
+  async set(params) {
+    const {
+      redisKey,
+      value = '1',
+      time = 3600,
+    } = params || {};
+
+    try {
+      await addKeyRedis(redisKey, value, time);
+
+    } catch (err) {
+      logger.error('CacheClient.get set', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        ...params
+      });
+    }
+  }
+
+  async get(redisKey) {
+    try {
+      const result = await retrieveKeyRedis(redisKey);
+      return result;
+    } catch (err) {
+      logger.error('CacheClient.get error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        redisKey
+      });
+    }
+  }
+
+
+  async delete(key) {
+    try {
+      await deleteKeyRedis(key);
+      logger.debug('CacheClient.delete key from redis', { key });
+    } catch (err) {
+      logger.error('CacheClient.delete error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        key
+      });
+    }
+  }
+
+  generateRedisKey(type, key, version) {
+    let suffix = '';
+    if (version) {
+      suffix = `:version:${version}`;
+    }
+
+    switch (type) {
+      case 'reset-link':
+        return `reset-link:${key}`;
+      case 'jwt':
+      default:
+        return `jwt:${hashString(key)}${suffix}`;
+    }
+  }
+}
+
+const cacheClient = new CacheClient();
+
+module.exports = { cacheClient };

lib/helpers/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require('./cache-client'),
+  ...require('./realtimedb-helpers'),
+};

lib/helpers/realtimedb-helpers.js

@@ -0,0 +1,54 @@
+const logger = require('../logger');
+
+const JAMBONES_REDIS_SENTINELS = process.env.JAMBONES_REDIS_SENTINELS ? {
+  sentinels: process.env.JAMBONES_REDIS_SENTINELS.split(',').map((sentinel) => {
+    let host, port = 26379;
+    if (sentinel.includes(':')) {
+      const arr = sentinel.split(':');
+      host = arr[0];
+      port = parseInt(arr[1], 10);
+    } else {
+      host = sentinel;
+    }
+    return {host, port};
+  }),
+  name: process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+  ...(process.env.JAMBONES_REDIS_SENTINEL_PASSWORD && {
+    password: process.env.JAMBONES_REDIS_SENTINEL_PASSWORD
+  }),
+  ...(process.env.JAMBONES_REDIS_SENTINEL_USERNAME && {
+    username: process.env.JAMBONES_REDIS_SENTINEL_USERNAME
+  })
+} : null;
+
+const {
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  incrKey,
+  client: redisClient,
+} = require('@jambonz/realtimedb-helpers')(JAMBONES_REDIS_SENTINELS || {
+  host: process.env.JAMBONES_REDIS_HOST || 'localhost',
+  port: process.env.JAMBONES_REDIS_PORT || 6379
+}, logger);
+
+module.exports = {
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  redisClient,
+  incrKey,
+  JAMBONES_REDIS_SENTINELS
+};

lib/logger.js

@@ -0,0 +1,7 @@
+const opts = {
+  level: process.env.JAMBONES_LOGLEVEL || 'info'
+};
+const pino = require('pino');
+const logger = pino(opts, pino.destination(1, {sync: false}));
+
+module.exports = logger;

lib/middleware.js

@@ -0,0 +1,32 @@
+const logger = require('./logger');
+
+function delayLoginMiddleware(req, res, next) {
+  if (req.path.includes('/login') || req.path.includes('/signin')) {
+    const min = 200;
+    const max = 1000;
+    /* Random delay between 200 - 1000ms */
+    const sendStatusDelay = Math.floor(Math.random() * (max - min + 1)) + min;
+
+    /* the res.json take longer, we decrease the max delay slightly to 0-800ms */
+    const jsonDelay = Math.floor(Math.random() * 800);
+    logger.debug(`delayLoginMiddleware: sendStatus ${sendStatusDelay} - json ${jsonDelay}`);
+    const sendStatus = res.sendStatus;
+    const json = res.json;
+
+    res.sendStatus = function(status) {
+      setTimeout(() => {
+        sendStatus.call(res, status);
+      }, sendStatusDelay);
+    };
+    res.json = function(body) {
+      setTimeout(() => {
+        json.call(res, body);
+      }, jsonDelay);
+    };
+  }
+  next();
+}
+
+module.exports = {
+  delayLoginMiddleware
+};

lib/models/account-limits.js

@@ -0,0 +1,41 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sql = 'SELECT * FROM account_limits WHERE account_sid = ?';
+
+class AccountLimits extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieve(account_sid) {
+    const [rows] = await promisePool.query(sql, [account_sid]);
+    return rows;
+  }
+
+}
+
+AccountLimits.table = 'account_limits';
+AccountLimits.fields = [
+  {
+    name: 'account_limits_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'category',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'quantity',
+    type: 'number',
+    required: true
+  }
+];
+
+module.exports = AccountLimits;

lib/models/account.js

@@ -4,7 +4,7 @@ const {getMysqlConnection} = require('../db');
 const {promisePool} = require('../db');
 const { v4: uuid } = require('uuid');
 
-const {encrypt} = require('../utils/encrypt-decrypt');
+const {encrypt, decrypt} = require('../utils/encrypt-decrypt');
 
 const retrieveSql = `SELECT * from accounts acc
 LEFT JOIN webhooks AS rh
@@ -55,6 +55,13 @@ WHERE account_sid = ?
 AND effective_end_date IS NULL 
 AND pending = 0`;
 
+const extractBucketCredential = (obj) => {
+  const {bucket_credential} = obj;
+  if (bucket_credential) {
+    obj.bucket_credential = JSON.parse(decrypt(bucket_credential));
+  }
+};
+
 function transmogrifyResults(results) {
   return results.map((row) => {
     const obj = row.acc;
@@ -75,6 +82,8 @@ function transmogrifyResults(results) {
     else obj.queue_event_hook = null;
     delete obj.queue_event_hook_sid;
 
+    extractBucketCredential(obj);
+
     return obj;
   });
 }
@@ -238,7 +247,6 @@ class Account extends Model {
     }));
     return account_subscription_sid;
   }
-
 }
 
 Account.table = 'accounts';
@@ -314,6 +322,22 @@ Account.fields = [
     name: 'subspace_sip_teleport_destinations',
     type: 'string',
   },
+  {
+    name: 'siprec_hook_sid',
+    type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number'
+  },
+  {
+    name: 'record_format',
+    type: 'string'
+  },
+  {
+    name: 'bucket_credential',
+    type: 'string'
+  }
 ];
 
 module.exports = Account;

lib/models/application.js

@@ -120,6 +120,10 @@ Application.fields = [
   {
     name: 'messaging_hook_sid',
     type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number',
   }
 ];
 

lib/models/client.js

@@ -0,0 +1,58 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Client extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(service_provider_sid) {
+    const sql = `SELECT c.client_sid, c.account_sid, c.is_active, c.username, c.hashed_password
+      FROM ${this.table} AS c LEFT JOIN accounts AS acc ON c.account_sid = acc.account_sid
+      LEFT JOIN service_providers AS sp ON sp.service_provider_sid = accs.service_provider_sid
+      WHERE sp.service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, service_provider_sid);
+    return rows;
+  }
+
+  static async retrieveByAccountSidAndUserName(account_sid, username) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ? AND username = ?`;
+    const [rows] = await promisePool.query(sql, [account_sid, username]);
+    return rows;
+  }
+}
+
+Client.table = 'clients';
+Client.fields = [
+  {
+    name: 'client_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+  {
+    name: 'username',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'password',
+    type: 'string'
+  }
+];
+
+module.exports = Client;

lib/models/lcr-carrier-set-entry.js

@@ -0,0 +1,47 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class LcrCarrierSetEntry extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrRouteSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_route_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrRouteSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_route_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+}
+
+LcrCarrierSetEntry.table = 'lcr_carrier_set_entry';
+LcrCarrierSetEntry.fields = [
+  {
+    name: 'lcr_carrier_set_entry_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'workload',
+    type: 'number'
+  },
+  {
+    name: 'lcr_route_sid',
+    type: 'string'
+  },
+  {
+    name: 'voip_carrier_sid',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrCarrierSetEntry;

lib/models/lcr-route.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+
+class LcrRoutes extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+
+  static async countAllByLcrSid(sid) {
+    const sql = `SELECT COUNT(*) AS count FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.length ? rows[0].count : 0;
+  }
+}
+
+LcrRoutes.table = 'lcr_routes';
+LcrRoutes.fields = [
+  {
+    name: 'lcr_route_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
+  },
+  {
+    name: 'regex',
+    type: 'string'
+  },
+  {
+    name: 'description',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrRoutes;

lib/models/lcr.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Lcr extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async releaseDefaultEntry(sid) {
+    const sql = `UPDATE ${this.table} SET default_carrier_set_entry_sid = null WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+}
+
+Lcr.table = 'lcr';
+Lcr.fields = [
+  {
+    name: 'lcr_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  },
+  {
+    name: 'default_carrier_set_entry_sid',
+    type: 'string'
+  }
+];
+
+module.exports = Lcr;

lib/models/model.js

@@ -107,7 +107,7 @@ class Model extends Emitter {
       if (pk.name in obj) throw new DbErrorBadRequest(`primary key ${pk.name} is immutable`);
       getMysqlConnection((err, conn) => {
         if (err) return reject(err);
-        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = '${sid}'`, obj, (err, results, fields) => {
+        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = ?`, [obj, sid], (err, results, fields) => {
           conn.release();
           if (err) return reject(err);
           resolve(results.affectedRows);

lib/models/password-settings.js

@@ -0,0 +1,70 @@
+const {promisePool} = require('../db');
+
+class PasswordSettings {
+
+  /**
+   * Retrieve object from database
+   */
+  static async retrieve() {
+    const [r] = await promisePool.execute(`SELECT * FROM ${this.table}`);
+    return r;
+  }
+
+  /**
+  * Update object into the database
+  */
+  static async update(obj) {
+    let sql = `UPDATE ${this.table} SET `;
+    const values = [];
+    const keys = Object.keys(obj);
+    this.fields.forEach(({name}) => {
+      if (keys.includes(name)) {
+        sql = sql + `${name} = ?,`;
+        values.push(obj[name]);
+      }
+    });
+    if (values.length) {
+      sql = sql.slice(0, -1);
+      await promisePool.execute(sql, values);
+    }
+  }
+
+  /**
+     * insert object into the database
+     */
+  static async make(obj) {
+    let params = '', marks = '';
+    const values = [];
+    const keys = Object.keys(obj);
+    this.fields.forEach(({name}) => {
+      if (keys.includes(name)) {
+        params = params + `${name},`;
+        marks = marks + '?,';
+        values.push(obj[name]);
+      }
+    });
+    if (values.length) {
+      params = `(${params.slice(0, -1)})`;
+      marks = `values(${marks.slice(0, -1)})`;
+      return await promisePool.execute(`INSERT into ${this.table} ${params} ${marks}`, values);
+    }
+  }
+}
+
+
+PasswordSettings.table = 'password_settings';
+PasswordSettings.fields = [
+  {
+    name: 'min_password_length',
+    type: 'number'
+  },
+  {
+    name: 'require_digit',
+    type: 'number'
+  },
+  {
+    name: 'require_special_character',
+    type: 'number'
+  }
+];
+module.exports = PasswordSettings;

lib/models/phone-number.js

@@ -1,6 +1,6 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
-const sql = 'SELECT * from phone_numbers WHERE account_sid = ?';
+const sql = 'SELECT * from phone_numbers WHERE account_sid = ? ORDER BY number';
 const sqlSP = `SELECT * 
 FROM phone_numbers 
 WHERE account_sid IN 
@@ -8,7 +8,7 @@ WHERE account_sid IN
   SELECT account_sid 
   FROM accounts 
   WHERE service_provider_sid = ?
-)`;
+) ORDER BY number`;
 
 class PhoneNumber extends Model {
   constructor() {

lib/models/service-provider-limits.js

@@ -0,0 +1,39 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sql = 'SELECT * FROM service_provider_limits WHERE service_provider_sid = ?';
+
+class ServiceProviderLimits extends Model {
+  constructor() {
+    super();
+  }
+  static async retrieve(service_provider_sid) {
+    const [rows] = await promisePool.query(sql, [service_provider_sid]);
+    return rows;
+  }
+}
+
+ServiceProviderLimits.table = 'service_provider_limits';
+ServiceProviderLimits.fields = [
+  {
+    name: 'service_provider_limits_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'category',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'quantity',
+    type: 'number',
+    required: true
+  }
+];
+
+module.exports = ServiceProviderLimits;

lib/models/service-provider.js

@@ -89,6 +89,10 @@ ServiceProvider.fields = [
   {
     name: 'ms_teams_fqdn',
     type: 'string',
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
   }
 
 ];

lib/models/sip-gateway.js

@@ -58,6 +58,10 @@ SipGateway.fields = [
   {
     name: 'application_sid',
     type: 'string'
+  },
+  {
+    name: 'protocol',
+    type: 'string'
   }
 ];
 

lib/models/speech-credential.js

@@ -1,7 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
 const retrieveSql = 'SELECT * from speech_credentials WHERE account_sid = ?';
-const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ?';
+const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ? and account_sid is null';
 
 class SpeechCredential extends Model {
   constructor() {

lib/models/system-information.js

@@ -0,0 +1,38 @@
+const Model = require('./model');
+const { promisePool } = require('../db');
+class SystemInformation extends Model {
+  constructor() {
+    super();
+  }
+
+  static async add(body) {
+    let [sysInfo] = await this.retrieveAll();
+    if (sysInfo) {
+      const sql = `UPDATE ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    } else {
+      const sql = `INSERT INTO ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    }
+    [sysInfo] = await this.retrieveAll();
+    return sysInfo;
+  }
+}
+
+SystemInformation.table = 'system_information';
+SystemInformation.fields = [
+  {
+    name: 'domain_name',
+    type: 'string',
+  },
+  {
+    name: 'sip_domain_name',
+    type: 'string',
+  },
+  {
+    name: 'monitoring_domain_name',
+    type: 'string',
+  },
+];
+
+module.exports = SystemInformation;

lib/models/user.js

@@ -0,0 +1,117 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sqlAll = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+`;
+const sqlAccount = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+WHERE u.account_sid = ? 
+`;
+const sqlSP = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+WHERE u.service_provider_sid = ? 
+`;
+
+class User extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAll() {
+    const [rows] = await promisePool.query(sqlAll);
+    return rows;
+  }
+
+  static async retrieveAllForAccount(account_sid) {
+    const [rows] = await promisePool.query(sqlAccount, [account_sid]);
+    return rows;
+  }
+
+  static async retrieveAllForServiceProvider(service_provider_sid) {
+    const [rows] = await promisePool.query(sqlSP, [service_provider_sid]);
+    return rows;
+  }
+}
+
+User.table = 'users';
+User.fields = [
+  {
+    name: 'user_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'email',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'pending_email',
+    type: 'string'
+  },
+  {
+    name: 'phone',
+    type: 'string'
+  },
+  {
+    name: 'hashed_password',
+    type: 'string'
+  },
+  {
+    name: 'account_sid',
+    type: 'string'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  },
+  {
+    name: 'force_change',
+    type: 'number'
+  },
+  {
+    name: 'provider',
+    type: 'string'
+  },
+  {
+    name: 'provider_userid',
+    type: 'string'
+  },
+  {
+    name: 'email_activation_code',
+    type: 'string'
+  },
+  {
+    name: 'email_validated',
+    type: 'number'
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+];
+
+module.exports = User;

lib/models/voip-carrier.js

@@ -11,10 +11,16 @@ class VoipCarrier extends Model {
   static async retrieveAll(account_sid) {
     if (!account_sid) return super.retrieveAll();
     const [rows] = await promisePool.query(retrieveSql, account_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
     return rows;
   }
   static async retrieveAllForSP(service_provider_sid) {
     const [rows] = await promisePool.query(retrieveSqlForSP, service_provider_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
     return rows;
   }
 }
@@ -111,6 +117,22 @@ VoipCarrier.fields = [
     name: 'smpp_system_id',
     type: 'string'
   },
+  {
+    name: 'register_from_user',
+    type: 'string'
+  },
+  {
+    name: 'register_from_domain',
+    type: 'string'
+  },
+  {
+    name: 'register_public_ip_in_contact',
+    type: 'number'
+  },
+  {
+    name: 'register_status',
+    type: 'string'
+  }
 ];
 
 module.exports = VoipCarrier;

lib/record/encoder.js

@@ -0,0 +1,54 @@
+const { Transform } = require('stream');
+const lamejs = require('@jambonz/lamejs');
+
+class PCMToMP3Encoder extends Transform {
+  constructor(options) {
+    super(options);
+
+    const channels = options.channels || 1;
+    const sampleRate = options.sampleRate || 8000;
+    const bitRate = options.bitRate || 128;
+
+    this.encoder = new lamejs.Mp3Encoder(channels, sampleRate, bitRate);
+    this.channels = channels;
+  }
+
+  _transform(chunk, encoding, callback) {
+    // Convert chunk buffer into Int16Array for lamejs
+    const samples = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
+
+    // Split input samples into left and right channel arrays if stereo
+    let leftChannel, rightChannel;
+    if (this.channels === 2) {
+      leftChannel = new Int16Array(samples.length / 2);
+      rightChannel = new Int16Array(samples.length / 2);
+
+      for (let i = 0; i < samples.length; i += 2) {
+        leftChannel[i / 2] = samples[i];
+        rightChannel[i / 2] = samples[i + 1];
+      }
+    } else {
+      leftChannel = samples;
+    }
+
+    // Encode the input data
+    const mp3Data = this.encoder.encodeBuffer(leftChannel, rightChannel);
+
+    if (mp3Data.length > 0) {
+      this.push(Buffer.from(mp3Data));
+    }
+    callback();
+  }
+
+  _flush(callback) {
+    // Finalize encoding and flush the internal buffers
+    const mp3Data = this.encoder.flush();
+
+    if (mp3Data.length > 0) {
+      this.push(Buffer.from(mp3Data));
+    }
+    callback();
+  }
+}
+
+module.exports = PCMToMP3Encoder;

lib/record/google-storage.js

@@ -0,0 +1,41 @@
+const { Storage } = require('@google-cloud/storage');
+const { Writable } = require('stream');
+
+class GoogleStorageUploadStream extends Writable {
+
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.metadata = opts.metadata;
+
+    const storage = new Storage(opts.bucketCredential);
+    this.gcsFile = storage.bucket(opts.bucketName).file(opts.Key);
+    this.writeStream = this.gcsFile.createWriteStream();
+
+    this.writeStream.on('error', (err) => this.logger.error(err));
+    this.writeStream.on('finish', () => {
+      this.logger.info('google storage Upload completed.');
+      this._addMetadata();
+    });
+  }
+
+  _write(chunk, encoding, callback) {
+    this.writeStream.write(chunk, encoding, callback);
+  }
+
+  _final(callback) {
+    this.writeStream.end();
+    this.writeStream.once('finish', callback);
+  }
+
+  async _addMetadata() {
+    try {
+      await this.gcsFile.setMetadata({metadata: this.metadata});
+      this.logger.info('Google storage Upload and metadata setting completed.');
+    } catch (err) {
+      this.logger.error(err, 'Google storage  An error occurred while setting metadata');
+    }
+  }
+}
+
+module.exports = GoogleStorageUploadStream;

lib/record/index.js

@@ -0,0 +1,6 @@
+
+async function record(logger, socket) {
+  return require('./upload')(logger, socket);
+}
+
+module.exports = record;

lib/record/s3-multipart-upload-stream.js

@@ -0,0 +1,103 @@
+const { Writable } = require('stream');
+const {
+  S3Client,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+} = require('@aws-sdk/client-s3');
+
+class S3MultipartUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.bucketName = opts.bucketName;
+    this.objectKey = opts.Key;
+    this.uploadId = null;
+    this.partNumber = 1;
+    this.multipartETags = [];
+    this.buffer = Buffer.alloc(0);
+    this.minPartSize = 5 * 1024 * 1024; // 5 MB
+    this.s3 = new S3Client(opts.bucketCredential);
+    this.metadata = opts.metadata;
+  }
+
+  async _initMultipartUpload() {
+    const command = new CreateMultipartUploadCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      Metadata: this.metadata
+    });
+    const response = await this.s3.send(command);
+    return response.UploadId;
+  }
+
+  async _uploadBuffer() {
+    const uploadPartCommand = new UploadPartCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      PartNumber: this.partNumber,
+      UploadId: this.uploadId,
+      Body: this.buffer,
+    });
+
+    const uploadPartResponse = await this.s3.send(uploadPartCommand);
+    this.multipartETags.push({
+      ETag: uploadPartResponse.ETag,
+      PartNumber: this.partNumber,
+    });
+    this.partNumber += 1;
+  }
+
+  async _write(chunk, encoding, callback) {
+    try {
+      if (!this.uploadId) {
+        this.uploadId = await this._initMultipartUpload();
+      }
+
+      this.buffer = Buffer.concat([this.buffer, chunk]);
+
+      if (this.buffer.length >= this.minPartSize) {
+        await this._uploadBuffer();
+        this.buffer = Buffer.alloc(0);
+      }
+
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _finalize(err) {
+    try {
+      if (this.buffer.length > 0) {
+        await this._uploadBuffer();
+      }
+
+      const completeMultipartUploadCommand = new CompleteMultipartUploadCommand({
+        Bucket: this.bucketName,
+        Key: this.objectKey,
+        MultipartUpload: {
+          Parts: this.multipartETags.sort((a, b) => a.PartNumber - b.PartNumber),
+        },
+        UploadId: this.uploadId,
+      });
+
+      await this.s3.send(completeMultipartUploadCommand);
+      this.logger.info('Finished upload to S3');
+    } catch (error) {
+      this.logger.error('Error completing multipart upload:', error);
+      throw error;
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this._finalize();
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = S3MultipartUploadStream;

lib/record/upload.js

@@ -0,0 +1,87 @@
+const Account = require('../models/account');
+const Websocket = require('ws');
+const PCMToMP3Encoder = require('./encoder');
+const wav = require('wav');
+const { getUploader } = require('./utils');
+
+async function upload(logger, socket) {
+
+  socket._recvInitialMetadata = false;
+  socket.on('message', async function(data, isBinary) {
+    try {
+      if (!isBinary && !socket._recvInitialMetadata) {
+        socket._recvInitialMetadata = true;
+        logger.debug(`initial metadata: ${data}`);
+        const obj = JSON.parse(data.toString());
+        logger.info({obj}, 'received JSON message from jambonz');
+        const {sampleRate, accountSid, callSid, direction, from, to,
+          callId, applicationSid, originatingSipIp, originatingSipTrunkName} = obj;
+        const account = await Account.retrieve(accountSid);
+        if (account && account.length && account[0].bucket_credential) {
+          const obj = account[0].bucket_credential;
+          // add tags to metadata
+          const metadata = {
+            accountSid,
+            callSid,
+            direction,
+            from,
+            to,
+            callId,
+            applicationSid,
+            originatingSipIp,
+            originatingSipTrunkName,
+            sampleRate: `${sampleRate}`
+          };
+          if (obj.tags && obj.tags.length) {
+            obj.tags.forEach((tag) => {
+              metadata[tag.Key] = tag.Value;
+            });
+          }
+          // create S3 path
+          const day = new Date();
+          let Key = `${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
+          Key += `/${day.getDate().toString().padStart(2, '0')}/${callSid}.${account[0].record_format}`;
+
+          // Uploader
+          const uploadStream = getUploader(Key, metadata, obj, logger);
+          if (!uploadStream) {
+            logger.info('There is no available record uploader, close the socket.');
+            socket.close();
+          }
+
+          /**encoder */
+          let encoder;
+          if (obj.output_format === 'wav') {
+            encoder = new wav.Writer({ channels: 2, sampleRate, bitDepth: 16 });
+          } else {
+            // default is mp3
+            encoder = new PCMToMP3Encoder({
+              channels: 2,
+              sampleRate: sampleRate,
+              bitrate: 128
+            });
+          }
+          /* start streaming data */
+          const duplex = Websocket.createWebSocketStream(socket);
+          duplex.pipe(encoder).pipe(uploadStream);
+        } else {
+          logger.info(`account ${accountSid} does not have any bucket credential, close the socket`);
+          socket.close();
+        }
+      }
+    } catch (err) {
+      logger.error({err, data}, 'error parsing message during connection');
+    }
+  });
+  socket.on('error', function(err) {
+    logger.error({err}, 'aws upload: error');
+  });
+  socket.on('close', (data) => {
+    logger.info({data}, 'aws_s3: close');
+  });
+  socket.on('end', function(err) {
+    logger.error({err}, 'aws upload: socket closed from jambonz');
+  });
+}
+
+module.exports = upload;

lib/record/utils.js

@@ -0,0 +1,40 @@
+const GoogleStorageUploadStream = require('./google-storage');
+const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
+
+const getUploader = (Key, metadata, bucket_credential, logger) => {
+  const uploaderOpts = {
+    bucketName: bucket_credential.name,
+    Key,
+    metadata
+  };
+  switch (bucket_credential.vendor) {
+    case 'aws_s3':
+      uploaderOpts.bucketCredential = {
+        credentials: {
+          accessKeyId: bucket_credential.access_key_id,
+          secretAccessKey: bucket_credential.secret_access_key,
+        },
+        region: bucket_credential.region || 'us-east-1'
+      };
+      return new S3MultipartUploadStream(logger, uploaderOpts);
+    case 'google':
+      const serviceKey = JSON.parse(bucket_credential.service_key);
+      uploaderOpts.bucketCredential = {
+        projectId: serviceKey.project_id,
+        credentials: {
+          client_email: serviceKey.client_email,
+          private_key: serviceKey.private_key
+        }
+      };
+      return new GoogleStorageUploadStream(logger, uploaderOpts);
+
+    default:
+      logger.error(`unknown bucket vendor: ${bucket_credential.vendor}`);
+      break;
+  }
+  return null;
+};
+
+module.exports = {
+  getUploader
+};

lib/routes/api/accounts.js

@@ -1,6 +1,7 @@
 const router = require('express').Router();
+const assert = require('assert');
 const request = require('request');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden, DbErrorUnprocessableRequest} = require('../../utils/errors');
 const Account = require('../../models/account');
 const Application = require('../../models/application');
 const Webhook = require('../../models/webhook');
@@ -12,15 +13,27 @@ const { v4: uuidv4 } = require('uuid');
 const snakeCase = require('../../utils/snake-case');
 const sysError = require('../error');
 const {promisePool} = require('../../db');
-const {hasAccountPermissions, parseAccountSid, enableSubspace, disableSubspace} = require('./utils');
+const {
+  hasAccountPermissions,
+  parseAccountSid,
+  parseCallSid,
+  enableSubspace,
+  disableSubspace,
+  parseVoipCarrierSid
+} = require('./utils');
 const short = require('short-uuid');
 const VoipCarrier = require('../../models/voip-carrier');
+const { encrypt } = require('../../utils/encrypt-decrypt');
+const { testAwsS3, testGoogleStorage } = require('../../utils/storage-utils');
 const translator = short();
 
 let idx = 0;
 
 const getFsUrl = async(logger, retrieveSet, setName) => {
-  if (process.env.K8S) return `http://${process.env.K8S_FEATURE_SERVER_SERVICE_NAME}:3000/v1/createCall`;
+  if (process.env.K8S) {
+    const port = process.env.K8S_FEATURE_SERVER_SERVICE_PORT || 3000;
+    return `http://${process.env.K8S_FEATURE_SERVER_SERVICE_NAME}:${port}/v1/createCall`;
+  }
 
   try {
     const fs = await retrieveSet(setName);
@@ -28,18 +41,42 @@ const getFsUrl = async(logger, retrieveSet, setName) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/createCall`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/createCall`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
+const validateRequest = async(req, account_sid) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('account')) {
+      if (account_sid === req.user.account_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+      );
+
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  } catch (error) {
+    throw error;
+  }
 };
 
 router.use('/:sid/SpeechCredentials', hasAccountPermissions, require('./speech-credentials'));
@@ -48,10 +85,13 @@ router.use('/:sid/Alerts', hasAccountPermissions, require('./alerts'));
 router.use('/:sid/Charges', hasAccountPermissions, require('./charges'));
 router.use('/:sid/SipRealms', hasAccountPermissions, require('./sip-realm'));
 router.use('/:sid/PredefinedCarriers', hasAccountPermissions, require('./add-from-predefined-carrier'));
+router.use('/:sid/Limits', hasAccountPermissions, require('./limits'));
+router.use('/:sid/TtsCache', hasAccountPermissions, require('./tts-cache'));
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const results = await Application.retrieveAll(null, account_sid);
     res.status(200).json(results);
   } catch (err) {
@@ -62,17 +102,39 @@ router.get('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const results = await VoipCarrier.retrieveAll(account_sid);
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
   }
 });
+router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const sid = parseVoipCarrierSid(req);
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
+    const rowsAffected = await VoipCarrier.update(sid, req.body);
+    if (rowsAffected === 0) {
+      return res.sendStatus(404);
+    }
+
+    return res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 router.post('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   const payload = req.body;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
     logger.debug({payload}, 'POST /:sid/VoipCarriers');
     const uuid = await VoipCarrier.make({
       account_sid,
@@ -117,7 +179,8 @@ function validateUpdateCall(opts) {
     'conf_hold_status',
     'conf_mute_status',
     'mute_status',
-    'sip_request'
+    'sip_request',
+    'record'
   ]
     .reduce((acc, prop) => (opts[prop] ? ++acc : acc), 0);
 
@@ -156,6 +219,12 @@ function validateUpdateCall(opts) {
     (!opts.sip_request.method && !opts.sip_request.content_type ||  !opts.sip_request.content_type)) {
     throw new DbErrorBadRequest('sip_request requires content_type and content properties');
   }
+  if (opts.record && !opts.record.action) {
+    throw new DbErrorBadRequest('record requires action property');
+  }
+  if ('startCallRecording' === opts.record?.action && !opts.record.siprecServerURL) {
+    throw new DbErrorBadRequest('record requires siprecServerURL property when starting recording');
+  }
 }
 
 function validateTo(to) {
@@ -175,6 +244,7 @@ function validateTo(to) {
   }
   throw new DbErrorBadRequest(`missing or invalid to property: ${JSON.stringify(to)}`);
 }
+
 async function validateCreateCall(logger, sid, req) {
   const {lookupAppBySid} = req.app.locals;
   const obj = req.body;
@@ -191,6 +261,7 @@ async function validateCreateCall(logger, sid, req) {
       const application = await lookupAppBySid(obj.application_sid);
       Object.assign(obj, {
         call_hook: application.call_hook,
+        app_json: application.app_json,
         call_status_hook: application.call_status_hook,
         speech_synthesis_vendor: application.speech_synthesis_vendor,
         speech_synthesis_language: application.speech_synthesis_language,
@@ -206,15 +277,14 @@ async function validateCreateCall(logger, sid, req) {
   }
   else {
     delete obj.application_sid;
-
-    // TODO: these should be retrieved from account, using account_sid if provided
-    Object.assign(obj, {
-      speech_synthesis_vendor: 'google',
-      speech_synthesis_voice: 'en-US-Wavenet-C',
-      speech_synthesis_language: 'en-US',
-      speech_recognizer_vendor: 'google',
-      speech_recognizer_language: 'en-US'
-    });
+    if (!obj.speech_synthesis_vendor ||
+      !obj.speech_synthesis_language ||
+      !obj.speech_synthesis_voice ||
+      !obj.speech_recognizer_vendor ||
+      !obj.speech_recognizer_language)
+      throw new DbErrorBadRequest('either application_sid or set of' +
+        ' speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice,' +
+        ' speech_recognizer_vendor, speech_recognizer_language required');
   }
 
   if (!obj.call_hook && !obj.application_sid) {
@@ -276,7 +346,7 @@ async function validateCreateMessage(logger, sid, req) {
 async function validateAdd(req) {
   /* account-level token can not be used to add accounts */
   if (req.user.hasAccountAuth) {
-    throw new DbErrorUnprocessableRequest('insufficient permissions to create accounts');
+    throw new DbErrorForbidden('insufficient permissions');
   }
   if (req.user.hasServiceProviderAuth && req.user.service_provider_sid) {
     /* service providers can only create accounts under themselves */
@@ -295,9 +365,10 @@ async function validateAdd(req) {
     throw new DbErrorBadRequest('\'queue_event_hook\' must be an object when adding an account');
   }
 }
+
 async function validateUpdate(req, sid) {
   if (req.user.hasAccountAuth && req.user.account_sid !== sid) {
-    throw new DbErrorUnprocessableRequest('insufficient privileges to update this account');
+    throw new DbErrorForbidden('insufficient privileges');
   }
   if (req.user.hasAccountAuth && req.body.sip_realm) {
     throw new DbErrorBadRequest('use POST /Accounts/:sid/sip_realm/:realm to set or change the sip realm');
@@ -305,12 +376,23 @@ async function validateUpdate(req, sid) {
 
   if (req.user.service_provider_sid && !req.user.hasScope('admin')) {
     const result = await Account.retrieve(sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${sid}`);
+    }
     if (result[0].service_provider_sid !== req.user.service_provider_sid) {
-      throw new DbErrorUnprocessableRequest('cannot update account from different service provider');
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
+  if (req.user.hasScope('admin')) {
+    /* check to be sure that the account_sid exists */
+    const result = await Account.retrieve(sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${sid}`);
     }
   }
   if (req.body.service_provider_sid) throw new DbErrorBadRequest('service_provider_sid may not be modified');
 }
+
 async function validateDelete(req, sid) {
   if (req.user.hasAccountAuth && req.user.account_sid !== sid) {
     throw new DbErrorUnprocessableRequest('insufficient privileges to update this account');
@@ -318,12 +400,11 @@ async function validateDelete(req, sid) {
   if (req.user.service_provider_sid && !req.user.hasScope('admin')) {
     const result = await Account.retrieve(sid);
     if (result[0].service_provider_sid !== req.user.service_provider_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete account from different service provider');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
 }
 
-
 /* add */
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
@@ -365,8 +446,10 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     return res.status(200).json(results[0]);
   }
@@ -378,13 +461,15 @@ router.get('/:sid', async(req, res) => {
 router.get('/:sid/WebhookSecret', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     let {webhook_secret} = results[0];
     if (req.query.regenerate) {
       const secret = `wh_secret_${translator.generate()}`;
-      await Account.update(req.params.sid, {webhook_secret: secret});
+      await Account.update(account_sid, {webhook_secret: secret});
       webhook_secret = secret;
     }
     return res.status(200).json({webhook_secret});
@@ -397,8 +482,9 @@ router.get('/:sid/WebhookSecret', async(req, res) => {
 router.post('/:sid/SubspaceTeleport', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     const {subspace_client_id, subspace_client_secret} = results[0];
     const {destination} = req.body;
@@ -411,7 +497,7 @@ router.post('/:sid/SubspaceTeleport', async(req, res) => {
       destination: dest
     });
     logger.info({destination, teleport}, 'SubspaceTeleport - create teleport');
-    await Account.update(req.params.sid, {
+    await Account.update(account_sid, {
       subspace_sip_teleport_id: teleport.id,
       subspace_sip_teleport_destinations: JSON.stringify(teleport.teleport_entry_points)//hacky
     });
@@ -429,13 +515,14 @@ router.post('/:sid/SubspaceTeleport', async(req, res) => {
 router.delete('/:sid/SubspaceTeleport', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     const {subspace_client_id, subspace_client_secret, subspace_sip_teleport_id} = results[0];
 
     await disableSubspace({subspace_client_id, subspace_client_secret, subspace_sip_teleport_id});
-    await Account.update(req.params.sid, {
+    await Account.update(account_sid, {
       subspace_sip_teleport_id: null,
       subspace_sip_teleport_destinations: null
     });
@@ -446,11 +533,49 @@ router.delete('/:sid/SubspaceTeleport', async(req, res) => {
   }
 });
 
-/* update */
+function encryptBucketCredential(obj) {
+  if (!obj.bucket_credential) return;
+  const {
+    vendor,
+    region,
+    name,
+    access_key_id,
+    secret_access_key,
+    tags,
+    service_key
+  } = obj.bucket_credential;
+
+  switch (vendor) {
+    case 'aws_s3':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(region, 'invalid aws bucket region: region is required');
+      const awsData = JSON.stringify({vendor, region, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(awsData);
+      break;
+    case 'google':
+      assert(service_key, 'invalid aws S3 bucket credential: service_key is required');
+      const googleData = JSON.stringify({vendor, name, service_key, tags});
+      obj.bucket_credential = encrypt(googleData);
+      break;
+    case 'none':
+      obj.bucket_credential = null;
+      break;
+    default:
+      throw DbErrorBadRequest(`unknow storage vendor: ${vendor}`);
+  }
+}
+
+/**
+ * update
+ */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
 
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
@@ -489,6 +614,8 @@ router.put('/:sid', async(req, res) => {
       delete obj.registration_hook;
       delete obj.queue_event_hook;
 
+      encryptBucketCredential(obj);
+
       const rowsAffected = await Account.update(sid, obj);
       if (rowsAffected === 0) {
         return res.status(404).end();
@@ -510,12 +637,14 @@ router.put('/:sid', async(req, res) => {
 
 /* delete */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   const sqlDeleteGateways = `DELETE from sip_gateways 
     WHERE voip_carrier_sid IN
     (SELECT voip_carrier_sid from voip_carriers where account_sid = ?)`;
+
   try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
     await validateDelete(req, sid);
 
     const [account] = await promisePool.query('SELECT * FROM accounts WHERE account_sid = ?', sid);
@@ -579,13 +708,49 @@ account_subscriptions WHERE account_sid = ?)
   }
 });
 
-/* retrieve account level api keys */
-router.get('/:sid/ApiKeys', async(req, res) => {
+/* Test Bucket credential Keys */
+router.post('/:sid/BucketCredentialTest', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await ApiKey.retrieveAll(req.params.sid);
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const {vendor, name, region, access_key_id, secret_access_key, service_key} = req.body;
+    const ret = {
+      status: 'not tested'
+    };
+
+    switch (vendor) {
+      case 'aws_s3':
+        await testAwsS3(logger, {vendor, name, region, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 'google':
+        await testGoogleStorage(logger, {vendor, name, service_key});
+        ret.status = 'ok';
+        break;
+      default:
+        throw new DbErrorBadRequest(`Does not support test for ${vendor}`);
+    }
+    return res.status(200).json(ret);
+  }
+  catch (err) {
+    return res.status(200).json({status: 'failed', reason: err.message});
+  }
+});
+
+/**
+ * retrieve account level api keys
+ */
+router.get('/:sid/ApiKeys', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
+
+    const results = await ApiKey.retrieveAll(sid);
     res.status(200).json(results);
-    updateLastUsed(logger, req.params.sid, req).catch((err) => {});
+    updateLastUsed(logger, sid, req).catch((err) => {});
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -595,15 +760,19 @@ router.get('/:sid/ApiKeys', async(req, res) => {
  * create a new Call
  */
 router.post('/:sid/Calls', async(req, res) => {
-  const sid = req.params.sid;
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const {retrieveSet, logger} = req.app.locals;
-
+  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
   const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
-  if (!serviceUrl) res.json({msg: 'no available feature servers at this time'}).status(480);
-  try {
-    await validateCreateCall(logger, sid, req);
 
+  if (!serviceUrl) {
+    return res.status(480).json({msg: 'no available feature servers at this time'});
+  }
+
+  try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
+
+    await validateCreateCall(logger, sid, req);
     updateLastUsed(logger, sid, req).catch((err) => {});
     request({
       url: serviceUrl,
@@ -615,11 +784,13 @@ router.post('/:sid/Calls', async(req, res) => {
         logger.error(err, `Error sending createCall POST to ${serviceUrl}`);
         return res.sendStatus(500);
       }
+
       if (response.statusCode !== 201) {
         logger.error({statusCode: response.statusCode}, `Non-success response returned by createCall ${serviceUrl}`);
         return res.sendStatus(500);
       }
-      res.status(201).json(body);
+
+      return res.status(201).json(body);
     });
   } catch (err) {
     sysError(logger, res, err);
@@ -630,11 +801,18 @@ router.post('/:sid/Calls', async(req, res) => {
  * retrieve info for a group of calls under an account
  */
 router.get('/:sid/Calls', async(req, res) => {
-  const accountSid = req.params.sid;
   const {logger, listCalls} = req.app.locals;
-
+  const {direction, from, to, callStatus} = req.query || {};
   try {
-    const calls = await listCalls(accountSid);
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const calls = await listCalls({
+      accountSid,
+      direction,
+      from,
+      to,
+      callStatus
+    });
     logger.debug(`retrieved ${calls.length} calls for account sid ${accountSid}`);
     res.status(200).json(coerceNumbers(snakeCase(calls)));
     updateLastUsed(logger, accountSid, req).catch((err) => {});
@@ -647,11 +825,12 @@ router.get('/:sid/Calls', async(req, res) => {
  * retrieve single call
  */
 router.get('/:sid/Calls/:callSid', async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, retrieveCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     const callInfo = await retrieveCall(accountSid, callSid);
     if (callInfo) {
       logger.debug(callInfo, `retrieved call info for call sid ${callSid}`);
@@ -671,11 +850,12 @@ router.get('/:sid/Calls/:callSid', async(req, res) => {
  * delete call
  */
 router.delete('/:sid/Calls/:callSid', async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, deleteCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     const result = await deleteCall(accountSid, callSid);
     if (result) {
       logger.debug(`successfully deleted call ${callSid}`);
@@ -695,11 +875,12 @@ router.delete('/:sid/Calls/:callSid', async(req, res) => {
  * update a call
  */
 const updateCall = async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, retrieveCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     validateUpdateCall(req.body);
     const call = await retrieveCall(accountSid, callSid);
     if (call) {
@@ -726,6 +907,7 @@ const updateCall = async(req, res) => {
 router.post('/:sid/Calls/:callSid', async(req, res) => {
   await updateCall(req, res);
 });
+
 router.put('/:sid/Calls/:callSid', async(req, res) => {
   await updateCall(req, res);
 });
@@ -734,11 +916,13 @@ router.put('/:sid/Calls/:callSid', async(req, res) => {
  * create a new Message
  */
 router.post('/:sid/Messages', async(req, res) => {
-  const account_sid = parseAccountSid(req);
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const {retrieveSet, logger} = req.app.locals;
 
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
+    const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
     const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
     if (!serviceUrl) res.json({msg: 'no available feature servers at this time'}).status(480);
     await validateCreateMessage(logger, account_sid, req);
@@ -771,4 +955,22 @@ router.post('/:sid/Messages', async(req, res) => {
   }
 });
 
+/**
+ * retrieve info for a group of queues under an account
+ */
+router.get('/:sid/Queues', async(req, res) => {
+  const {logger, listSortedSets} = req.app.locals;
+  const { search } = req.query || {};
+  try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const queues = search ? await listSortedSets(accountSid, search) : await listSortedSets(accountSid);
+    logger.debug(`retrieved ${queues.length} queues for account sid ${accountSid}`);
+    res.status(200).json(queues);
+    updateLastUsed(logger, accountSid, req).catch((err) => {});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 module.exports = router;

lib/routes/api/add-from-predefined-carrier.js

@@ -8,9 +8,7 @@ const short = require('short-uuid');
 const {promisePool} = require('../../db');
 const sysError = require('../error');
 
-const sqlSelectCarrierByName = `SELECT * FROM voip_carriers 
-WHERE account_sid = ? 
-AND name = ?`;
+
 const sqlSelectCarrierByNameForSP = `SELECT * FROM voip_carriers 
 WHERE service_provider_sid = ? 
 AND name = ?`;
@@ -25,22 +23,20 @@ router.post('/:sid', async(req, res) => {
   const {sid } = req.params;
   let service_provider_sid;
   const {account_sid} = req.user;
-  if (!account_sid) {
-    if (!req.user.hasScope('service_provider')) {
-      logger.error({user: req.user}, 'invalid creds');
-      return res.sendStatus(403);
-    }
-    service_provider_sid = parseServiceProviderSid(req);
-  }
+
   try {
+    if (!account_sid) {
+      service_provider_sid = parseServiceProviderSid(req);
+    } else {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
     const [template] = await PredefinedCarrier.retrieve(sid);
     logger.debug({template}, `Retrieved template carrier for sid ${sid}`);
     if (!template) return res.sendStatus(404);
 
     /* make sure not to add the same carrier twice */
-    const [r2] = account_sid ?
-      await promisePool.query(sqlSelectCarrierByName, [account_sid, template.name]) :
-      await promisePool.query(sqlSelectCarrierByNameForSP, [service_provider_sid, template.name]);
+    const [r2] = await promisePool.query(sqlSelectCarrierByNameForSP, [service_provider_sid, template.name]);
 
     if (r2.length > 0) {
       template.name =  `${template.name}-${short.generate()}`;

lib/routes/api/alerts.js

@@ -1,6 +1,7 @@
 const router = require('express').Router();
 const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
+const { parseServiceProviderSid } = require('./utils');
 
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
@@ -8,25 +9,41 @@ const parseAccountSid = (url) => {
 };
 
 router.get('/', async(req, res) => {
-  const {logger, queryAlerts} = req.app.locals;
+  const {logger, queryAlerts, queryAlertsSP} = req.app.locals;
   try {
     logger.debug({opts: req.query}, 'GET /Alerts');
     const account_sid = parseAccountSid(req.originalUrl);
+    const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
     const {page, count, alert_type, days, start, end} = req.query || {};
     if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
     if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
 
-    const data = await queryAlerts({
-      account_sid,
-      page,
-      page_size: count,
-      alert_type,
-      days,
-      start: days ? undefined : start,
-      end: days ? undefined : end,
-    });
+    if (account_sid) {
+      const data = await queryAlerts({
+        account_sid,
+        page,
+        page_size: count,
+        alert_type,
+        days,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+      });
 
-    res.status(200).json(data);
+      res.status(200).json(data);
+    }
+    else {
+      const data = await queryAlertsSP({
+        service_provider_sid,
+        page,
+        page_size: count,
+        alert_type,
+        days,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+      });
+
+      res.status(200).json(data);
+    }
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/applications.js

@@ -1,16 +1,48 @@
 const router = require('express').Router();
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const Application = require('../../models/application');
 const Account = require('../../models/account');
 const Webhook = require('../../models/webhook');
 const {promisePool} = require('../../db');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const { validate } = require('@jambonz/verb-specifications');
+const { parseApplicationSid } = require('./utils');
 const preconditions = {
   'add': validateAdd,
   'update': validateUpdate
 };
 
+const validateRequest = async(req, account_sid) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('account')) {
+      if (account_sid === req.user.account_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+      );
+
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  } catch (error) {
+    throw error;
+  }
+};
+
 /* only user-level tokens can add applications */
 async function validateAdd(req) {
   if (req.user.account_sid) {
@@ -21,7 +53,7 @@ async function validateAdd(req) {
     if (!req.body.account_sid) throw new DbErrorBadRequest('missing required field: \'account_sid\'');
     const result = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
     if (result.length === 0) {
-      throw new DbErrorBadRequest('insufficient privileges to create an application under the specified account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
@@ -33,12 +65,26 @@ async function validateAdd(req) {
 }
 
 async function validateUpdate(req, sid) {
-  if (req.user.account_sid) {
-    const app = await Application.retrieve(sid);
-    if (!app || !app.length || app[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorBadRequest('you may not update or delete an application associated with a different account');
+  const app = await Application.retrieve(sid);
+  if (req.user.hasAccountAuth) {
+    if (!app || 0 === app.length || app[0].account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
+
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [app[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  }
+
+
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
     throw new DbErrorBadRequest('\'call_hook\' must be an object when updating an application');
   }
@@ -48,13 +94,24 @@ async function validateUpdate(req, sid) {
 }
 
 async function validateDelete(req, sid) {
+  const result = await Application.retrieve(sid);
   if (req.user.hasAccountAuth) {
-    const result = await Application.retrieve(sid);
     if (!result || 0 === result.length) throw new DbErrorBadRequest('application does not exist');
     if (result[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete application owned by a different account');
+      throw new DbErrorUnprocessableRequest('insufficient permissions');
     }
   }
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [result[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  }
   const assignedPhoneNumbers = await Application.getForeignKeyReferences('phone_numbers.application_sid', sid);
   if (assignedPhoneNumbers > 0) throw new DbErrorUnprocessableRequest('cannot delete application with phone numbers');
 }
@@ -76,6 +133,16 @@ router.post('/', async(req, res) => {
       }
     }
 
+    // validate app json if required
+    if (obj['app_json']) {
+      const app_json = JSON.parse(obj['app_json']);
+      try {
+        validate(logger, app_json);
+      } catch (err) {
+        throw new DbErrorBadRequest(err);
+      }
+    }
+
     const uuid = await Application.make(obj);
     res.status(201).json({sid: uuid});
   } catch (err) {
@@ -100,11 +167,13 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const application_sid = parseApplicationSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await Application.retrieve(req.params.sid, service_provider_sid, account_sid);
+    const results = await Application.retrieve(application_sid, service_provider_sid, account_sid);
     if (results.length === 0) return res.status(404).end();
-    return res.status(200).json(results);
+    await validateRequest(req, results[0].account_sid);
+    return res.status(200).json(results[0]);
   }
   catch (err) {
     sysError(logger, res, err);
@@ -113,9 +182,9 @@ router.get('/:sid', async(req, res) => {
 
 /* delete */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseApplicationSid(req);
     await validateDelete(req, sid);
 
     const [application] = await promisePool.query('SELECT * FROM applications WHERE application_sid = ?', sid);
@@ -154,9 +223,9 @@ router.delete('/:sid', async(req, res) => {
 
 /* update */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseApplicationSid(req);
     await validateUpdate(req, sid);
 
     // create webhooks if provided
@@ -179,6 +248,16 @@ router.put('/:sid', async(req, res) => {
       delete obj[prop];
     }
 
+    // validate app json if required
+    if (obj['app_json']) {
+      const app_json = JSON.parse(obj['app_json']);
+      try {
+        validate(logger, app_json);
+      } catch (err) {
+        throw new DbErrorBadRequest(err);
+      }
+    }
+
     const rowsAffected = await Application.update(sid, obj);
     if (rowsAffected === 0) {
       return res.status(404).end();

lib/routes/api/change-password.js

@@ -1,15 +1,15 @@
 const router = require('express').Router();
-//const debug = require('debug')('jambonz:api-server');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {generateHashedPassword, verifyPassword} = require('../../utils/password-utils');
 const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 const sqlUpdatePassword = `UPDATE users 
 SET hashed_password= ? 
 WHERE user_sid = ?`;
 
 router.post('/', async(req, res) => {
-  const {logger, retrieveKey, deleteKey} = req.app.locals;
+  const {logger} = req.app.locals;
   const {user_sid} = req.user;
   const {old_password, new_password} = req.body;
   try {
@@ -26,10 +26,10 @@ router.post('/', async(req, res) => {
 
       const isCorrect = await verifyPassword(r[0].hashed_password, old_password);
       if (!isCorrect) {
-        const key = `reset-link:${old_password}`;
-        const user_sid = await retrieveKey(key);
+        const key = cacheClient.generateRedisKey('reset-link', old_password);
+        const user_sid = await cacheClient.get(key);
         if (!user_sid) throw new DbErrorBadRequest('old_password is incorrect');
-        await deleteKey(key);
+        await cacheClient.delete(key);
       }
     }
 
@@ -42,6 +42,9 @@ router.post('/', async(req, res) => {
     sysError(logger, res, err);
     return;
   }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+  await cacheClient.delete(redisKey);
 });
 
 module.exports = router;

lib/routes/api/clients.js

@@ -0,0 +1,88 @@
+const router = require('express').Router();
+const decorate = require('./decorate');
+const sysError = require('../error');
+const Client = require('../../models/client');
+const Account = require('../../models/account');
+const { DbErrorBadRequest, DbErrorForbidden } = require('../../utils/errors');
+const { encrypt, decrypt, obscureKey } = require('../../utils/encrypt-decrypt');
+
+const commonCheck = async(req) => {
+  if (req.user.hasAccountAuth) {
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth && req.body.account_sid) {
+    const accounts = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
+    if (accounts.length === 0) {
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  }
+
+  if (req.body.password) {
+    req.body.password = encrypt(req.body.password);
+  }
+};
+
+const validateAdd = async(req) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length && clients[0].client_sid !== sid) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, Client, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Client.retrieveAll() : req.user.hasAccountAuth ?
+        await Client.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Client.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+    const ret = results.map((c) => {
+      c.password = obscureKey(decrypt(c.password), 1);
+      return c;
+    });
+    res.status(200).json(ret);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Client.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const client = results[0];
+    client.password = obscureKey(decrypt(client.password), 1);
+    if (req.user.hasAccountAuth && client.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth) {
+      const accounts = await Account.retrieve(client.account_sid, req.user.service_provider_sid);
+      if (!accounts.length) {
+        return res.sendStatus(404);
+      }
+    }
+    return res.status(200).json(client);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/error.js

@@ -1,6 +1,10 @@
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const { BadRequestError, DbErrorBadRequest, DbErrorUnprocessableRequest } = require('../../utils/errors');
 
 function sysError(logger, res, err) {
+  if (err instanceof BadRequestError) {
+    logger.info(err, err.message);
+    return res.status(400).json({msg: 'Bad request'});
+  }
   if (err instanceof DbErrorBadRequest) {
     logger.info(err, 'invalid client request');
     return res.status(400).json({msg: err.message});

lib/routes/api/forgot-password.js

@@ -4,6 +4,7 @@ const short = require('short-uuid');
 const translator = short();
 const {validateEmail, emailSimpleText} = require('../../utils/email-utils');
 const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 const sql = `SELECT * from users user 
 LEFT JOIN accounts AS acc 
@@ -45,6 +46,7 @@ function createResetEmailText(link) {
 router.post('/', async(req, res) => {
   const {logger, addKey} = req.app.locals;
   const {email} = req.body;
+
   let obj;
   try {
     if (!email || !validateEmail(email)) {
@@ -53,11 +55,16 @@ router.post('/', async(req, res) => {
 
     const [r] = await promisePool.query({sql, nestTables: true}, email);
     if (0 === r.length) {
-      return res.status(400).json({error: 'email does not exist'});
+      logger.info('user not found');
+      return res.status(400).json({error: 'failed to reset your password'});
     }
     obj = r[0];
-    if (!obj.acc.is_active) {
-      return res.status(400).json({error: 'you may not reset the password of an inactive account'});
+    if (!obj.user.is_active) {
+      logger.info(obj.user.name, 'user is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
+    } else if (obj.acc.account_sid !== null && !obj.acc.is_active) {
+      logger.info(obj.acc.account_sid, 'account is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
     }
     res.sendStatus(204);
   } catch (err) {
@@ -73,10 +80,14 @@ router.post('/', async(req, res) => {
   else {
     /* generate a link for this user to reset, send email */
     const link = translator.generate();
-    addKey(`reset-link:${link}`, obj.user.user_sid, 3600)
+    const redisKey = cacheClient.generateRedisKey('reset-link', link);
+    addKey(redisKey, obj.user.user_sid, 3600)
       .catch((err) => logger.error({err}, 'Error adding reset link to redis'));
     emailSimpleText(logger, email, 'Reset password request', createResetEmailText(link));
   }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', obj.user.user_sid, 'v2');
+  await cacheClient.delete(redisKey);
 });
 
 module.exports = router;

lib/routes/api/index.js

@@ -7,16 +7,18 @@ const isAdminScope = (req, res, next) => {
     message: 'insufficient privileges'
   });
 };
-const isAdminOrSPScope = (req, res, next) => {
-  if (req.user.hasScope('admin') || req.user.hasScope('service_provider')) return next();
-  res.status(403).json({
-    status: 'fail',
-    message: 'insufficient privileges'
-  });
-};
+// const isAdminOrSPScope = (req, res, next) => {
+//   if (req.user.hasScope('admin') || req.user.hasScope('service_provider')) return next();
+//   res.status(403).json({
+//     status: 'fail',
+//     message: 'insufficient privileges'
+//   });
+// };
 
 api.use('/BetaInviteCodes', isAdminScope, require('./beta-invite-codes'));
-api.use('/ServiceProviders', isAdminOrSPScope, require('./service-providers'));
+api.use('/SystemInformation', isAdminScope, require('./system-information'));
+api.use('/TtsCache', isAdminScope, require('./tts-cache'));
+api.use('/ServiceProviders', require('./service-providers'));
 api.use('/VoipCarriers', require('./voip-carriers'));
 api.use('/Webhooks', require('./webhooks'));
 api.use('/SipGateways', require('./sip-gateways'));
@@ -44,6 +46,12 @@ api.use('/Subscriptions', require('./subscriptions'));
 api.use('/Invoices', require('./invoices'));
 api.use('/InviteCodes', require('./invite-codes'));
 api.use('/PredefinedCarriers', require('./predefined-carriers'));
+api.use('/PasswordSettings', require('./password-settings'));
+// Least Cost Routing
+api.use('/Lcrs', require('./lcrs'));
+api.use('/LcrRoutes', require('./lcr-routes'));
+api.use('/LcrCarrierSetEntries', require('./lcr-carrier-set-entries'));
+api.use('/Clients', require('./clients'));
 
 // messaging
 api.use('/Smpps', require('./smpps'));   // our smpp server info

lib/routes/api/lcr-carrier-set-entries.js

@@ -0,0 +1,65 @@
+const router = require('express').Router();
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoute = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!req.body.lcr_route_sid) {
+    throw new DbErrorBadRequest('missing lcr_route_sid');
+  }
+  // check lcr_route_sid is exist
+  const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+  if (lcrRoute.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_route_sid');
+  }
+  // check voip_carrier_sid is exist
+  if (!req.body.voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+  const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+  if (!carrier) {
+    throw new DbErrorBadRequest('unknown voip_carrier_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (req.body.lcr_route_sid) {
+    const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+    if (lcrRoute.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_route_sid');
+    }
+  }
+
+  // check voip_carrier_sid is exist
+  if (req.body.voip_carrier_sid) {
+    const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+    if (!carrier) {
+      throw new DbErrorBadRequest('unknown voip_carrier_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, LcrCarrierSetEntry, ['add', 'retrieve', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.query.lcr_route_sid;
+  try {
+    const results = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(lcr_route_sid);
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcr-routes.js

@@ -0,0 +1,96 @@
+const router = require('express').Router();
+const LcrRoute = require('../../models/lcr-route');
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  // check if lcr sid is available
+  if (!req.body.lcr_sid) {
+    throw new DbErrorBadRequest('missing parameter lcr_sid');
+  }
+
+  const lcr = await Lcr.retrieve(req.body.lcr_sid);
+  if (lcr.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  if (req.body.lcr_sid) {
+    const lcr = await Lcr.retrieve(req.body.lcr_sid);
+    if (lcr.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateDelete = async(req, sid) => {
+  // delete all lcr carrier set entries
+  await LcrCarrierSetEntry.deleteByLcrRouteSid(sid);
+};
+
+const checkUserScope = async(req, lcr_sid) => {
+  if (!lcr_sid) {
+    throw new DbErrorBadRequest('missing lcr_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+
+  const lcrList = await Lcr.retrieve(lcr_sid);
+  if (lcrList.length === 0) throw new DbErrorBadRequest('unknown lcr_sid');
+  const lcr = lcrList[0];
+  if (req.user.hasAccountAuth) {
+    if (!lcr.account_sid || lcr.account_sid !== req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+
+  if (req.user.hasServiceProviderAuth) {
+    if (!lcr.service_provider_sid  || lcr.service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete,
+};
+
+decorate(router, LcrRoute, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_sid = req.query.lcr_sid;
+  try {
+    await checkUserScope(req, lcr_sid);
+    const results = await LcrRoute.retrieveAllByLcrSid(lcr_sid);
+    for (const r of results) {
+      r.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(r.lcr_route_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.params.sid;
+  try {
+    const results = await LcrRoute.retrieve(lcr_route_sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const route = results[0];
+    route.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(route.lcr_route_sid);
+    res.status(200).json(route);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcrs.js

@@ -0,0 +1,138 @@
+const router = require('express').Router();
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoutes = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+const ServiceProvider = require('../../models/service-provider');
+
+const validateAssociatedTarget = async(req, sid) => {
+  const {lookupAccountBySid} = req.app.locals;
+  if (req.body.account_sid) {
+    // Add only for account
+    req.body.service_provider_sid = null;
+    const account = await lookupAccountBySid(req.body.account_sid);
+    if (!account) throw new DbErrorBadRequest('unknown account_sid');
+    const lcr = await Lcr.retrieveAllByAccountSid(req.body.account_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Account: ${account.name}  already has an active call routing table.`);
+    }
+  } else if (req.body.service_provider_sid) {
+    const serviceProviders = await ServiceProvider.retrieve(req.body.service_provider_sid);
+    if (serviceProviders.length === 0) throw new DbErrorBadRequest('unknown service_provider_sid');
+    const serviceProvider = serviceProviders[0];
+    const lcr = await Lcr.retrieveAllByServiceProviderSid(req.body.service_provider_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Service Provider: ${serviceProvider.name} already 
+      has an active call routing table.`);
+    }
+  }
+};
+
+const validateAdd = async(req) => {
+  if (req.user.hasAccountAuth) {
+    // Account just create LCR for himself
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth) {
+    // SP just can create LCR for himself
+    req.body.service_provider_sid = req.user.service_provider_sid;
+    req.body.account_sid = null;
+  }
+
+  await validateAssociatedTarget(req);
+  // check if lcr_carrier_set_entry is available
+  if (req.body.lcr_carrier_set_entry) {
+    const e = await LcrCarrierSetEntry.retrieve(req.body.lcr_carrier_set_entry);
+    if (e.length === 0) throw new DbErrorBadRequest('unknown lcr_carrier_set_entry');
+  }
+
+};
+
+const validateUserPermissionForExistingEntity = async(req, sid) => {
+  const r = await Lcr.retrieve(sid);
+  if (r.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+  const lcr = r[0];
+  if (req.user.hasAccountAuth) {
+    if (lcr.account_sid != req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  } else if (req.user.hasServiceProviderAuth) {
+    if (lcr.service_provider_sid != req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await validateUserPermissionForExistingEntity(req, sid);
+  await validateAssociatedTarget(req, sid);
+};
+
+const validateDelete = async(req, sid) => {
+  if (req.user.hasAccountAuth) {
+    /* can only delete Lcr for the user's account */
+    const r = await Lcr.retrieve(sid);
+    const lcr = r.length > 0 ? r[0] : null;
+    if (!lcr || (req.user.account_sid && lcr.account_sid != req.user.account_sid)) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+  await Lcr.releaseDefaultEntry(sid);
+  // fetch lcr route
+  const lcr_routes = await LcrRoutes.retrieveAllByLcrSid(sid);
+  // delete all lcr carrier set entries
+  for (const e of lcr_routes) {
+    await LcrCarrierSetEntry.deleteByLcrRouteSid(e.lcr_route_sid);
+  }
+
+  // delete all lcr routes
+  await LcrRoutes.deleteByLcrSid(sid);
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete
+};
+
+decorate(router, Lcr, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Lcr.retrieveAll() : req.user.hasAccountAuth ?
+        await Lcr.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Lcr.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+
+    for (const lcr of results) {
+      lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Lcr.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const lcr = results[0];
+    if (req.user.hasAccountAuth && lcr.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth && lcr.service_provider_sid !== req.user.service_provider_sid) {
+      return res.sendStatus(404);
+    }
+    lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    return res.status(200).json(lcr);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/limits.js

@@ -0,0 +1,133 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const AccountLimits = require('../../models/account-limits');
+const ServiceProviderLimits = require('../../models/service-provider-limits');
+const {parseAccountSid, parseServiceProviderSid} = require('./utils');
+const {promisePool} = require('../../db');
+const sqlDeleteSPLimits = `
+DELETE FROM service_provider_limits 
+WHERE service_provider_sid = ? 
+`;
+const sqlDeleteSPLimitsByCategory = `
+DELETE FROM service_provider_limits 
+WHERE service_provider_sid = ? 
+AND category = ?
+`;
+const sqlDeleteAccountLimits = `
+DELETE FROM account_limits 
+WHERE account_sid = ? 
+`;
+const sqlDeleteAccountLimitsByCategory = `
+DELETE FROM account_limits 
+WHERE account_sid = ? 
+AND category = ?
+`;
+
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {
+    category,
+    quantity
+  } = req.body;
+
+  try {
+    let service_provider_sid;
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) {
+      if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
+        logger.error('POST /SpeechCredentials invalid credentials');
+        return res.sendStatus(403);
+      }
+      service_provider_sid = parseServiceProviderSid(req);
+    }
+
+    let uuid;
+    if (account_sid) {
+      const existing = (await AccountLimits.retrieve(account_sid) || [])
+        .find((el) => el.category === category);
+      if (existing) {
+        uuid = existing.account_limits_sid;
+        await AccountLimits.update(uuid, {category, quantity});
+      }
+      else {
+        uuid = await AccountLimits.make({
+          account_sid,
+          category,
+          quantity
+        });
+      }
+    }
+    else {
+      const existing = (await ServiceProviderLimits.retrieve(service_provider_sid) || [])
+        .find((el) => el.category === category);
+      if (existing) {
+        uuid = existing.service_provider_limits_sid;
+        await ServiceProviderLimits.update(uuid, {category, quantity});
+      }
+      else {
+        uuid = await ServiceProviderLimits.make({
+          service_provider_sid,
+          category,
+          quantity
+        });
+      }
+    }
+    res.status(201).json({sid: uuid});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/**
+ * retrieve all limits for an account or service provider
+ */
+router.get('/', async(req, res) => {
+  let service_provider_sid;
+  const logger = req.app.locals.logger;
+
+  try {
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) service_provider_sid = parseServiceProviderSid(req);
+    const limits = account_sid ?
+      await AccountLimits.retrieve(account_sid) :
+      await ServiceProviderLimits.retrieve(service_provider_sid);
+
+    if (req.query?.category) {
+      return res.status(200).json(limits.filter((el) => el.category === req.query.category));
+    }
+    res.status(200).json(limits);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.delete('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const account_sid = parseAccountSid(req);
+    const {category} = req.query;
+    const service_provider_sid = parseServiceProviderSid(req);
+    if (account_sid) {
+      if (category) {
+        await promisePool.execute(sqlDeleteAccountLimitsByCategory, [account_sid, category]);
+      }
+      else {
+        await promisePool.execute(sqlDeleteAccountLimits, [account_sid]);
+      }
+    }
+    else {
+      if (category) {
+        await promisePool.execute(sqlDeleteSPLimitsByCategory, [service_provider_sid, category]);
+      }
+      else {
+        await promisePool.execute(sqlDeleteSPLimits, [service_provider_sid]);
+      }
+    }
+    res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/login.js

@@ -1,61 +1,116 @@
 const router = require('express').Router();
-const {getMysqlConnection} = require('../../db');
+const jwt = require('jsonwebtoken');
 const {verifyPassword} = require('../../utils/password-utils');
-
+const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
+const Account = require('../../models/account');
+const ServiceProvider = require('../../models/service-provider');
+const sysError = require('../error');
+const retrievePemissionsSql = `
+SELECT p.name 
+FROM permissions p, user_permissions up 
+WHERE up.permission_sid = p.permission_sid 
+AND up.user_sid = ? 
+`;
 const retrieveSql = 'SELECT * from users where name = ?';
 const tokenSql = 'SELECT token from api_keys where account_sid IS NULL AND service_provider_sid IS NULL';
 
 
-router.post('/', (req, res) => {
-  const logger = req.app.locals.logger;
+router.post('/', async(req, res) => {
+  const {logger, incrKey, retrieveKey} = req.app.locals;
   const {username, password} = req.body;
   if (!username || !password) {
     logger.info('Bad POST to /login is missing username or password');
     return res.sendStatus(400);
   }
 
-  getMysqlConnection((err, conn) => {
-    if (err) {
-      logger.error({err}, 'Error getting db connection');
+  try {
+    const [r] = await promisePool.query(retrieveSql, username);
+    if (r.length === 0) {
+      logger.info(`Failed login attempt for user ${username}`);
+      return res.sendStatus(403);
+    }
+    logger.info({r}, 'successfully retrieved user account');
+
+    const maxLoginAttempts = process.env.LOGIN_ATTEMPTS_MAX_RETRIES || 6;
+    const loginAttempsBlocked = await retrieveKey(`login:${r[0].user_sid}`) >= maxLoginAttempts;
+
+    if (loginAttempsBlocked) {
+      logger.info(`User ${r[0].user_sid} was blocked due to excessive login attempts with incorrect credentials.`);
+      return res.status(403)
+        .json({error: 'Maximum login attempts reached. Please try again later or reset your password.'});
+    }
+
+    const isCorrect = await verifyPassword(r[0].hashed_password, password);
+    if (!isCorrect) {
+      const attempTime = process.env.LOGIN_ATTEMPTS_TIME || 1800;
+      const newAttempt = await incrKey(`login:${r[0].user_sid}`, attempTime)
+        .catch((err) => logger.error({err}, 'Error adding logging attempt to redis'));
+      if (newAttempt >= maxLoginAttempts) {
+        logger.info(`User ${r[0].user_sid} is now blocked due to excessive login attempts with incorrect credentials.`);
+        return res.status(403)
+          .json({error: `Maximum login attempts reached. Please try again in ${attempTime} seconds.`});
+      }
+      return res.sendStatus(403);
+    }
+    const force_change = !!r[0].force_change;
+    const [t] = await promisePool.query(tokenSql);
+    if (t.length === 0) {
+      logger.error('Database has no admin token provisioned...run reset_admin_password');
       return res.sendStatus(500);
     }
-    conn.query(retrieveSql, [username], async(err, results) => {
-      conn.release();
-      if (err) {
-        logger.error({err}, 'Error getting db connection');
-        return res.sendStatus(500);
-      }
-      if (0 === results.length) {
-        logger.info(`Failed login attempt for user ${username}`);
-        return res.sendStatus(403);
-      }
 
-      logger.info({results}, 'successfully retrieved account');
-      const isCorrect = await verifyPassword(results[0].hashed_password, password);
-      if (!isCorrect) return res.sendStatus(403);
+    const [p] = await promisePool.query(retrievePemissionsSql, r[0].user_sid);
+    const permissions = p.map((x) => x.name);
+    const obj = {user_sid: r[0].user_sid, scope: 'admin', force_change, permissions};
+    if (r[0].service_provider_sid && r[0].account_sid) {
+      const account = await Account.retrieve(r[0].account_sid);
+      const service_provider = await ServiceProvider.retrieve(r[0].service_provider_sid);
+      obj.scope = 'account';
+      obj.service_provider_sid = r[0].service_provider_sid;
+      obj.account_sid = r[0].account_sid;
+      obj.account_name = account[0].name;
+      obj.service_provider_name = service_provider[0].name;
+    }
+    else if (r[0].service_provider_sid) {
+      const service_provider = await ServiceProvider.retrieve(r[0].service_provider_sid);
+      obj.scope = 'service_provider';
+      obj.service_provider_sid = r[0].service_provider_sid;
+      obj.service_provider_name = service_provider[0].name;
+    }
+    const payload = {
+      scope: obj.scope,
+      permissions,
+      ...(obj.service_provider_sid && {
+        service_provider_sid: obj.service_provider_sid,
+        service_provider_name: obj.service_provider_name
+      }),
+      ...(obj.account_sid && {
+        account_sid: obj.account_sid,
+        account_name: obj.account_name,
+        service_provider_name: obj.service_provider_name
+      }),
+      user_sid: obj.user_sid
+    };
 
-      const force_change = !!results[0].force_change;
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60;
+    const token = jwt.sign(
+      payload,
+      process.env.JWT_SECRET,
+      { expiresIn }
+    );
+    res.json({token, ...obj});
 
-      getMysqlConnection((err, conn) => {
-        if (err) {
-          logger.error({err}, 'Error getting db connection');
-          return res.sendStatus(500);
-        }
-        conn.query(tokenSql, (err, tokenResults) => {
-          conn.release();
-          if (err) {
-            logger.error({err}, 'Error getting db connection');
-            return res.sendStatus(500);
-          }
-          if (0 === tokenResults.length) {
-            logger.error('Database has no admin token provisioned...run reset_admin_password');
-            return res.sendStatus(500);
-          }
-          res.json({user_sid: results[0].user_sid, force_change, token: tokenResults[0].token});
-        });
-      });
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', obj.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
     });
-  });
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
 });
 
 

lib/routes/api/logout.js

@@ -1,19 +1,18 @@
 const router = require('express').Router();
 const debug = require('debug')('jambonz:api-server');
-const {hashString} = require('../../utils/password-utils');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 
 router.post('/', async(req, res) => {
-  const {logger, addKey} = req.app.locals;
-  const {jwt} = req.user;
+  const {logger} = req.app.locals;
+  const {user_sid} = req.user;
 
-  debug(`adding jwt to blacklist: ${jwt}`);
+  debug(`logout user and invalidate jwt token for user: ${user_sid}`);
 
   try {
-    /* add key to blacklist */
-    const s = `jwt:${hashString(jwt)}`;
-    const result = await addKey(s, '1', 3600);
-    debug(`result from adding ${s}: ${result}`);
+    const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+    await cacheClient.delete(redisKey);
+
     res.sendStatus(204);
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/password-settings.js

@@ -0,0 +1,45 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const PasswordSettings = require('../../models/password-settings');
+const { DbErrorBadRequest } = require('../../utils/errors');
+
+const validate = (obj) => {
+  if (obj.min_password_length && (
+    obj.min_password_length < 8 ||
+    obj.min_password_length > 20
+  )) {
+    throw new DbErrorBadRequest('invalid min_password_length property: should be between 8-20');
+  }
+};
+
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    if (!req.user.hasAdminAuth) {
+      return res.sendStatus(403);
+    }
+    validate(req.body);
+    const [existing] = (await PasswordSettings.retrieve() || []);
+    if (existing) {
+      await PasswordSettings.update(req.body);
+    } else {
+      await PasswordSettings.make(req.body);
+    }
+    res.status(201).json({});
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const [results] = (await PasswordSettings.retrieve() || []);
+    return res.status(200).json(results || {min_password_length: 8});
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+module.exports = router;

lib/routes/api/phone-numbers.js

@@ -1,8 +1,10 @@
 const router = require('express').Router();
-const {DbErrorUnprocessableRequest, DbErrorBadRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
 const PhoneNumber = require('../../models/phone-number');
 const VoipCarrier = require('../../models/voip-carrier');
+const Account = require('../../models/account');
 const decorate = require('./decorate');
+const {promisePool} = require('../../db');
 const {e164} = require('../../utils/phone-number-utils');
 const preconditions = {
   'add': validateAdd,
@@ -10,6 +12,7 @@ const preconditions = {
   'update': validateUpdate
 };
 const sysError = require('../error');
+const { parsePhoneNumberSid } = require('./utils');
 
 
 /* check for required fields when adding */
@@ -20,6 +23,10 @@ async function validateAdd(req) {
       req.body.account_sid = req.user.account_sid;
     }
 
+    if (req.user.hasServiceProviderAuth) {
+      req.body.service_provider_sid = req.user.service_provider_sid;
+    }
+
     if (!req.body.number) throw new DbErrorBadRequest('number is required');
     const formattedNumber = e164(req.body.number);
     req.body.number = formattedNumber;
@@ -41,11 +48,11 @@ async function checkInUse(req, sid) {
   const phoneNumber = await PhoneNumber.retrieve(sid);
   if (req.user.hasAccountAuth) {
     if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete a phone number that belongs to another account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
   if (!req.user.hasAccountAuth && phoneNumber.account_sid) {
-    throw new DbErrorUnprocessableRequest('cannot delete phone number that is assigned to an account');
+    throw new DbErrorForbidden('insufficient privileges');
   }
 }
 
@@ -57,10 +64,23 @@ async function validateUpdate(req, sid) {
   const phoneNumber = await PhoneNumber.retrieve(sid);
   if (req.user.hasAccountAuth) {
     if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot operate on a phone number that belongs to another account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
+  if (req.user.hasServiceProviderAuth) {
+    let service_provider_sid;
 
+    if (!phoneNumber[0].service_provider_sid) {
+      const [r] = await Account.retrieve(phoneNumber[0].account_sid);
+      service_provider_sid = r.service_provider_sid;
+    } else {
+      service_provider_sid = phoneNumber[0].service_provider_sid;
+    }
+
+    if (phoneNumber && phoneNumber.length && service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
   // TODO: if we are assigning to an account, verify it exists
 
   // TODO: if we are assigning to an application, verify it is associated to the same account
@@ -74,7 +94,9 @@ decorate(router, PhoneNumber, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
+    const results = req.user.hasAdminAuth ?
+      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null) :
+      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid);
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -85,9 +107,19 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parsePhoneNumberSid(req);
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await PhoneNumber.retrieve(req.params.sid, account_sid);
+    const results = await PhoneNumber.retrieve(sid, account_sid);
     if (results.length === 0) return res.status(404).end();
+
+    if (req.user.hasServiceProviderAuth && results.length === 1) {
+      const account_sid = results[0].account_sid;
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]);
+      if (r.length === 1 && r[0].service_provider_sid !== req.user.service_provider_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/recent-calls.js

@@ -2,33 +2,60 @@ const router = require('express').Router();
 const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
+const {getJaegerTrace} = require('../../utils/jaeger-utils');
+const Account = require('../../models/account');
+const { getS3Object, getGoogleStorageObject } = require('../../utils/storage-utils');
+
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
   if (arr) return arr[1];
 };
 
+const parseServiceProviderSid = (url) => {
+  const arr = /ServiceProviders\/([^\/]*)/.exec(url);
+  if (arr) return arr[1];
+};
+
 router.get('/', async(req, res) => {
-  const {logger, queryCdrs} = req.app.locals;
+  const {logger, queryCdrs, queryCdrsSP} = req.app.locals;
   try {
     logger.debug({opts: req.query}, 'GET /RecentCalls');
     const account_sid = parseAccountSid(req.originalUrl);
-    const {page, count, trunk, direction, days, answered, start, end} = req.query || {};
+    const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
+    const {page, count, trunk, direction, days, answered, start, end, filter} = req.query || {};
     if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
     if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
 
-    const data = await queryCdrs({
-      account_sid,
-      page,
-      page_size: count,
-      trunk,
-      direction,
-      days,
-      answered,
-      start: days ? undefined : start,
-      end: days ? undefined : end,
-    });
-
-    res.status(200).json(data);
+    if (account_sid) {
+      const data = await queryCdrs({
+        account_sid,
+        page,
+        page_size: count,
+        trunk,
+        direction,
+        days,
+        answered,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+        filter
+      });
+      res.status(200).json(data);
+    }
+    else {
+      const data = await queryCdrsSP({
+        service_provider_sid,
+        page,
+        page_size: count,
+        trunk,
+        direction,
+        days,
+        answered,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+        filter
+      });
+      res.status(200).json(data);
+    }
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -51,12 +78,12 @@ router.get('/:call_id', async(req, res) => {
   }
 });
 
-router.get('/:call_id/pcap', async(req, res) => {
+router.get('/:call_id/:method/pcap', async(req, res) => {
   const {logger} = req.app.locals;
   try {
     const token = await getHomerApiKey(logger);
     if (!token) return res.sendStatus(400, {msg: 'getHomerApiKey: Failed to get Homer API token; check server config'});
-    const stream = await getHomerPcap(logger, token, [req.params.call_id]);
+    const stream = await getHomerPcap(logger, token, [req.params.call_id], req.params.method);
     if (!stream) {
       logger.info(`getHomerApiKey: unable to get sip traces from Homer for ${req.params.call_id}`);
       return res.sendStatus(404);
@@ -72,4 +99,54 @@ router.get('/:call_id/pcap', async(req, res) => {
   }
 });
 
+router.get('/trace/:trace_id', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {trace_id} = req.params;
+  try {
+    const obj = await getJaegerTrace(logger, trace_id);
+    if (!obj) {
+      logger.info(`/RecentCalls: unable to get spans from jaeger for ${trace_id}`);
+      return res.sendStatus(404);
+    }
+    res.status(200).json(obj.result);
+  } catch (err) {
+    logger.error({err}, `/RecentCalls error retrieving jaeger trace ${trace_id}`);
+    res.sendStatus(500);
+  }
+});
+
+router.get('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+    const getOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+    let stream;
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+        stream = await getS3Object(logger, getOptions);
+        break;
+      case 'google':
+        stream = await getGoogleStorageObject(logger, getOptions);
+        break;
+      default:
+        logger.error(`There is no handler for fetching record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.set({
+      'Content-Type': `audio/${format || 'mp3'}`
+    });
+    stream.pipe(res);
+  }  catch (err) {
+    logger.error({err}, ` error retrieving recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
 module.exports = router;

lib/routes/api/register.js

@@ -4,6 +4,7 @@ const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/er
 const {promisePool} = require('../../db');
 const {doGithubAuth, doGoogleAuth, doLocalAuth} = require('../../utils/oauth-utils');
 const {validateEmail} = require('../../utils/email-utils');
+const {cacheClient} = require('../../helpers');
 const { v4: uuid } = require('uuid');
 const short = require('short-uuid');
 const translator = short();
@@ -338,16 +339,19 @@ router.post('/', async(req, res) => {
         /* deactivate the old/replaced user */
         const [r] = await promisePool.execute('DELETE FROM users WHERE user_sid = ?', [user_sid]);
         logger.debug({r}, 'register - removed old user');
+        const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+        await cacheClient.delete(redisKey);
       }
     }
 
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60 ;
     // generate a json web token for this user
     const token = jwt.sign({
       user_sid: userProfile.user_sid,
       account_sid: userProfile.account_sid,
       email: userProfile.email,
       name: userProfile.name
-    }, process.env.JWT_SECRET, { expiresIn: '1h' });
+    }, process.env.JWT_SECRET, { expiresIn });
 
     logger.debug({
       user_sid: userProfile.user_sid,
@@ -356,6 +360,13 @@ router.post('/', async(req, res) => {
 
     res.json({jwt: token, ...userProfile});
 
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', userProfile.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
   } catch (err) {
     debug(err, 'Error');
     sysError(logger, res, err);

lib/routes/api/sbcs.js

@@ -2,25 +2,46 @@ const router = require('express').Router();
 const Sbc = require('../../models/sbc');
 const decorate = require('./decorate');
 const sysError = require('../error');
-//const {DbErrorBadRequest} = require('../../utils/errors');
-//const {promisePool} = require('../../db');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
 
-decorate(router, Sbc, ['add', 'delete']);
+const validate = (req, res) => {
+  if (req.user.hasScope('admin')) return;
+  res.status(403).json({
+    status: 'fail',
+    message: 'insufficient privileges'
+  });
+};
+
+const preconditions = {
+  'add': validate,
+  'delete': validate
+};
+
+decorate(router, Sbc, ['add', 'delete'], preconditions);
 
 /* list */
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const service_provider_sid = req.query.service_provider_sid;
-    /*
+    let service_provider_sid = req.query.service_provider_sid;
+
     if (req.user.hasAccountAuth) {
       const [r] = await promisePool.query('SELECT * from accounts WHERE account_sid = ?', req.user.account_sid);
-      if (0 === r.length) throw new Error('invalid account_sid');
+      if (0 === r.length) throw new DbErrorBadRequest('invalid account_sid');
+
       service_provider_sid = r[0].service_provider_sid;
     }
-    if (!service_provider_sid) throw new DbErrorBadRequest('missing service_provider_sid in query');
-    */
-    const results = await Sbc.retrieveAll(service_provider_sid);
+
+    if (req.user.hasServiceProviderAuth) {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
+    /** generally, we have a global set of SBCs that all accounts use.
+     * However, we can have a set of SBCs that are specific for use by a service provider.
+     */
+    let results = await Sbc.retrieveAll(service_provider_sid);
+    if (results.length === 0) results = await Sbc.retrieveAll();
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/service-providers.js

@@ -1,6 +1,6 @@
 const router = require('express').Router();
 const {promisePool} = require('../../db');
-const {DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorForbidden} = require('../../utils/errors');
 const Webhook = require('../../models/webhook');
 const ServiceProvider = require('../../models/service-provider');
 const Account = require('../../models/account');
@@ -8,11 +8,15 @@ const VoipCarrier = require('../../models/voip-carrier');
 const Application = require('../../models/application');
 const PhoneNumber = require('../../models/phone-number');
 const ApiKey = require('../../models/api-key');
-const {hasServiceProviderPermissions, parseServiceProviderSid} = require('./utils');
+const {
+  hasServiceProviderPermissions,
+  parseServiceProviderSid,
+  parseVoipCarrierSid,
+} = require('./utils');
 const sysError = require('../error');
 const decorate = require('./decorate');
 const preconditions = {
-  'delete': noActiveAccounts
+  'delete': noActiveAccountsOrUsers
 };
 const sqlDeleteSipGateways = `DELETE from sip_gateways 
 WHERE voip_carrier_sid IN (
@@ -27,37 +31,100 @@ WHERE voip_carrier_sid IN (
   WHERE service_provider_sid = ?
 )`;
 
-/* can not delete a service provider if it has any active accounts */
-async function noActiveAccounts(req, sid) {
+/* only admin users can add a service provider */
+function validateAdd(req) {
+  if (!req.user.hasAdminAuth) {
+    throw new DbErrorForbidden('only admin users can add a service provider');
+  }
+}
+
+async function validateRetrieve(req) {
+  try {
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider') || req.user.hasScope('account')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  } catch (error) {
+    throw error;
+  }
+}
+
+function validateUpdate(req) {
+  try {
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions to update service provider');
+  } catch (error) {
+    throw error;
+  }
+}
+
+/* can not delete a service provider if it has any active accounts or users*/
+async function noActiveAccountsOrUsers(req, sid) {
+  if (!req.user.hasAdminAuth) {
+    throw new DbErrorForbidden('only admin users can delete a service provider');
+  }
   const activeAccounts = await ServiceProvider.getForeignKeyReferences('accounts.service_provider_sid', sid);
-  if (activeAccounts > 0) throw new DbErrorUnprocessableRequest('cannot delete service provider with active accounts');
+  const activeUsers = await ServiceProvider.getForeignKeyReferences('users.service_provider_sid', sid);
+  if (activeAccounts > 0 && activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
+
+  if (activeAccounts > 0) throw new DbErrorForbidden('insufficient privileges');
+  if (activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
 
   /* ok we can delete -- no active accounts.  remove carriers and speech credentials */
   await promisePool.execute('DELETE from speech_credentials WHERE service_provider_sid = ?', [sid]);
   await promisePool.query(sqlDeleteSipGateways, [sid]);
   await promisePool.query(sqlDeleteSmppGateways, [sid]);
   await promisePool.query('DELETE from voip_carriers WHERE service_provider_sid = ?', [sid]);
+  await promisePool.query('DELETE from api_keys WHERE service_provider_sid = ?', [sid]);
 }
 
 decorate(router, ServiceProvider, ['delete'], preconditions);
 
-router.use('/:sid/SpeechCredentials', hasServiceProviderPermissions, require('./speech-credentials'));
+router.use('/:sid/RecentCalls', hasServiceProviderPermissions, require('./recent-calls'));
+router.use('/:sid/Alerts', hasServiceProviderPermissions, require('./alerts'));
+router.use('/:sid/SpeechCredentials', require('./speech-credentials'));
+router.use('/:sid/Limits', hasServiceProviderPermissions, require('./limits'));
 router.use('/:sid/PredefinedCarriers', hasServiceProviderPermissions, require('./add-from-predefined-carrier'));
 router.get('/:sid/Accounts', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await Account.retrieveAll(service_provider_sid);
+    let results = await Account.retrieveAll(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
   }
 });
+
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await Application.retrieveAll(service_provider_sid);
+    let results = await Application.retrieveAll(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -66,8 +133,12 @@ router.get('/:sid/Applications', async(req, res) => {
 router.get('/:sid/PhoneNumbers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await PhoneNumber.retrieveAllForSP(service_provider_sid);
+    let results = await PhoneNumber.retrieveAllForSP(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -76,9 +147,15 @@ router.get('/:sid/PhoneNumbers', async(req, res) => {
 router.get('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await VoipCarrier.retrieveAllForSP(service_provider_sid);
-    res.status(200).json(results);
+    const carriers = await VoipCarrier.retrieveAllForSP(service_provider_sid);
+
+    if (req.user.hasScope('account')) {
+      return res.status(200).json(carriers.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid));
+    }
+
+    res.status(200).json(carriers);
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -86,6 +163,7 @@ router.get('/:sid/VoipCarriers', async(req, res) => {
 router.post('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    validateUpdate(req);
     const service_provider_sid = parseServiceProviderSid(req);
     const uuid = await VoipCarrier.make({...req.body, service_provider_sid});
     res.status(201).json({sid: uuid});
@@ -96,7 +174,9 @@ router.post('/:sid/VoipCarriers', async(req, res) => {
 router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const rowsAffected = await VoipCarrier.update(req.params.voip_carrier_sid, req.body);
+    validateUpdate(req);
+    const sid = parseVoipCarrierSid(req);
+    const rowsAffected = await VoipCarrier.update(sid, req.body);
     if (rowsAffected === 0) {
       return res.sendStatus(404);
     }
@@ -105,21 +185,15 @@ router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
     sysError(logger, res, err);
   }
 });
-router.get(':sid/Acccounts', async(req, res) => {
-  const logger = req.app.locals.logger;
-  try {
-    const service_provider_sid = parseServiceProviderSid(req);
-    const results = await Account.retrieveAll(service_provider_sid);
-    res.status(200).json(results);
-  } catch (err) {
-    sysError(logger, res, err);
-  }
-});
 router.get('/:sid/ApiKeys', async(req, res) => {
   const logger = req.app.locals.logger;
   const {sid} = req.params;
   try {
-    const results = await ApiKey.retrieveAllForSP(sid);
+    await validateRetrieve(req);
+    let results = await ApiKey.retrieveAllForSP(sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
     await ApiKey.updateLastUsed(sid);
   } catch (err) {
@@ -127,12 +201,11 @@ router.get('/:sid/ApiKeys', async(req, res) => {
   }
 });
 
-
 /* add */
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-
+    validateAdd(req);
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
     for (const prop of ['registration_hook']) {
@@ -155,6 +228,12 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const results = await ServiceProvider.retrieveAll();
+    logger.debug({results, user: req.user}, 'ServiceProvider.retrieveAll');
+    if (req.user.hasScope('service_provider') || req.user.hasScope('account')) {
+      logger.debug(`Filtering results for ${req.user.service_provider_sid}`);
+      return res.status(200).json(results.filter((e) => req.user.service_provider_sid === e.service_provider_sid));
+    }
+
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -165,7 +244,9 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await ServiceProvider.retrieve(req.params.sid);
+    await validateRetrieve(req);
+    const sid = parseServiceProviderSid(req);
+    const results = await ServiceProvider.retrieve(sid);
     if (results.length === 0) return res.status(404).end();
     return res.status(200).json(results[0]);
   }
@@ -176,9 +257,11 @@ router.get('/:sid', async(req, res) => {
 
 /* update */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    validateUpdate(req);
+    const sid = parseServiceProviderSid(req);
+
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
     for (const prop of ['registration_hook']) {
@@ -187,15 +270,14 @@ router.put('/:sid', async(req, res) => {
           const sid = obj[prop]['webhook_sid'];
           delete obj[prop]['webhook_sid'];
           await Webhook.update(sid, obj[prop]);
-        }
-        else {
+        } else {
           const sid = await Webhook.make(obj[prop]);
           obj[`${prop}_sid`] = sid;
         }
-      }
-      else {
+      } else {
         obj[`${prop}_sid`] = null;
       }
+
       delete obj[prop];
     }
 
@@ -203,6 +285,7 @@ router.put('/:sid', async(req, res) => {
     if (rowsAffected === 0) {
       return res.status(404).end();
     }
+
     res.status(204).end();
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/signin.js

@@ -3,10 +3,17 @@ const router = require('express').Router();
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {promisePool} = require('../../db');
 const {verifyPassword} = require('../../utils/password-utils');
+const {cacheClient} = require('../../helpers');
 const jwt = require('jsonwebtoken');
 const sysError = require('../error');
+const retrievePermissionsSql = `
+SELECT p.name 
+FROM permissions p, user_permissions up 
+WHERE up.permission_sid = p.permission_sid 
+AND up.user_sid = ? 
+`;
 
-const validateRequest = async(req) => {
+const validateRequest = (req) => {
   const {email, password} = req.body || {};
 
   /* check required properties are there */
@@ -52,6 +59,7 @@ router.post('/', async(req, res) => {
       email: user.email,
       phone: user.phone,
       account_sid: user.account_sid,
+      service_provider_sid: a[0].service_provider_sid,
       force_change: !!user.force_change,
       provider: user.provider,
       provider_userid: user.provider_userid,
@@ -64,11 +72,22 @@ router.post('/', async(req, res) => {
       pristine: false
     });
 
+    const [p] = await promisePool.query(retrievePermissionsSql, user.user_sid);
+    const permissions = p.map((x) => x.name);
+
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60;
     // generate a json web token for this session
-    const token = jwt.sign({
+    const payload = {
+      scope: 'account',
+      permissions,
       user_sid: userProfile.user_sid,
-      account_sid: userProfile.account_sid
-    }, process.env.JWT_SECRET, { expiresIn: '1h' });
+      account_sid: userProfile.account_sid,
+      service_provider_sid: userProfile.service_provider_sid
+    };
+    const token = jwt.sign(payload,
+      process.env.JWT_SECRET,
+      { expiresIn }
+    );
 
     logger.debug({
       user_sid: userProfile.user_sid,
@@ -76,6 +95,14 @@ router.post('/', async(req, res) => {
     }, 'generated jwt');
 
     res.json({jwt: token, ...userProfile});
+
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', userProfile.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/sip-gateways.js

@@ -1,11 +1,46 @@
 const router = require('express').Router();
 const SipGateway = require('../../models/sip-gateway');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
+//const {parseSipGatewaySid} = require('./utils');
 const decorate = require('./decorate');
 const sysError = require('../error');
 
+const checkUserScope = async(req, voip_carrier_sid) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+  if (req.user.hasAccountAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+
+    if ((!carrier.service_provider_sid || carrier.service_provider_sid === req.user.service_provider_sid) &&
+      (!carrier.account_sid || carrier.account_sid === req.user.account_sid)) {
+
+      if (req.method !== 'GET' && !carrier.account_sid) {
+        throw new DbErrorForbidden('insufficient privileges');
+      }
+
+      return;
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) {
+      throw new DbErrorBadRequest('invalid voip_carrier_sid');
+    }
+
+    if (carrier.service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+  }
+  throw new DbErrorForbidden('insufficient privileges');
+};
+
 const validate = async(req, sid) => {
-  const {lookupCarrierBySid, lookupSipGatewayBySid} = req.app.locals;
+  const {lookupSipGatewayBySid} = req.app.locals;
   let voip_carrier_sid;
 
   if (sid) {
@@ -17,13 +52,7 @@ const validate = async(req, sid) => {
     voip_carrier_sid = req.body.voip_carrier_sid;
     if (!voip_carrier_sid) throw new DbErrorBadRequest('missing voip_carrier_sid');
   }
-  if (req.hasAccountAuth) {
-    const carrier = await lookupCarrierBySid(voip_carrier_sid);
-    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
-    if (carrier.account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('user can not add gateway for voip_carrier belonging to other account');
-    }
-  }
+  await checkUserScope(req, voip_carrier_sid);
 };
 
 const preconditions = {
@@ -39,6 +68,7 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   const voip_carrier_sid = req.query.voip_carrier_sid;
   try {
+    await checkUserScope(req, voip_carrier_sid);
     if (!voip_carrier_sid) {
       logger.info('GET /SipGateways missing voip_carrier_sid param');
       return res.status(400).json({message: 'missing voip_carrier_sid query param'});

lib/routes/api/smpp-gateways.js

@@ -1,11 +1,38 @@
 const router = require('express').Router();
 const SmppGateway = require('../../models/smpp-gateway');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
 const decorate = require('./decorate');
 const sysError = require('../error');
 
+const checkUserScope = async(req, voip_carrier_sid) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+
+  if (req.user.hasAccountAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+
+    if ((!carrier.service_provider_sid || carrier.service_provider_sid === req.user.service_provider_sid) &&
+      (!carrier.account_sid || carrier.account_sid === req.user.account_sid)) {
+      return;
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+    if (carrier.service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+  }
+  throw new DbErrorForbidden('insufficient privileges');
+};
+
 const validate = async(req, sid) => {
-  const {lookupCarrierBySid, lookupSmppGatewayBySid} = req.app.locals;
+  const {lookupSmppGatewayBySid} = req.app.locals;
   let voip_carrier_sid;
 
   if (sid) {
@@ -17,13 +44,8 @@ const validate = async(req, sid) => {
     voip_carrier_sid = req.body.voip_carrier_sid;
     if (!voip_carrier_sid) throw new DbErrorBadRequest('missing voip_carrier_sid');
   }
-  if (req.hasAccountAuth) {
-    const carrier = await lookupCarrierBySid(voip_carrier_sid);
-    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
-    if (carrier.account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('user can not add gateway for voip_carrier belonging to other account');
-    }
-  }
+
+  await checkUserScope(req, voip_carrier_sid);
 };
 
 const preconditions = {
@@ -39,6 +61,7 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   const voip_carrier_sid = req.query.voip_carrier_sid;
   try {
+    await checkUserScope(req, voip_carrier_sid);
     if (!voip_carrier_sid) {
       logger.info('GET /SmppGateways missing voip_carrier_sid param');
       return res.status(400).json({message: 'missing voip_carrier_sid query param'});

lib/routes/api/sms-inbound.js

@@ -14,20 +14,14 @@ const getFsUrl = async(logger, retrieveSet, setName, provider) => {
       logger.info('No available feature servers to handle createCall API request');
       return ;
     }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    return `http://${ip}:3000/v1/messaging/${provider}`;
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/messaging/${provider}`;
   } catch (err) {
     logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
   }
 };
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
-};
-
 const doSendResponse = async(res, respondFn, body) => {
   if (typeof respondFn === 'number') res.sendStatus(respondFn);
   else if (typeof respondFn !== 'function') res.sendStatus(200);
@@ -44,7 +38,7 @@ router.post('/:provider', async(req, res) => {
     lookupAppByPhoneNumber,
     logger
   } = req.app.locals;
-  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:active-fs`;
+  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:fs-service-url`;
   logger.debug({path: req.path, body: req.body}, 'incomingSMS from carrier');
 
   // search for provider module

lib/routes/api/speech-credentials.js

@@ -1,10 +1,11 @@
 const router = require('express').Router();
 const assert = require('assert');
+const Account = require('../../models/account');
 const SpeechCredential = require('../../models/speech-credential');
 const sysError = require('../error');
-const {decrypt, encrypt} = require('../../utils/encrypt-decrypt');
-const {parseAccountSid, parseServiceProviderSid} = require('./utils');
-const {DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {decrypt, encrypt, obscureKey} = require('../../utils/encrypt-decrypt');
+const {parseAccountSid, parseServiceProviderSid, parseSpeechCredentialSid} = require('./utils');
+const {DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const {
   testGoogleTts,
   testGoogleStt,
@@ -12,9 +13,91 @@ const {
   testAwsStt,
   testMicrosoftStt,
   testMicrosoftTts,
-  testWellSaidTts
+  testWellSaidTts,
+  testNuanceStt,
+  testNuanceTts,
+  testDeepgramStt,
+  testSonioxStt,
+  testIbmTts,
+  testIbmStt
 } = require('../../utils/speech-utils');
+const {promisePool} = require('../../db');
 
+const validateAdd = async(req) => {
+  const account_sid = parseAccountSid(req);
+  const service_provider_sid = parseServiceProviderSid(req);
+
+  if (service_provider_sid) {
+    if (req.user.hasServiceProviderAuth && service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+    if (req.user.hasAccountAuth && service_provider_sid !== req.user.service_provider_sid &&
+      req.body.account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+
+  if (account_sid) {
+    if (req.user.hasAccountAuth && account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+    );
+
+    if (req.user.hasServiceProviderAuth && r[0].service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+  return;
+};
+
+const validateRetrieveUpdateDelete = async(req, speech_credentials) => {
+  if (req.user.hasServiceProviderAuth && speech_credentials[0].service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+
+  if (req.user.hasAccountAuth && speech_credentials[0].account_sid !== req.user.account_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+  return;
+};
+
+const validateRetrieveList = async(req) => {
+  const service_provider_sid = parseServiceProviderSid(req);
+
+  if (service_provider_sid) {
+    if ((req.user.hasServiceProviderAuth || req.user.hasAccountAuth) &&
+     service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+  return;
+};
+
+const validateTest = async(req, speech_credentials) => {
+  if (req.user.hasAdminAuth) {
+    return;
+  }
+
+  if (!req.user.hasAdminAuth && speech_credentials.service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+
+  if (speech_credentials.service_provider_sid === req.user.service_provider_sid) {
+    if (req.user.hasServiceProviderAuth) {
+      return;
+    }
+
+    if (req.user.hasAccountAuth && (!speech_credentials.account_sid ||
+       speech_credentials.account_sid === req.user.account_sid)) {
+      return;
+    }
+
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+};
 
 const encryptCredential = (obj) => {
   const {
@@ -24,7 +107,24 @@ const encryptCredential = (obj) => {
     secret_access_key,
     aws_region,
     api_key,
-    region
+    region,
+    client_id,
+    secret,
+    nuance_tts_uri,
+    nuance_stt_uri,
+    use_custom_tts,
+    custom_tts_endpoint,
+    use_custom_stt,
+    custom_stt_endpoint,
+    tts_api_key,
+    tts_region,
+    stt_api_key,
+    stt_region,
+    riva_server_uri,
+    instance_id,
+    custom_stt_url,
+    custom_tts_url,
+    auth_token = ''
   } = obj;
 
   switch (vendor) {
@@ -49,7 +149,14 @@ const encryptCredential = (obj) => {
     case 'microsoft':
       assert(region, 'invalid azure speech credential: region is required');
       assert(api_key, 'invalid azure speech credential: api_key is required');
-      const azureData = JSON.stringify({region, api_key});
+      const azureData = JSON.stringify({
+        region,
+        api_key,
+        use_custom_tts,
+        custom_tts_endpoint,
+        use_custom_stt,
+        custom_stt_endpoint
+      });
       return encrypt(azureData);
 
     case 'wellsaid':
@@ -57,28 +164,63 @@ const encryptCredential = (obj) => {
       const wsData = JSON.stringify({api_key});
       return encrypt(wsData);
 
+    case 'nuance':
+      const checked = (client_id && secret) || (nuance_tts_uri || nuance_stt_uri);
+      assert(checked, 'invalid nuance speech credential: either entered client id and\
+        secret or entered a nuance_tts_uri or nuance_stt_uri');
+      const nuanceData = JSON.stringify({client_id, secret, nuance_tts_uri, nuance_stt_uri});
+      return encrypt(nuanceData);
+
+    case 'deepgram':
+      assert(api_key, 'invalid deepgram speech credential: api_key is required');
+      const deepgramData = JSON.stringify({api_key});
+      return encrypt(deepgramData);
+
+    case 'ibm':
+      const ibmData = JSON.stringify({tts_api_key, tts_region, stt_api_key, stt_region, instance_id});
+      return encrypt(ibmData);
+
+    case 'nvidia':
+      assert(riva_server_uri, 'invalid riva server uri: riva_server_uri is required');
+      const nvidiaData = JSON.stringify({ riva_server_uri });
+      return encrypt(nvidiaData);
+
+    case 'soniox':
+      assert(api_key, 'invalid soniox speech credential: api_key is required');
+      const sonioxData = JSON.stringify({api_key});
+      return encrypt(sonioxData);
+
     default:
-      assert(false, `invalid or missing vendor: ${vendor}`);
+      if (vendor.startsWith('custom:')) {
+        const customData = JSON.stringify({auth_token, custom_stt_url, custom_tts_url});
+        return encrypt(customData);
+      }
+      else assert(false, `invalid or missing vendor: ${vendor}`);
   }
 };
 
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
-  const {
-    use_for_stt,
-    use_for_tts,
-    vendor,
-  } = req.body;
-  const account_sid = req.user.account_sid || req.body.account_sid;
-  let service_provider_sid;
-  if (!account_sid) {
-    if (!req.user.hasServiceProviderAuth) {
-      logger.error('POST /SpeechCredentials invalid credentials');
-      return res.send(403);
-    }
-    service_provider_sid = parseServiceProviderSid(req);
-  }
+
   try {
+    const {
+      use_for_stt,
+      use_for_tts,
+      vendor,
+    } = req.body;
+    const account_sid = req.user.account_sid || req.body.account_sid;
+    const service_provider_sid = req.user.service_provider_sid ||
+    req.body.service_provider_sid || parseServiceProviderSid(req);
+
+    await validateAdd(req);
+
+    if (!account_sid) {
+      if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
+        logger.error('POST /SpeechCredentials invalid credentials');
+        return res.sendStatus(403);
+      }
+    }
+
     const encrypted_credential = encryptCredential(req.body);
     const uuid = await SpeechCredential.make({
       account_sid,
@@ -98,36 +240,96 @@ router.post('/', async(req, res) => {
  * retrieve all speech credentials for an account
  */
 router.get('/', async(req, res) => {
-  let service_provider_sid;
-  const account_sid = parseAccountSid(req);
-  if (!account_sid) service_provider_sid = parseServiceProviderSid(req);
   const logger = req.app.locals.logger;
+
   try {
-    const creds = account_sid ?
-      await SpeechCredential.retrieveAll(account_sid) :
-      await SpeechCredential.retrieveAllForSP(service_provider_sid);
+    const account_sid = parseAccountSid(req) ? parseAccountSid(req) : req.user.account_sid;
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    await validateRetrieveList(req);
+
+    const credsAccount = account_sid ? await SpeechCredential.retrieveAll(account_sid) : [];
+    const credsSP = service_provider_sid ?
+      await SpeechCredential.retrieveAllForSP(service_provider_sid) :
+      await SpeechCredential.retrieveAllForSP((await Account.retrieve(account_sid))[0].service_provider_sid);
+
+    // filter out duplicates and discard those from other non-matching accounts
+    let creds = [...new Set([...credsAccount, ...credsSP].map((c) => JSON.stringify(c)))].map((c) => JSON.parse(c));
+    if (req.user.hasScope('account')) {
+      creds = creds.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid);
+    }
 
     res.status(200).json(creds.map((c) => {
       const {credential, ...obj} = c;
+
       if ('google' === obj.vendor) {
-        obj.service_key = JSON.parse(decrypt(credential));
+        const o = JSON.parse(decrypt(credential));
+        const key_header = '-----BEGIN PRIVATE KEY-----\n';
+        const obscured = {
+          ...o,
+          private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
+        };
+        obj.service_key = obscured;
       }
       else if ('aws' === obj.vendor) {
         const o = JSON.parse(decrypt(credential));
         obj.access_key_id = o.access_key_id;
-        obj.secret_access_key = o.secret_access_key;
+        obj.secret_access_key = obscureKey(o.secret_access_key);
         obj.aws_region = o.aws_region;
         logger.info({obj, o}, 'retrieving aws speech credential');
       }
       else if ('microsoft' === obj.vendor) {
         const o = JSON.parse(decrypt(credential));
-        obj.api_key = o.api_key;
+        obj.api_key = obscureKey(o.api_key);
         obj.region = o.region;
+        obj.use_custom_tts = o.use_custom_tts;
+        obj.custom_tts_endpoint = o.custom_tts_endpoint;
+        obj.use_custom_stt = o.use_custom_stt;
+        obj.custom_stt_endpoint = o.custom_stt_endpoint;
         logger.info({obj, o}, 'retrieving azure speech credential');
       }
       else if ('wellsaid' === obj.vendor) {
         const o = JSON.parse(decrypt(credential));
-        obj.api_key = o.api_key;
+        obj.api_key = obscureKey(o.api_key);
+      }
+      else if ('nuance' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.client_id = o.client_id;
+        obj.secret = o.secret ? obscureKey(o.secret) : null;
+      }
+      else if ('deepgram' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.api_key = obscureKey(o.api_key);
+      }
+      else if ('ibm' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.tts_api_key = obscureKey(o.tts_api_key);
+        obj.tts_region = o.tts_region;
+        obj.stt_api_key = obscureKey(o.stt_api_key);
+        obj.stt_region = o.stt_region;
+        obj.instance_id = o.instance_id;
+      } else if ('nvidia' == obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.riva_server_uri = o.riva_server_uri;
+      }
+      else if ('soniox' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.api_key = obscureKey(o.api_key);
+      }
+      else if (obj.vendor.startsWith('custom:')) {
+        const o = JSON.parse(decrypt(credential));
+        obj.auth_token = obscureKey(o.auth_token);
+        obj.custom_stt_url = o.custom_stt_url;
+        obj.custom_tts_url = o.custom_tts_url;
+      }
+
+      if (req.user.hasAccountAuth && obj.account_sid === null) {
+        delete obj.api_key;
+        delete obj.secret_access_key;
+        delete obj.secret;
+        delete obj.auth_token;
+        delete obj.stt_api_key;
+        delete obj.tts_api_key;
       }
       return obj;
     }));
@@ -140,26 +342,85 @@ router.get('/', async(req, res) => {
  * retrieve a specific speech credential
  */
 router.get('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
     const cred = await SpeechCredential.retrieve(sid);
     if (0 === cred.length) return res.sendStatus(404);
+
+    await validateRetrieveUpdateDelete(req, cred);
+
     const {credential, ...obj} = cred[0];
     if ('google' === obj.vendor) {
-      obj.service_key = decrypt(credential);
+      const o = JSON.parse(decrypt(credential));
+      const key_header = '-----BEGIN PRIVATE KEY-----\n';
+      const obscured = {
+        ...o,
+        private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
+      };
+      obj.service_key = JSON.stringify(obscured);
     }
     else if ('aws' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.access_key_id = o.access_key_id;
-      obj.secret_access_key = o.secret_access_key;
+      obj.secret_access_key = obscureKey(o.secret_access_key);
       obj.aws_region = o.aws_region;
     }
     else if ('microsoft' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
-      obj.api_key = o.api_key;
+      obj.api_key = obscureKey(o.api_key);
       obj.region = o.region;
+      obj.use_custom_tts = o.use_custom_tts;
+      obj.custom_tts_endpoint = o.custom_tts_endpoint;
+      obj.use_custom_stt = o.use_custom_stt;
+      obj.custom_stt_endpoint = o.custom_stt_endpoint;
     }
+    else if ('wellsaid' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = obscureKey(o.api_key);
+    }
+    else if ('nuance' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.client_id = o.client_id;
+      obj.secret = o.secret ? obscureKey(o.secret) : null;
+      obj.nuance_tts_uri = o.nuance_tts_uri;
+      obj.nuance_stt_uri = o.nuance_stt_uri;
+    }
+    else if ('deepgram' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = obscureKey(o.api_key);
+    }
+    else if ('ibm' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.tts_api_key = obscureKey(o.tts_api_key);
+      obj.tts_region = o.tts_region;
+      obj.stt_api_key = obscureKey(o.stt_api_key);
+      obj.stt_region = o.stt_region;
+      obj.instance_id = o.instance_id;
+    } else if ('nvidia' == obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.riva_server_uri = o.riva_server_uri;
+    }
+    else if ('soniox' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = obscureKey(o.api_key);
+    }
+    else if (obj.vendor.startsWith('custom:')) {
+      const o = JSON.parse(decrypt(credential));
+      obj.auth_token = obscureKey(o.auth_token);
+      obj.custom_stt_url = o.custom_stt_url;
+      obj.custom_tts_url = o.custom_tts_url;
+    }
+
+    if (req.user.hasAccountAuth && obj.account_sid === null) {
+      delete obj.api_key;
+      delete obj.secret_access_key;
+      delete obj.secret;
+      delete obj.auth_token;
+      delete obj.stt_api_key;
+      delete obj.tts_api_key;
+    }
+
     res.status(200).json(obj);
   } catch (err) {
     sysError(logger, res, err);
@@ -170,9 +431,11 @@ router.get('/:sid', async(req, res) => {
  * delete a speech credential
  */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
+    const cred = await SpeechCredential.retrieve(sid);
+    await validateRetrieveUpdateDelete(req, cred);
     const count = await SpeechCredential.remove(sid);
     if (0 === count) return res.sendStatus(404);
     res.sendStatus(204);
@@ -186,10 +449,11 @@ router.delete('/:sid', async(req, res) => {
  * update a speech credential -- we only allow use_for_tts and use_for_stt to be updated
  */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
-    const {use_for_tts, use_for_stt} = req.body;
+    const sid = parseSpeechCredentialSid(req);
+    const {use_for_tts, use_for_stt, region, aws_region, stt_region, tts_region,
+      riva_server_uri, nuance_tts_uri, nuance_stt_uri} = req.body;
     if (typeof use_for_tts === 'undefined' && typeof use_for_stt === 'undefined') {
       throw new DbErrorUnprocessableRequest('use_for_tts and use_for_stt are the only updateable fields');
     }
@@ -203,9 +467,51 @@ router.put('/:sid', async(req, res) => {
 
     /* update the credential if provided */
     try {
-      obj.credential = encryptCredential(req.body);
-    } catch (err) {}
+      const cred = await SpeechCredential.retrieve(sid);
 
+      await validateRetrieveUpdateDelete(req, cred);
+
+      if (1 === cred.length) {
+        const {credential, vendor} = cred[0];
+        const o = JSON.parse(decrypt(credential));
+        const {
+          use_custom_tts,
+          custom_tts_endpoint,
+          use_custom_stt,
+          custom_stt_endpoint,
+          custom_stt_url,
+          custom_tts_url
+        } = req.body;
+
+        const newCred = {
+          ...o,
+          region,
+          vendor,
+          aws_region,
+          use_custom_tts,
+          custom_tts_endpoint,
+          use_custom_stt,
+          custom_stt_endpoint,
+          stt_region,
+          tts_region,
+          riva_server_uri,
+          nuance_stt_uri,
+          nuance_tts_uri,
+          custom_stt_url,
+          custom_tts_url
+        };
+        logger.info({o, newCred}, 'updating speech credential with this new credential');
+        obj.credential = encryptCredential(newCred);
+        obj.vendor = vendor;
+      }
+      else {
+        logger.info({sid}, 'speech credential not found!!');
+      }
+    } catch (err) {
+      logger.error({err}, 'error updating speech credential');
+    }
+
+    logger.info({obj}, 'updating speech credential with changes');
     const rowsAffected = await SpeechCredential.update(sid, obj);
     if (rowsAffected === 0) {
       return res.sendStatus(404);
@@ -221,12 +527,15 @@ router.put('/:sid', async(req, res) => {
  * Test a credential
  */
 router.get('/:sid/test', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
     const creds = await SpeechCredential.retrieve(sid);
+
     if (!creds || 0 === creds.length) return res.sendStatus(404);
 
+    await validateTest(req, creds[0]);
+
     const cred = creds[0];
     const credential = JSON.parse(decrypt(cred.credential));
     const results = {
@@ -244,7 +553,8 @@ router.get('/:sid/test', async(req, res) => {
 
       if (cred.use_for_tts) {
         try {
-          await testGoogleTts(logger, credential);
+          const {getTtsVoices} = req.app.locals;
+          await testGoogleTts(logger, getTtsVoices, credential);
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
         } catch (err) {
@@ -266,8 +576,9 @@ router.get('/:sid/test', async(req, res) => {
     }
     else if (cred.vendor === 'aws') {
       if (cred.use_for_tts) {
+        const {getTtsVoices} = req.app.locals;
         try {
-          await testAwsTts(logger, {
+          await testAwsTts(logger, getTtsVoices, {
             accessKeyId: credential.access_key_id,
             secretAccessKey: credential.secret_access_key,
             region: credential.aws_region || process.env.AWS_REGION
@@ -295,10 +606,24 @@ router.get('/:sid/test', async(req, res) => {
       }
     }
     else if (cred.vendor === 'microsoft') {
-      const {api_key, region} = credential;
+      const {
+        api_key,
+        region,
+        use_custom_tts,
+        custom_tts_endpoint,
+        use_custom_stt,
+        custom_stt_endpoint
+      } = credential;
       if (cred.use_for_tts) {
         try {
-          await testMicrosoftTts(logger, {api_key, region});
+          await testMicrosoftTts(logger, {
+            api_key,
+            region,
+            use_custom_tts,
+            custom_tts_endpoint,
+            use_custom_stt,
+            custom_stt_endpoint
+          });
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
         } catch (err) {
@@ -330,7 +655,106 @@ router.get('/:sid/test', async(req, res) => {
         }
       }
     }
+    else if (cred.vendor === 'nuance') {
+      const {getTtsVoices} = req.app.locals;
+
+      const {
+        client_id,
+        secret,
+        nuance_tts_uri,
+        nuance_stt_uri
+      } = credential;
+      if (cred.use_for_tts) {
+        try {
+          await testNuanceTts(logger, getTtsVoices, {
+            client_id,
+            secret,
+            nuance_tts_uri
+          });
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          logger.error({err}, 'error testing nuance tts');
+          const reason = err.statusCode === 401 ?
+            'invalid client_id or secret' :
+            (err.message || 'error accessing nuance tts service with provided credentials');
+          results.tts = {status: 'fail', reason};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+      if (cred.use_for_stt) {
+        try {
+          await testNuanceStt(logger, {client_id, secret, nuance_stt_uri});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'deepgram') {
+      const {api_key} = credential;
+      if (cred.use_for_stt) {
+        try {
+          await testDeepgramStt(logger, {api_key});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'ibm') {
+      const {getTtsVoices} = req.app.locals;
+
+      if (cred.use_for_tts) {
+        const {tts_api_key, tts_region} = credential;
+        try {
+          await testIbmTts(logger, getTtsVoices, {
+            tts_api_key,
+            tts_region
+          });
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          logger.error({err}, 'error testing ibm tts');
+          const reason = err.statusCode === 401 ?
+            'invalid api_key or region' :
+            (err.message || 'error accessing ibm tts service with provided credentials');
+          results.tts = {status: 'fail', reason};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+      if (cred.use_for_stt) {
+        const {stt_api_key, stt_region, instance_id} = credential;
+        try {
+          await testIbmStt(logger, {stt_region, stt_api_key, instance_id});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'soniox') {
+      const {api_key} = credential;
+      if (cred.use_for_stt) {
+        try {
+          await testSonioxStt(logger, {api_key});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+
     res.status(200).json(results);
+
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/system-information.js

@@ -0,0 +1,14 @@
+const router = require('express').Router();
+const SystemInformation = require('../../models/system-information');
+
+router.post('/', async(req, res) => {
+  const sysInfo = await SystemInformation.add(req.body);
+  res.status(201).json(sysInfo);
+});
+
+router.get('/', async(req, res) => {
+  const [sysInfo] = await SystemInformation.retrieveAll();
+  res.status(200).json(sysInfo || {});
+});
+
+module.exports = router;

lib/routes/api/tts-cache.js

@@ -0,0 +1,29 @@
+const router = require('express').Router();
+const {
+  parseAccountSid
+} = require('./utils');
+
+router.delete('/', async(req, res) => {
+  const {purgeTtsCache} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  if (account_sid) {
+    await purgeTtsCache({account_sid});
+  } else {
+    await purgeTtsCache();
+  }
+  res.sendStatus(204);
+});
+
+router.get('/', async(req, res) => {
+  const {getTtsSize} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  let size = 0;
+  if (account_sid) {
+    size = await getTtsSize(`tts:${account_sid}:*`);
+  } else {
+    size = await getTtsSize();
+  }
+  res.status(200).json({size});
+});
+
+module.exports = router;

lib/routes/api/users.js

@@ -1,16 +1,22 @@
-//const assert = require('assert');
-//const debug = require('debug')('jambonz:api-server');
 const router = require('express').Router();
-const {DbErrorBadRequest} = require('../../utils/errors');
+const User = require('../../models/user');
+const {DbErrorBadRequest, BadRequestError, DbErrorForbidden} = require('../../utils/errors');
 const {generateHashedPassword, verifyPassword} = require('../../utils/password-utils');
 const {promisePool} = require('../../db');
+const {validatePasswordSettings, parseUserSid} = require('./utils');
 const {decrypt} = require('../../utils/encrypt-decrypt');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 const retrieveMyDetails = `SELECT * 
 FROM users user
 JOIN accounts AS account ON account.account_sid = user.account_sid 
 LEFT JOIN service_providers as sp ON account.service_provider_sid = sp.service_provider_sid  
 WHERE user.user_sid = ?`;
+const retrieveMyDetails2 = `SELECT * 
+FROM users user
+LEFT JOIN accounts AS account ON account.account_sid = user.account_sid 
+LEFT JOIN service_providers as sp ON sp.service_provider_sid = user.service_provider_sid 
+WHERE user.user_sid = ?`;
 const retrieveSql = 'SELECT * from users where user_sid = ?';
 const retrieveProducts = `SELECT * 
 FROM account_products 
@@ -22,109 +28,290 @@ AND account_subscriptions.pending=0`;
 const updateSql = 'UPDATE users set hashed_password = ?, force_change = false WHERE user_sid = ?';
 const retrieveStaticIps = 'SELECT * FROM account_static_ips WHERE account_sid = ?';
 
-const validateRequest = async(user_sid, payload) => {
-  const {old_password, new_password, name, email, email_activation_code} = payload;
+const validateRequest = async(user_sid, req) => {
+  const payload = req.body;
+  const {
+    old_password,
+    new_password,
+    initial_password,
+    name,
+    email,
+    email_activation_code,
+    force_change,
+    is_active
+  } = payload;
 
   const [r] = await promisePool.query(retrieveSql, user_sid);
-  if (r.length === 0) return null;
+  if (r.length === 0) {
+    throw new DbErrorBadRequest('Invalid request: user_sid does not exist');
+  }
   const user = r[0];
 
+  /* it is not allowed for anyone to promote a user to a higher level of authority */
+  if (null === payload.account_sid || null === payload.service_provider_sid) {
+    throw new DbErrorBadRequest('Invalid request: user may not be promoted');
+  }
+
+  if (req.user.hasAccountAuth) {
+    /* account user may not change modify account_sid or service_provider_sid */
+    if ('account_sid' in payload && payload.account_sid !== user.account_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another account');
+    }
+    if ('service_provider_sid' in payload && payload.service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another service provider');
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    if ('service_provider_sid' in payload && payload.service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another service provider');
+    }
+  }
+  if ('account_sid' in payload) {
+    const [r] = await promisePool.query('SELECT * FROM accounts WHERE account_sid = ?', payload.account_sid);
+    if (r.length === 0) throw new DbErrorBadRequest('Invalid request: account_sid does not exist');
+    const {service_provider_sid} = r[0];
+    if (service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be moved to another service provider');
+    }
+  }
+
+  if (initial_password) {
+    await validatePasswordSettings(initial_password);
+  }
+
   if ((old_password && !new_password) || (new_password && !old_password)) {
     throw new DbErrorBadRequest('new_password and old_password both required');
   }
+  if (new_password) {
+    await validatePasswordSettings(new_password);
+  }
   if (new_password && name) throw new DbErrorBadRequest('can not change name and password simultaneously');
   if (new_password && user.provider !== 'local') {
     throw new DbErrorBadRequest('can not change password when using oauth2');
   }
 
-  if ((email && !email_activation_code) || (email_activation_code && !email)) {
+  if (email_activation_code && !email) {
     throw new DbErrorBadRequest('email and email_activation_code both required');
   }
-  if (!name && !new_password && !email) throw new DbErrorBadRequest('no updates requested');
+  if (!name && !new_password && !email && !initial_password && !force_change && !is_active)
+    throw new DbErrorBadRequest('no updates requested');
 
   return user;
 };
 
+const getActiveAdminUsers = (users) => {
+  return users.filter((e) => !e.account_sid && !e.service_provider_sid && e.is_active);
+};
+
+const ensureUserActionIsAllowed = (req, user) => {
+  if (req.user.hasAdminAuth) {
+    return;
+  }
+
+  if (req.user.hasServiceProviderAuth && req.user.service_provider_sid === user.service_provider_sid) {
+    return;
+  }
+
+  if (req.user.hasAccountAuth && req.user.account_sid === user.account_sid) {
+    return;
+  }
+
+  throw new DbErrorForbidden('insufficient permissions');
+};
+
+const ensureUserDeletionIsAllowed = (req, activeAdminUsers, user) => {
+  try {
+    if (req.user.hasAdminAuth && activeAdminUsers.length === 1 && activeAdminUsers[0].user_sid === user[0].user_sid) {
+      throw new BadRequestError('cannot delete this admin user - there are no other active admin users');
+    }
+
+    ensureUserActionIsAllowed(req, user[0]);
+    return;
+  } catch (error) {
+    throw error;
+  }
+};
+
+const ensureUserRetrievalIsAllowed = (req, user) => {
+  try {
+    ensureUserActionIsAllowed(req, user);
+    return;
+  } catch (error) {
+    throw error;
+  }
+};
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  let usersList;
+  try {
+    let results;
+    if (req.user.hasAdminAuth) {
+      results = await User.retrieveAll();
+    }
+    else if (req.user.hasAccountAuth) {
+      results = await User.retrieveAllForAccount(req.user.account_sid, true);
+    }
+    else if (req.user.hasServiceProviderAuth) {
+      results = await User.retrieveAllForServiceProvider(req.user.service_provider_sid, true);
+    }
+
+    if (results.length === 0) throw new Error('failure retrieving users list');
+
+    usersList = results.map((user) => {
+      const {
+        user_sid,
+        name,
+        email,
+        force_change,
+        is_active,
+        account_sid,
+        service_provider_sid,
+        account_name,
+        service_provider_name
+      } = user;
+      let scope;
+      if (account_sid && service_provider_sid) {
+        scope = 'account';
+      } else if (service_provider_sid) {
+        scope = 'service_provider';
+      } else {
+        scope = 'admin';
+      }
+
+      const obj = {
+        user_sid,
+        name,
+        email,
+        scope,
+        force_change,
+        is_active,
+        ...(account_sid && {account_sid}),
+        ...(account_name && {account_name}),
+        ...(service_provider_sid && {service_provider_sid}),
+        ...(service_provider_name && {service_provider_name})
+      };
+      return obj;
+    });
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+  res.status(200).json(usersList);
+});
+
 router.get('/me', async(req, res) => {
   const logger = req.app.locals.logger;
   const {user_sid} = req.user;
 
   if (!user_sid) return res.sendStatus(403);
 
+  let payload;
   try {
-    const [r] = await promisePool.query({sql: retrieveMyDetails, nestTables: true}, user_sid);
-    logger.debug(r, 'retrieved user details');
-    const payload = r[0];
-    const {user, account, sp} = payload;
-    ['hashed_password', 'salt', 'phone_activation_code', 'email_activation_code', 'account_sid'].forEach((prop) => {
-      delete user[prop];
-    });
-    ['email_validated', 'phone_validated', 'force_change'].forEach((prop) => user[prop] = !!user[prop]);
-    ['is_active'].forEach((prop) => account[prop] = !!account[prop]);
-    account.root_domain = sp.root_domain;
-    delete payload.sp;
-
-    /* get api keys */
-    const [keys] = await promisePool.query('SELECT * from api_keys WHERE account_sid = ?', account.account_sid);
-    payload.api_keys = keys.map((k) => {
-      return {
-        api_key_sid: k.api_key_sid,
-        //token: k.token.replace(/.(?=.{4,}$)/g, '*'),
-        token: k.token,
-        last_used: k.last_used,
-        created_at: k.created_at
-      };
-    });
-
-    /* get products */
-    const [products] = await promisePool.query({sql: retrieveProducts, nestTables: true}, account.account_sid);
-    if (!products.length || !products[0].account_subscriptions) {
-      throw new Error('account is missing a subscription');
-    }
-    const account_subscription = products[0].account_subscriptions;
-    payload.subscription = {
-      status: 'active',
-      account_subscription_sid: account_subscription.account_subscription_sid,
-      start_date: account_subscription.effective_start_date,
-      products: products.map((prd) => {
-        return {
-          name: prd.products.name,
-          units: prd.products.unit_label,
-          quantity: prd.account_products.quantity
-        };
-      })
-    };
-    if (account_subscription.pending) {
-      Object.assign(payload.subscription, {
-        status: 'suspended',
-        suspend_reason: account_subscription.pending_reason
+    if (process.env.JAMBONES_HOSTING) {
+      const [r] = await promisePool.query({sql: retrieveMyDetails, nestTables: true}, user_sid);
+      logger.debug(r, 'retrieved user details');
+      payload = r[0];
+      const {user, account, sp} = payload;
+      ['hashed_password', 'salt', 'phone_activation_code', 'email_activation_code', 'account_sid'].forEach((prop) => {
+        delete user[prop];
       });
-    }
-    const {
-      last4,
-      exp_month,
-      exp_year,
-      card_type,
-      stripe_statement_descriptor
-    } = account_subscription;
-    if (last4) {
-      const real_last4 = decrypt(last4);
-      Object.assign(payload.subscription, {
-        last4: real_last4,
+      ['email_validated', 'phone_validated', 'force_change'].forEach((prop) => user[prop] = !!user[prop]);
+      ['is_active'].forEach((prop) => account[prop] = !!account[prop]);
+      account.root_domain = sp.root_domain;
+      delete payload.sp;
+
+      /* get api keys */
+      const [keys] = await promisePool.query('SELECT * from api_keys WHERE account_sid = ?', account.account_sid);
+      payload.api_keys = keys.map((k) => {
+        return {
+          api_key_sid: k.api_key_sid,
+          //token: k.token.replace(/.(?=.{4,}$)/g, '*'),
+          token: k.token,
+          last_used: k.last_used,
+          created_at: k.created_at
+        };
+      });
+
+      /* get products */
+      const [products] = await promisePool.query({sql: retrieveProducts, nestTables: true}, account.account_sid);
+      if (!products.length || !products[0].account_subscriptions) {
+        throw new Error('account is missing a subscription');
+      }
+      const account_subscription = products[0].account_subscriptions;
+      payload.subscription = {
+        status: 'active',
+        account_subscription_sid: account_subscription.account_subscription_sid,
+        start_date: account_subscription.effective_start_date,
+        products: products.map((prd) => {
+          return {
+            name: prd.products.name,
+            units: prd.products.unit_label,
+            quantity: prd.account_products.quantity
+          };
+        })
+      };
+      if (account_subscription.pending) {
+        Object.assign(payload.subscription, {
+          status: 'suspended',
+          suspend_reason: account_subscription.pending_reason
+        });
+      }
+      const {
+        last4,
         exp_month,
         exp_year,
         card_type,
-        statement_descriptor: stripe_statement_descriptor
+        stripe_statement_descriptor
+      } = account_subscription;
+      if (last4) {
+        const real_last4 = decrypt(last4);
+        Object.assign(payload.subscription, {
+          last4: real_last4,
+          exp_month,
+          exp_year,
+          card_type,
+          statement_descriptor: stripe_statement_descriptor
+        });
+      }
+
+      /* get static ips */
+      const [static_ips] = await promisePool.query(retrieveStaticIps, account.account_sid);
+      payload.static_ips = static_ips.map((r) => r.public_ipv4);
+    }
+    else {
+      const [r] = await promisePool.query({sql: retrieveMyDetails2, nestTables: true}, user_sid);
+      logger.debug(r, 'retrieved user details');
+      payload = r[0];
+      const {user} = payload;
+      ['hashed_password', 'salt', 'phone_activation_code', 'email_activation_code'].forEach((prop) => {
+        delete user[prop];
       });
+      ['email_validated', 'phone_validated', 'force_change'].forEach((prop) => user[prop] = !!user[prop]);
+    }
+    logger.debug({payload}, 'returning user details');
+    res.json(payload);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:user_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const user_sid = parseUserSid(req);
+    const [user] = await User.retrieve(user_sid);
+
+    if (!user) {
+      throw new Error('failure retrieving user');
     }
 
-    /* get static ips */
-    const [static_ips] = await promisePool.query(retrieveStaticIps, account.account_sid);
-    payload.static_ips = static_ips.map((r) => r.public_ipv4);
+    ensureUserRetrievalIsAllowed(req, user);
 
-    logger.debug({payload}, 'returning user details');
-
-    res.json(payload);
+    // eslint-disable-next-line no-unused-vars
+    const { hashed_password, ...rest } = user;
+    return res.status(200).json(rest);
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -133,12 +320,32 @@ router.get('/me', async(req, res) => {
 router.put('/:user_sid', async(req, res) => {
   const logger = req.app.locals.logger;
   const {user_sid} = req.params;
-  const {old_password, new_password, name, email, email_activation_code} = req.body;
+  const user = await User.retrieve(user_sid);
+  const {hasAccountAuth, hasServiceProviderAuth, hasAdminAuth} = req.user;
+  const {
+    old_password,
+    new_password,
+    initial_password,
+    email_activation_code,
+    email,
+    name,
+    is_active,
+    force_change,
+    account_sid,
+    service_provider_sid
+  } = req.body;
 
-  if (req.user.user_sid && req.user.user_sid !== user_sid) return res.sendStatus(403);
+  //if (req.user.user_sid && req.user.user_sid !== user_sid) return res.sendStatus(403);
+
+  if (!hasAdminAuth &&
+  !(hasAccountAuth && req.user.account_sid === user[0].account_sid) &&
+  !(hasServiceProviderAuth && req.user.service_provider_sid === user[0].service_provider_sid) &&
+  (req.user.user_sid && req.user.user_sid !== user_sid)) {
+    return res.sendStatus(403);
+  }
 
   try {
-    const user = await validateRequest(user_sid, req.body);
+    const user = await validateRequest(user_sid, req);
     if (!user) return res.sendStatus(404);
 
     if (new_password) {
@@ -149,6 +356,11 @@ router.put('/:user_sid', async(req, res) => {
         //debug(`PUT /Users/:sid pwd ${old_password} does not match hash ${old_hashed_password}`);
         return res.sendStatus(403);
       }
+
+      if (old_password === new_password) {
+        throw new Error('new password cannot be your old password');
+      }
+
       const passwordHash = await generateHashedPassword(new_password);
       //debug(`updating hashed_password to ${passwordHash}`);
       const r = await promisePool.execute(updateSql, [passwordHash, user_sid]);
@@ -160,10 +372,55 @@ router.put('/:user_sid', async(req, res) => {
       if (0 === r.changedRows) throw new Error('database update failed');
     }
 
-    if (email) {
+    if (initial_password) {
+      const passwordHash = await generateHashedPassword(initial_password);
       const r = await promisePool.execute(
-        'UPDATE users SET email = ?, email_activation_code = ?, email_validated = 0 WHERE user_sid = ?',
-        [email, email_activation_code, user_sid]);
+        'UPDATE users SET hashed_password = ? WHERE user_sid = ?',
+        [passwordHash, user_sid]
+      );
+      if (0 === r.changedRows) throw new Error('database update failed');
+    }
+
+    if (typeof is_active !== 'undefined') {
+      const r = await promisePool.execute('UPDATE users SET is_active = ? WHERE user_sid = ?', [is_active, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+    }
+
+    if (typeof force_change !== 'undefined') {
+      const r = await promisePool.execute(
+        'UPDATE users SET force_change = ? WHERE user_sid = ?',
+        [force_change, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+    }
+
+    if (account_sid || account_sid === null) {
+      const r = await promisePool.execute(
+        'UPDATE users SET account_sid = ? WHERE user_sid = ?',
+        [account_sid, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+      const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+      await cacheClient.delete(redisKey);
+    }
+
+    if (service_provider_sid || service_provider_sid === null) {
+      const r = await promisePool.execute(
+        'UPDATE users SET service_provider_sid = ? WHERE user_sid = ?',
+        [service_provider_sid, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+      const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+      await cacheClient.delete(redisKey);
+    }
+
+    if (email) {
+      if (email_activation_code) {
+        const r = await promisePool.execute(
+          'UPDATE users SET email = ?, email_activation_code = ?, email_validated = 0 WHERE user_sid = ?',
+          [email, email_activation_code, user_sid]);
+        if (0 === r.changedRows) throw new Error('database update failed');
+      }
+      const r = await promisePool.execute(
+        'UPDATE users SET email = ? WHERE user_sid = ?',
+        [email, user_sid]);
       if (0 === r.changedRows) throw new Error('database update failed');
 
       if (process.env.NODE_ENV !== 'test') {
@@ -176,5 +433,80 @@ router.put('/:user_sid', async(req, res) => {
   }
 });
 
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const passwordHash = await generateHashedPassword(req.body.initial_password);
+  const payload = {
+    ...req.body,
+    provider: 'local',
+    hashed_password: passwordHash,
+  };
+  const allUsers = await User.retrieveAll();
+  delete payload.initial_password;
+
+  try {
+    if (req.body.initial_password) {
+      await validatePasswordSettings(req.body.initial_password);
+    }
+    const email = allUsers.find((e) => e.email === payload.email);
+    const name = allUsers.find((e) => e.name === payload.name);
+
+    if (name) {
+      logger.debug({payload}, 'user with this username already exists');
+      return res.status(422).json({msg: 'invalid username or email'});
+    }
+
+    if (email) {
+      logger.debug({payload}, 'user with this email already exists');
+      return res.status(422).json({msg: 'invalid username or email'});
+    }
+
+    if (req.user.hasAdminAuth) {
+      logger.debug({payload}, 'POST /users');
+      const uuid = await User.make(payload);
+      res.status(201).json({user_sid: uuid});
+    }
+    else if (req.user.hasAccountAuth) {
+      logger.debug({payload}, 'POST /users');
+      const uuid = await User.make({
+        ...payload,
+        account_sid: req.user.account_sid,
+      });
+      res.status(201).json({user_sid: uuid});
+    }
+    else if (req.user.hasServiceProviderAuth) {
+      logger.debug({payload}, 'POST /users');
+      const uuid = await User.make({
+        ...payload,
+        service_provider_sid: req.user.service_provider_sid,
+      });
+      res.status(201).json({user_sid: uuid});
+    }
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.delete('/:user_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const user_sid = parseUserSid(req);
+    const allUsers = await User.retrieveAll();
+    const activeAdminUsers = getActiveAdminUsers(allUsers);
+    const user = allUsers.filter((user) => user.user_sid === user_sid);
+
+    ensureUserDeletionIsAllowed(req, activeAdminUsers, user);
+    await User.remove(user_sid);
+
+    /* invalidate the jwt of the deleted user */
+    const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+    await cacheClient.delete(redisKey);
+
+    return res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
 
 module.exports = router;

lib/routes/api/utils.js

@@ -1,9 +1,10 @@
-const { v4: uuid } = require('uuid');
+const { v4: uuid, validate } = require('uuid');
 const bent = require('bent');
 const Account = require('../../models/account');
 const {promisePool} = require('../../db');
 const {cancelSubscription, detachPaymentMethod} = require('../../utils/stripe-utils');
 const freePlans = require('../../utils/free_plans');
+const { BadRequestError, DbErrorBadRequest } = require('../../utils/errors');
 const insertAccountSubscriptionSql = `INSERT INTO account_subscriptions 
 (account_subscription_sid, account_sid) 
 values (?, ?)`;
@@ -126,7 +127,7 @@ const createTestAlerts = async(writeAlerts, AlertType, account_sid) => {
         data.push({timestamp, account_sid, alert_type: AlertType.CARRIER_NOT_PROVISIONED});
         break;
       case 4:
-        data.push({timestamp, account_sid, alert_type: AlertType.CALL_LIMIT, count: 50});
+        data.push({timestamp, account_sid, alert_type: AlertType.ACCOUNT_CALL_LIMIT, count: 50});
         break;
       default:
         break;
@@ -137,38 +138,179 @@ const createTestAlerts = async(writeAlerts, AlertType, account_sid) => {
 
 };
 
+const validateSid = (model, req) => {
+  const arr = new RegExp(`${model}\/([^\/]*)`).exec(req.originalUrl);
+
+  if (arr) {
+    const sid = arr[1];
+    const sid_validation = validate(sid);
+    if (!sid_validation) {
+      throw new BadRequestError(`invalid ${model}Sid format`);
+    }
+
+    return arr[1];
+  }
+
+  return;
+};
+
 const parseServiceProviderSid = (req) => {
-  const arr = /ServiceProviders\/([^\/]*)/.exec(req.originalUrl);
-  if (arr) return arr[1];
+  try {
+    return validateSid('ServiceProviders', req);
+  } catch (error) {
+    throw error;
+  }
 };
 
 const parseAccountSid = (req) => {
-  const arr = /Accounts\/([^\/]*)/.exec(req.originalUrl);
-  if (arr) return arr[1];
+  try {
+    return validateSid('Accounts', req);
+  } catch (error) {
+    throw error;
+  }
 };
 
-const hasAccountPermissions = (req, res, next) => {
-  if (req.user.hasScope('admin')) return next();
-  if (req.user.hasScope('account')) {
-    const account_sid = parseAccountSid(req);
-    if (account_sid === req.user.account_sid) return next();
+const parseApplicationSid = (req) => {
+  try {
+    return validateSid('Applications', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseCallSid = (req) => {
+  try {
+    return validateSid('Calls', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parsePhoneNumberSid = (req) => {
+  try {
+    return validateSid('PhoneNumbers', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseSpeechCredentialSid = (req) => {
+  try {
+    return validateSid('SpeechCredentials', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseVoipCarrierSid = (req) => {
+  try {
+    return validateSid('VoipCarriers', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseWebhookSid = (req) => {
+  try {
+    return validateSid('Webhooks', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseSipGatewaySid = (req) => {
+  try {
+    return validateSid('SipGateways', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseUserSid = (req) => {
+  try {
+    return validateSid('Users', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseLcrSid = (req) => {
+  try {
+    return validateSid('Lcrs', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const hasAccountPermissions = async(req, res, next) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return next();
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const service_provider_sid = parseServiceProviderSid(req);
+      const account_sid = parseAccountSid(req);
+      if (service_provider_sid) {
+        if (service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+      if (account_sid) {
+        const [r] = await Account.retrieve(account_sid);
+        if (r && r.service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+    }
+
+    if (req.user.hasScope('account')) {
+      const account_sid = parseAccountSid(req);
+      const service_provider_sid = parseServiceProviderSid(req);
+      const [r] = await Account.retrieve(account_sid);
+
+      if (account_sid) {
+        if (r && r.account_sid === req.user.account_sid) {
+          return next();
+        }
+      }
+
+      if (service_provider_sid) {
+        if (r && r.service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+    }
+
+    res.status(403).json({
+      status: 'fail',
+      message: 'insufficient privileges'
+    });
+  } catch (error) {
+    throw error;
   }
-  res.status(403).json({
-    status: 'fail',
-    message: 'insufficient privileges'
-  });
 };
 
 const hasServiceProviderPermissions = (req, res, next) => {
-  if (req.user.hasScope('admin')) return next();
-  if (req.user.hasScope('service_provider')) {
-    const service_provider_sid = parseServiceProviderSid(req);
-    if (service_provider_sid === req.user.service_provider_sid) return next();
+  try {
+    if (req.user.hasScope('admin')) {
+      return next();
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const service_provider_sid = parseServiceProviderSid(req);
+      if (service_provider_sid === req.user.service_provider_sid) {
+        return next();
+      }
+    }
+
+    res.status(403).json({
+      status: 'fail',
+      message: 'insufficient privileges'
+    });
+  } catch (error) {
+    throw error;
   }
-  res.status(403).json({
-    status: 'fail',
-    message: 'insufficient privileges'
-  });
 };
 
 const checkLimits = async(req, res, next) => {
@@ -273,15 +415,50 @@ const disableSubspace = async(opts) => {
   return;
 };
 
+const validatePasswordSettings = async(password) => {
+  const sql = 'SELECT * from password_settings';
+  const [rows] = await promisePool.execute(sql);
+  const specialChars = /[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]+/;
+  const numbers = /[0-9]+/;
+  if (rows.length === 0) {
+    if (password.length < 8 || password.length > 20) {
+      throw new DbErrorBadRequest('password length must be between 8 and 20');
+    }
+  } else {
+    if (rows[0].min_password_length && password.length < rows[0].min_password_length) {
+      throw new DbErrorBadRequest(`password must be at least ${rows[0].min_password_length} characters long`);
+    }
+
+    if (rows[0].require_digit === 1 && !numbers.test(password)) {
+      throw new DbErrorBadRequest('password must contain at least one digit');
+    }
+
+    if (rows[0].require_special_character === 1 && !specialChars.test(password)) {
+      throw new DbErrorBadRequest('password must contain at least one special character');
+    }
+  }
+  return;
+};
+
 module.exports = {
   setupFreeTrial,
   createTestCdrs,
   createTestAlerts,
   parseAccountSid,
+  parseApplicationSid,
+  parseCallSid,
+  parsePhoneNumberSid,
   parseServiceProviderSid,
+  parseSpeechCredentialSid,
+  parseVoipCarrierSid,
+  parseWebhookSid,
+  parseSipGatewaySid,
+  parseUserSid,
+  parseLcrSid,
   hasAccountPermissions,
   hasServiceProviderPermissions,
   checkLimits,
   enableSubspace,
-  disableSubspace
+  disableSubspace,
+  validatePasswordSettings
 };

lib/routes/api/voip-carriers.js

@@ -4,6 +4,7 @@ const VoipCarrier = require('../../models/voip-carrier');
 const {promisePool} = require('../../db');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const { parseVoipCarrierSid } = require('./utils');
 
 const validate = async(req) => {
   const {lookupAppBySid, lookupAccountBySid} = req.app.locals;
@@ -73,7 +74,14 @@ decorate(router, VoipCarrier, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await VoipCarrier.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
+    const results = req.user.hasAdminAuth ?
+      await VoipCarrier.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null) :
+      await VoipCarrier.retrieveAllForSP(req.user.service_provider_sid);
+
+    if (req.user.hasScope('account')) {
+      return res.status(200).json(results.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid));
+    }
+
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -84,9 +92,24 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parseVoipCarrierSid(req);
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await VoipCarrier.retrieve(req.params.sid, account_sid);
+    const results = await VoipCarrier.retrieve(sid, account_sid);
     if (results.length === 0) return res.status(404).end();
+    const ret = results[0];
+    ret.register_status = JSON.parse(ret.register_status || '{}');
+
+    if (req.user.hasServiceProviderAuth && results.length === 1) {
+      if (results.length === 1 && results[0].service_provider_sid !== req.user.service_provider_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+    if (req.user.hasAccountAuth && results.length === 1) {
+      if (results.length === 1 && results[0].account_sid !== req.user.account_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/webhooks.js

@@ -2,15 +2,37 @@ const router = require('express').Router();
 const Webhook = require('../../models/webhook');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const {DbErrorForbidden} = require('../../utils/errors');
+const { parseWebhookSid } = require('./utils');
+const {promisePool} = require('../../db');
 
 decorate(router, Webhook, ['add']);
 
 /* retrieve */
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
+
   try {
-    const results = await Webhook.retrieve(req.params.sid);
+    const sid = parseWebhookSid(req);
+    const results = await Webhook.retrieve(sid);
+
     if (results.length === 0) return res.status(404).end();
+
+    if (req.user.hasAccountAuth) {
+      /* can only update carriers for the user's account */
+      if (results[0].account_sid !== req.user.account_sid) {
+        throw new DbErrorForbidden('insufficient privileges');
+      }
+    }
+    if (req.user.hasServiceProviderAuth) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [results[0].account_sid]
+      );
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+      throw new DbErrorForbidden('insufficient permissions');
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/error.js

@@ -1,6 +1,15 @@
-const {DbErrorBadRequest, DbErrorUnprocessableRequest, DbErrorForbidden} = require('../utils/errors');
+const {
+  BadRequestError,
+  DbErrorBadRequest,
+  DbErrorUnprocessableRequest,
+  DbErrorForbidden
+} = require('../utils/errors');
 
 function sysError(logger, res, err) {
+  if (err instanceof BadRequestError) {
+    logger.info(err, err.message);
+    return res.status(400).json({msg: 'Bad request'});
+  }
   if (err instanceof DbErrorBadRequest) {
     logger.info(err, 'invalid client request');
     return res.status(400).json({msg: err.message});

lib/utils/email-utils.js

@@ -1,6 +1,7 @@
 const formData = require('form-data');
 const Mailgun = require('mailgun.js');
 const mailgun = new Mailgun(formData);
+const bent = require('bent');
 const validateEmail = (email) => {
   // eslint-disable-next-line max-len
   const re = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
@@ -8,6 +9,44 @@ const validateEmail = (email) => {
 };
 
 const emailSimpleText = async(logger, to, subject, text) => {
+  const from = 'jambonz Support <support@jambonz.org>';
+  if (process.env.CUSTOM_EMAIL_VENDOR_URL) {
+    await sendEmailByCustomVendor(logger, from, to, subject, text);
+  } else {
+    await sendEmailByMailgun(logger, from, to, subject, text);
+  }
+
+};
+
+const sendEmailByCustomVendor = async(logger, from, to, subject, text) => {
+  try {
+    const post = bent('POST', {
+      'Content-Type': 'application/json',
+      ...((process.env.CUSTOM_EMAIL_VENDOR_USERNAME && process.env.CUSTOM_EMAIL_VENDOR_PASSWORD) &&
+        ({
+          'Authorization':`Basic ${Buffer.from(
+            `${process.env.CUSTOM_EMAIL_VENDOR_USERNAME}:${process.env.CUSTOM_EMAIL_VENDOR_PASSWORD}`
+          ).toString('base64')}`
+        }))
+    });
+
+    const res = await post(process.env.CUSTOM_EMAIL_VENDOR_URL, {
+      from,
+      to,
+      subject,
+      text
+    });
+    logger.debug({
+      res
+    }, 'sent email to custom vendor.');
+  } catch (err) {
+    logger.info({
+      err
+    }, 'Error sending email From Custom email vendor');
+  }
+};
+
+const sendEmailByMailgun = async(logger, from, to, subject, text) => {
   const mg = mailgun.client({
     username: 'api',
     key: process.env.MAILGUN_API_KEY
@@ -17,14 +56,18 @@ const emailSimpleText = async(logger, to, subject, text) => {
 
   try {
     const res = await mg.messages.create(process.env.MAILGUN_DOMAIN, {
-      from: 'jambonz Support <support@jambonz.org>',
+      from,
       to,
       subject,
       text
     });
-    logger.debug({res}, 'sent email');
+    logger.debug({
+      res
+    }, 'sent email');
   } catch (err) {
-    logger.info({err}, 'Error sending email');
+    logger.info({
+      err
+    }, 'Error sending email From mailgun');
   }
 };
 

lib/utils/encrypt-decrypt.js

@@ -2,9 +2,9 @@ const crypto = require('crypto');
 const algorithm = process.env.LEGACY_CRYPTO ? 'aes-256-ctr' : 'aes-256-cbc';
 const iv = crypto.randomBytes(16);
 const secretKey = crypto.createHash('sha256')
-  .update(String(process.env.JWT_SECRET))
+  .update(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET)
   .digest('base64')
-  .substr(0, 32);
+  .substring(0, 32);
 
 const encrypt = (text) => {
   const cipher = crypto.createCipheriv(algorithm, secretKey, iv);
@@ -23,7 +23,18 @@ const decrypt = (data) => {
   return decrpyted.toString();
 };
 
+const obscureKey = (key, key_spoiler_length = 6) => {
+  const key_spoiler_char = 'X';
+
+  if (!key || key.length <= key_spoiler_length) {
+    return key;
+  }
+
+  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
+};
+
 module.exports = {
   encrypt,
-  decrypt
+  decrypt,
+  obscureKey
 };

lib/utils/errors.js

@@ -1,3 +1,9 @@
+class BadRequestError extends Error {
+  constructor(msg) {
+    super(msg);
+  }
+}
+
 class DbError extends Error {
   constructor(msg) {
     super(msg);
@@ -23,6 +29,7 @@ class DbErrorForbidden extends DbError {
 }
 
 module.exports = {
+  BadRequestError,
   DbError,
   DbErrorBadRequest,
   DbErrorUnprocessableRequest,

lib/utils/homer-utils.js

@@ -39,11 +39,17 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
     const obj = await postJSON('/api/v3/call/transaction', {
       param: {
         transaction: {
-          call: true
+          call: true,
+          registration: true,
+          rest: false
         },
+        orlogic: true,
         search: {
           '1_call': {
             callid: [callId]
+          },
+          '1_registration': {
+            callid: [callId]
           }
         },
       },
@@ -58,7 +64,7 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
   }
 };
 
-const getHomerPcap = async(logger, apiKey, callIds) => {
+const getHomerPcap = async(logger, apiKey, callIds, method) => {
   if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
     logger.debug('getHomerPcap: Homer integration not installed');
   }
@@ -67,12 +73,23 @@ const getHomerPcap = async(logger, apiKey, callIds) => {
     const stream = await postPcap('/api/v3/export/call/messages/pcap', {
       param: {
         transaction: {
-          call: true
+          call: method === 'invite',
+          registration: method === 'register',
+          rest: false
         },
+        orlogic: true,
         search: {
-          '1_call': {
-            callid: callIds
-          }
+          ...(method === 'invite' && {
+            '1_call': {
+              callid: callIds
+            }
+          })
+          ,
+          ...(method === 'register' && {
+            '1_registration': {
+              callid: callIds
+            }
+          })
         },
       },
       timestamp: {

lib/utils/jaeger-utils.js

@@ -0,0 +1,19 @@
+const bent = require('bent');
+const getJSON = bent(process.env.JAEGER_BASE_URL || 'http://127.0.0.1', 'GET', 'json', 200);
+
+const getJaegerTrace = async(logger, traceId) => {
+  if (!process.env.JAEGER_BASE_URL) {
+    logger.debug('getJaegerTrace: jaeger integration not installed');
+    return null;
+  }
+  try {
+    return await getJSON(`/api/v3/traces/${traceId}`);
+  } catch (err) {
+    const url = `${process.env.JAEGER_BASE_URL}/api/traces/${traceId}`;
+    logger.error({err, traceId}, `getJaegerTrace: Error retrieving spans from ${url}`);
+  }
+};
+
+module.exports = {
+  getJaegerTrace
+};

lib/utils/jambonz-sample.text

@@ -0,0 +1 @@
+Hello From Jambonz. This file was created because Record all call bucket credential test.

lib/utils/password-utils.js

@@ -1,18 +1,19 @@
 const crypto = require('crypto');
-const { argon2i } = require('argon2-ffi');
+const argon2 = require('argon2');
 const util = require('util');
 
+const { argon2i } = argon2;
+
 const getRandomBytes = util.promisify(crypto.randomBytes);
 
 const generateHashedPassword = async(password) => {
   const salt = await getRandomBytes(32);
-  const passwordHash =  await argon2i.hash(password, salt);
+  const passwordHash = await argon2.hash(password, { type: argon2i, salt });
   return passwordHash;
 };
 
-const verifyPassword = async(passwordHash, password) => {
-  const isCorrect = await argon2i.verify(passwordHash, password);
-  return isCorrect;
+const verifyPassword = (passwordHash, password) => {
+  return argon2.verify(passwordHash, password);
 };
 
 const hashString = (s) => crypto.createHash('md5').update(s).digest('hex');

lib/utils/speech-utils.js

@@ -1,13 +1,41 @@
-const ttsGoogle = require('@google-cloud/text-to-speech');
 const sttGoogle = require('@google-cloud/speech').v1p1beta1;
-const Polly = require('aws-sdk/clients/polly');
-const AWS = require('aws-sdk');
+const { TranscribeClient, ListVocabulariesCommand } = require('@aws-sdk/client-transcribe');
+const { Deepgram } = require('@deepgram/sdk');
+const sdk = require('microsoft-cognitiveservices-speech-sdk');
+const { SpeechClient } = require('@soniox/soniox-node');
 const bent = require('bent');
 const fs = require('fs');
 
-const testGoogleTts = async(logger, credentials) => {
-  const client = new ttsGoogle.TextToSpeechClient({credentials});
-  await client.listVoices();
+
+const testSonioxStt = async(logger, credentials) => {
+  const api_key = credentials;
+  const soniox = new SpeechClient(api_key);
+
+  return new Promise(async(resolve, reject) => {
+    try {
+      const result = await soniox.transcribeFileShort('data/test_audio.wav');
+      if (result.words.length > 0) resolve(result);
+      else reject(new Error('no transcript returned'));
+    } catch (error) {
+      logger.info({error}, 'failed to get soniox transcript');
+      reject(error);
+    }
+  });
+};
+
+const testNuanceTts = async(logger, getTtsVoices, credentials) => {
+  const voices = await getTtsVoices({vendor: 'nuance', credentials});
+  return voices;
+};
+const testNuanceStt = async(logger, credentials) => {
+  //TODO
+  return true;
+};
+
+const testGoogleTts = async(logger, getTtsVoices, credentials) => {
+  const voices = await getTtsVoices({vendor: 'google', credentials});
+  return voices;
+
 };
 
 const testGoogleStt = async(logger, credentials) => {
@@ -32,30 +60,116 @@ const testGoogleStt = async(logger, credentials) => {
   }
 };
 
-const testAwsTts = (logger, credentials) => {
-  const polly = new Polly(credentials);
+const testDeepgramStt = async(logger, credentials) => {
+  const {api_key} = credentials;
+  const deepgram = new Deepgram(api_key);
+
+  const mimetype = 'audio/wav';
+  const source = {
+    buffer: fs.readFileSync(`${__dirname}/../../data/test_audio.wav`),
+    mimetype: mimetype
+  };
+
   return new Promise((resolve, reject) => {
-    polly.describeVoices({LanguageCode: 'en-US'}, (err, data) => {
-      if (err) return reject(err);
-      resolve();
+    // Send the audio to Deepgram and get the response
+    deepgram.transcription
+      .preRecorded(source, {punctuate: true})
+      .then((response) => {
+        //logger.debug({response}, 'got transcript');
+        if (response?.results?.channels[0]?.alternatives?.length > 0) resolve(response);
+        else reject(new Error('no transcript returned'));
+        return;
+      })
+      .catch((err) => {
+        logger.info({err}, 'failed to get deepgram transcript');
+        reject(err);
+      });
+  });
+};
+
+const testMicrosoftStt = async(logger, credentials) => {
+  const {api_key, region} = credentials;
+
+  const speechConfig = sdk.SpeechConfig.fromSubscription(api_key, region);
+  const audioConfig = sdk.AudioConfig.fromWavFileInput(fs.readFileSync(`${__dirname}/../../data/test_audio.wav`));
+  speechConfig.speechRecognitionLanguage = 'en-US';
+  const speechRecognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);
+
+  return new Promise((resolve, reject) => {
+    speechRecognizer.recognizeOnceAsync((result) => {
+      switch (result.reason) {
+        case sdk.ResultReason.RecognizedSpeech:
+          resolve();
+          break;
+        case sdk.ResultReason.NoMatch:
+          reject('Speech could not be recognized.');
+          break;
+        case sdk.ResultReason.Canceled:
+          const cancellation = sdk.CancellationDetails.fromResult(result);
+          logger.info(`CANCELED: Reason=${cancellation.reason}`);
+          if (cancellation.reason == sdk.CancellationReason.Error) {
+            logger.info(`CANCELED: ErrorCode=${cancellation.ErrorCode}`);
+            logger.info(`CANCELED: ErrorDetails=${cancellation.errorDetails}`);
+          }
+          reject(cancellation.reason);
+          break;
+      }
+      speechRecognizer.close();
     });
   });
 };
 
-const testAwsStt = (logger, credentials) => {
-  const transcribeservice = new AWS.TranscribeService(credentials);
-  return new Promise((resolve, reject) => {
-    transcribeservice.listVocabularies((err, data) => {
-      if (err) return reject(err);
-      logger.info({data}, 'retrieved language models');
-      resolve();
+const testAwsTts = async(logger, getTtsVoices, credentials) => {
+  try {
+    const voices = await getTtsVoices({vendor: 'aws', credentials});
+    return voices;
+  } catch (err) {
+    logger.info({err}, 'testMicrosoftTts - failed to list voices for region ${region}');
+    throw err;
+  }
+};
+
+const testAwsStt = async(logger, credentials) => {
+  try {
+    const {region, accessKeyId, secretAccessKey} = credentials;
+    const client = new TranscribeClient({
+      region,
+      credentials: {
+        accessKeyId,
+        secretAccessKey
+      }
     });
-  });
+    const command = new ListVocabulariesCommand({});
+    const response =  await client.send(command);
+    return response;
+  } catch (err) {
+    logger.info({err}, 'testMicrosoftTts - failed to list voices for region ${region}');
+    throw err;
+  }
 };
 
 const testMicrosoftTts = async(logger, credentials) => {
-  const {api_key, region} = credentials;
+  const {
+    api_key,
+    region,
+    // eslint-disable-next-line no-unused-vars
+    use_custom_tts,
+    // eslint-disable-next-line no-unused-vars
+    custom_tts_endpoint,
+    // eslint-disable-next-line no-unused-vars
+    use_custom_stt,
+    // eslint-disable-next-line no-unused-vars
+    custom_stt_endpoint
+  } = credentials;
 
+  logger.info({
+    api_key,
+    region,
+    use_custom_tts,
+    custom_tts_endpoint,
+    use_custom_stt,
+    custom_stt_endpoint
+  }, 'testing microsoft tts');
   if (!api_key) throw new Error('testMicrosoftTts: credentials are missing api_key');
   if (!region) throw new Error('testMicrosoftTts: credentials are missing region');
   try {
@@ -70,11 +184,6 @@ const testMicrosoftTts = async(logger, credentials) => {
   }
 };
 
-const testMicrosoftStt = async(logger, credentials) => {
-  //TODO
-  return true;
-};
-
 const testWellSaidTts = async(logger, credentials) => {
   const {api_key} = credentials;
   try {
@@ -94,6 +203,35 @@ const testWellSaidTts = async(logger, credentials) => {
   }
 };
 
+const testIbmTts = async(logger, getTtsVoices, credentials) => {
+  const {tts_api_key, tts_region} = credentials;
+  const voices = await getTtsVoices({vendor: 'ibm', credentials: {tts_api_key, tts_region}});
+  return voices;
+};
+
+const testIbmStt = async(logger, credentials) => {
+  const {stt_api_key, stt_region} = credentials;
+  const SpeechToTextV1 = require('ibm-watson/speech-to-text/v1');
+  const { IamAuthenticator } = require('ibm-watson/auth');
+  const speechToText = new SpeechToTextV1({
+    authenticator: new IamAuthenticator({
+      apikey: stt_api_key
+    }),
+    serviceUrl: `https://api.${stt_region}.speech-to-text.watson.cloud.ibm.com`
+  });
+  return new Promise((resolve, reject) => {
+    speechToText.listModels()
+      .then((speechModels) => {
+        logger.debug({speechModels}, 'got IBM speech models');
+        return resolve();
+      })
+      .catch((err) => {
+        logger.info({err}, 'failed to get speech models');
+        reject(err);
+      });
+  });
+};
+
 const testWellSaidStt = async(logger, credentials) => {
   //TODO
   return true;
@@ -108,4 +246,10 @@ module.exports = {
   testMicrosoftTts,
   testMicrosoftStt,
   testWellSaidStt,
+  testNuanceTts,
+  testNuanceStt,
+  testDeepgramStt,
+  testIbmTts,
+  testIbmStt,
+  testSonioxStt
 };

lib/utils/storage-utils.js

@@ -0,0 +1,82 @@
+const { S3Client, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3');
+const {Storage} = require('@google-cloud/storage');
+const fs = require('fs');
+
+function testGoogleStorage(logger, opts) {
+  return new Promise((resolve, reject) => {
+    const serviceKey = JSON.parse(opts.service_key);
+    const storage = new Storage({
+      projectId: serviceKey.project_id,
+      credentials: {
+        client_email: serviceKey.client_email,
+        private_key: serviceKey.private_key
+      },
+    });
+
+    const blob = storage.bucket(opts.name).file('jambonz-sample.text');
+
+    fs.createReadStream(`${__dirname}/jambonz-sample.text`)
+      .pipe(blob.createWriteStream())
+      .on('error', (err) => reject(err))
+      .on('finish', () => resolve());
+  });
+}
+
+async function getGoogleStorageObject(logger, opts) {
+  const serviceKey = JSON.parse(opts.service_key);
+  const storage = new Storage({
+    projectId: serviceKey.project_id,
+    credentials: {
+      client_email: serviceKey.client_email,
+      private_key: serviceKey.private_key
+    },
+  });
+
+  const bucket = storage.bucket(opts.name);
+  const file = bucket.file(opts.key);
+
+  return file.createReadStream();
+}
+
+async function testAwsS3(logger, opts) {
+  const s3 = new S3Client({
+    credentials: {
+      accessKeyId: opts.access_key_id,
+      secretAccessKey: opts.secret_access_key,
+    },
+    region: opts.region || 'us-east-1'
+  });
+
+  const input = {
+    'Body': 'Hello From Jambonz',
+    'Bucket': opts.name,
+    'Key': 'jambonz-sample.text'
+  };
+
+  const command = new PutObjectCommand(input);
+
+  await s3.send(command);
+}
+
+async function getS3Object(logger, opts) {
+  const s3 = new S3Client({
+    credentials: {
+      accessKeyId: opts.access_key_id,
+      secretAccessKey: opts.secret_access_key,
+    },
+    region: opts.region || 'us-east-1'
+  });
+  const command = new GetObjectCommand({
+    Bucket: opts.name,
+    Key: opts.key
+  });
+  const res = await s3.send(command);
+  return res.Body;
+}
+
+module.exports = {
+  testAwsS3,
+  getS3Object,
+  testGoogleStorage,
+  getGoogleStorageObject
+};

package.json

@@ -1,15 +1,16 @@
 {
   "name": "jambonz-api-server",
-  "version": "v0.7.5",
+  "version": "0.8.4",
   "description": "",
   "main": "app.js",
   "scripts": {
     "start": "node app.js",
-    "test": "NODE_ENV=test APPLY_JAMBONZ_DB_LIMITS=1 JWT_SECRET=foobarbazzle JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=localhost JAMBONES_REDIS_PORT=16379 JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_LOGLEVEL=error JAMBONES_CREATE_CALL_URL=http://localhost/v1/createCall node test/ ",
-    "integration-test": "NODE_ENV=test JAMBONES_TIME_SERIES_HOST=127.0.0.1 AWS_REGION='us-east-1' JAMBONES_CURRENCY=USD JWT_SECRET=foobarbazzle JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=localhost JAMBONES_REDIS_PORT=16379 JAMBONES_LOGLEVEL=debug JAMBONES_CREATE_CALL_URL=http://localhost/v1/createCall node test/serve-integration.js",
+    "test": "NODE_ENV=test APPLY_JAMBONZ_DB_LIMITS=1 JWT_SECRET=foobarbazzle JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=localhost JAMBONES_REDIS_PORT=16379 JAMBONES_TIME_SERIES_HOST=127.0.0.1 JAMBONES_LOGLEVEL=error JAMBONES_CREATE_CALL_URL=http://localhost/v1/createCall K8S=true K8S_FEATURE_SERVER_SERVICE_NAME=127.0.0.1 K8S_FEATURE_SERVER_SERVICE_PORT=3100 node test/ ",
+    "integration-test": "NODE_ENV=test JAMBONES_AUTH_USE_JWT=1 JAMBONES_TIME_SERIES_HOST=127.0.0.1 AWS_REGION='us-east-1' JAMBONES_CURRENCY=USD JWT_SECRET=foobarbazzle JAMBONES_MYSQL_HOST=127.0.0.1 JAMBONES_MYSQL_PORT=3360 JAMBONES_MYSQL_USER=jambones_test JAMBONES_MYSQL_PASSWORD=jambones_test JAMBONES_MYSQL_DATABASE=jambones_test JAMBONES_REDIS_HOST=localhost JAMBONES_REDIS_PORT=16379 JAMBONES_LOGLEVEL=debug JAMBONES_CREATE_CALL_URL=http://localhost/v1/createCall node test/serve-integration.js",
     "upgrade-db": "node ./db/upgrade-jambonz-db.js",
     "coverage": "./node_modules/.bin/nyc --reporter html --report-dir ./coverage npm run test",
     "jslint": "eslint app.js lib",
+    "jslint:fix": "eslint app.js lib --fix",
     "prepare": "husky install"
   },
   "author": "Dave Horton",
@@ -18,42 +19,50 @@
     "url": "https://github.com/jambonz/jambonz-api-server.git"
   },
   "dependencies": {
-    "@google-cloud/speech": "^4.10.0",
-    "@google-cloud/text-to-speech": "^3.4.0",
-    "@jambonz/db-helpers": "^0.6.16",
-    "@jambonz/realtimedb-helpers": "^0.4.24",
-    "@jambonz/time-series": "^0.1.6",
-    "argon2-ffi": "^2.0.0",
-    "aws-sdk": "^2.1073.0",
+    "@aws-sdk/client-transcribe": "^3.363.0",
+    "@aws-sdk/client-s3": "^3.363.0",
+    "@deepgram/sdk": "^1.21.0",
+    "@google-cloud/speech": "^5.2.0",
+    "@jambonz/db-helpers": "^0.9.0",
+    "@jambonz/realtimedb-helpers": "^0.8.6",
+    "@jambonz/speech-utils": "^0.0.15",
+    "@jambonz/time-series": "^0.2.8",
+    "@jambonz/verb-specifications": "^0.0.26",
+    "@jambonz/lamejs": "^1.2.2",
+    "@soniox/soniox-node": "^1.1.1",
+    "argon2": "^0.30.3",
     "bent": "^7.3.12",
     "cors": "^2.8.5",
-    "debug": "^4.3.3",
-    "express": "^4.17.2",
-    "express-rate-limit": "^6.3.0",
+    "debug": "^4.3.4",
+    "express": "^4.18.1",
+    "express-rate-limit": "^6.4.0",
     "form-data": "^2.5.1",
-    "form-urlencoded": "^6.0.5",
-    "helmet": "^5.0.2",
-    "jsonwebtoken": "^8.5.1",
+    "helmet": "^5.1.0",
+    "ibm-watson": "^7.1.2",
+    "jsonwebtoken": "^9.0.0",
     "mailgun.js": "^3.7.3",
-    "microsoft-cognitiveservices-speech-sdk": "^1.19.0",
+    "microsoft-cognitiveservices-speech-sdk": "^1.24.1",
     "mysql2": "^2.3.3",
-    "passport": "^0.5.2",
+    "nocache": "3.0.4",
+    "passport": "^0.6.0",
     "passport-http-bearer": "^1.0.1",
     "pino": "^5.17.0",
-    "request-debug": "^0.2.0",
     "short-uuid": "^4.1.0",
-    "stripe": "^8.196.0",
-    "swagger-ui-express": "^4.3.0",
+    "stripe": "^8.222.0",
+    "swagger-ui-express": "^4.4.0",
     "uuid": "^8.3.2",
-    "yamljs": "^0.3.0"
+    "yamljs": "^0.3.0",
+    "ws": "^8.12.1",
+    "wav": "^1.0.2",
+    "@google-cloud/storage" : "^6.12.0"
   },
   "devDependencies": {
-    "eslint": "^7.17.0",
-    "eslint-plugin-promise": "^4.2.1",
+    "eslint": "^8.39.0",
+    "eslint-plugin-promise": "^6.1.1",
     "husky": "7.0.4",
     "nyc": "^15.1.0",
     "request": "^2.88.2",
     "request-promise-native": "^1.0.9",
-    "tape": "^5.2.2"
+    "tape": "^5.5.3"
   }
 }

test/accounts.js

@@ -1,6 +1,10 @@
 const test = require('tape') ;
 const ADMIN_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734de';
 const authAdmin = {bearer: ADMIN_TOKEN};
+const SP_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734ds';
+const authSP = {bearer: ADMIN_TOKEN};
+const ACC_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734da';
+const authAcc = {bearer: ADMIN_TOKEN};
 const request = require('request-promise-native').defaults({
   baseUrl: 'http://127.0.0.1:3000/v1'
 });
@@ -9,6 +13,12 @@ const {
   createServiceProvider, 
   createPhoneNumber, 
   deleteObjectBySid} = require('./utils');
+const logger = require('../lib/logger');
+const { addToSortedSet } = require('@jambonz/realtimedb-helpers')({
+  host: process.env.JAMBONES_REDIS_HOST,
+  port: process.env.JAMBONES_REDIS_PORT || 6379
+}, logger);
+
 
 process.on('unhandledRejection', (reason, p) => {
   console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
@@ -158,11 +168,33 @@ test('account tests', async(t) => {
         queue_event_hook: {
           url: 'http://example.com/q',
           method: 'post'
+        },
+        record_all_calls: true,
+        record_format: 'wav',
+        bucket_credential: {
+          vendor: 'aws_s3',
+          region: 'us-east-1',
+          name: 'recordings',
+          access_key_id: 'access_key_id',
+          secret_access_key: 'secret access key'
         }
+        
       }
     });
     t.ok(result.statusCode === 204, 'successfully updated account using account level token');
 
+    /* verify that bucket credential is updated*/
+    result = await request.get(`/Accounts/${sid}`, {
+      auth: {bearer: accountLevelToken},
+      json: true,
+    });
+
+    t.ok(result.bucket_credential.vendor === 'aws_s3', 'bucket_vendor was updated');
+    t.ok(result.bucket_credential.name === 'recordings', 'bucket_name was updated');
+    t.ok(result.bucket_credential.access_key_id === 'access_key_id', 'bucket_access_key_id was updated');
+    t.ok(result.record_all_calls === 1, 'record_all_calls was updated');
+    t.ok(result.record_format === 'wav', 'record_format was updated');
+
     /* verify that account level api key last_used was updated*/
     result = await request.get(`/Accounts/${sid}/ApiKeys`, {
       auth: {bearer: accountLevelToken},
@@ -188,6 +220,100 @@ test('account tests', async(t) => {
     });
     t.ok(result.statusCode === 204, 'successfully assigned phone number to account');
 
+    /* query all limits for an account */
+    result = await request.get(`/Accounts/${sid}/Limits`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 0, 'successfully queried account limits when there is none configured');
+    
+    /* add a new limit for a account */
+    result = await request.post(`/Accounts/${sid}/Limits`, {
+      auth: authAdmin,
+      json: true,
+      resolveWithFullResponse: true,
+      body: {
+        category: 'voice_call_session',
+        quantity: 200
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully added a call session limit to an account');
+    
+    /* update an existing limit for a account */
+    result = await request.post(`/Accounts/${sid}/Limits`, {
+      auth: authAdmin,
+      json: true,
+      resolveWithFullResponse: true,
+      body: {
+        category: 'voice_call_session',
+        quantity: 205
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully updated a call session limit to an account');
+
+    /* try to update an existing limit for an account giving a invalid sid */
+    try {
+      result = await request.post(`/Accounts/invalid-sid/Limits`, {
+        auth: authAdmin,
+        json: true,
+        resolveWithFullResponse: true,
+        body: {
+          category: 'voice_call_session',
+          quantity: 205
+        }
+      });
+    } catch (err) {
+      t.ok(err.statusCode === 400, 'returns 400 bad request if account sid param is not a valid uuid');
+    }
+
+    /* query all limits for an account */
+    result = await request.get(`/Accounts/${sid}/Limits`, {
+      auth: authAdmin,
+      json: true,
+    });
+    //console.log(result);
+    t.ok(result.length === 1 && result[0].quantity === 205, 'successfully queried account limits');
+
+    /* query all limits for an account by category*/
+    result = await request.get(`/Accounts/${sid}/Limits?category=voice_call_session`, {
+      auth: authAdmin,
+      json: true,
+    });
+    // console.log(result);
+    t.ok(result.length === 1  && result[0].quantity === 205, 'successfully queried account limits by category');
+
+    /* delete call session limits for a service provider */
+    result = await request.delete(`/Accounts/${sid}/Limits?category=voice_call_session`, {
+      auth: authAdmin,
+      resolveWithFullResponse: true
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted a call session limit for an account');
+
+    /* query account queues */
+    await addToSortedSet(`queue:${sid}:test`, 'url1');
+    await addToSortedSet(`queue:${sid}:dummy`, 'url2');
+
+    result = await request.get(`/Accounts/${sid}/Queues`, {
+      auth: authAdmin,
+      resolveWithFullResponse: true,
+      json: true,
+    });
+    t.ok(result.statusCode === 200 && result.body.length === 2, 'successfully queried account queues info for an account');
+
+    result = await request.get(`/Accounts/${sid}/Queues?search=test`, {
+      auth: authAdmin,
+      resolveWithFullResponse: true,
+      json: true,
+    });
+    t.ok(result.statusCode === 200 && result.body.length === 1, 'successfully queried account queue info with search for an account');
+
+    result = await request.get(`/Accounts/29d41725-9d3a-4f89-9f0b-f32b3e4d3159/Queues`, {
+      auth: authAdmin,
+      resolveWithFullResponse: true,
+      json: true,
+    });
+    t.ok(result.statusCode === 200 && result.body.length === 0, 'successfully queried account queue info with for an invalid account');
+
     /* delete account */
     result = await request.delete(`/Accounts/${sid}`, {
       auth: authAdmin,

test/applications.js

@@ -23,6 +23,37 @@ test('application tests', async(t) => {
     const service_provider_sid = await createServiceProvider(request);
     const phone_number_sid = await createPhoneNumber(request, voip_carrier_sid);
     const account_sid = await createAccount(request, service_provider_sid);
+
+    /* add an invalid application app_json */
+    result = await request.post('/Applications', {
+      resolveWithFullResponse: true,
+      simple: false,
+      auth: authAdmin,
+      json: true,
+      body: {
+        name: 'daveh',
+        account_sid,
+        call_hook: {
+          url: 'http://example.com'
+        },
+        call_status_hook: {
+          url: 'http://example.com/status',
+          method: 'POST'
+        },
+        messaging_hook: {
+          url: 'http://example.com/sms'
+        },
+        app_json : '[\
+            {\
+              "verb": "play",\
+              "timeoutSecs": 10,\
+              "seekOffset": 8000,\
+              "actionHook": "/play/action"\
+          }\
+        ]'
+      }
+    });
+    t.ok(result.statusCode === 400, 'Cant create application with invalid app_josn');
     
     /* add an application */
     result = await request.post('/Applications', {
@@ -41,7 +72,16 @@ test('application tests', async(t) => {
         },
         messaging_hook: {
           url: 'http://example.com/sms'
-        }
+        },
+        app_json : '[\
+            {\
+              "verb": "play",\
+              "url": "https://example.com/example.mp3",\
+              "timeoutSecs": 10,\
+              "seekOffset": 8000,\
+              "actionHook": "/play/action"\
+          }\
+        ]'
       }
     });
     t.ok(result.statusCode === 201, 'successfully created application');
@@ -60,8 +100,11 @@ test('application tests', async(t) => {
       auth: authAdmin,
       json: true,
     });
-    t.ok(result[0].name === 'daveh' , 'successfully retrieved application by sid');
-    t.ok(result[0].messaging_hook.url === 'http://example.com/sms' , 'successfully retrieved messaging_hook from application');
+    t.ok(result.name === 'daveh' , 'successfully retrieved application by sid');
+    t.ok(result.messaging_hook.url === 'http://example.com/sms' , 'successfully retrieved messaging_hook from application');
+    let app_json = JSON.parse(result.app_json);
+    t.ok(app_json[0].verb === 'play', 'successfully retrieved app_json from application')
+
 
     /* update applications */
     result = await request.put(`/Applications/${sid}`, {
@@ -74,7 +117,16 @@ test('application tests', async(t) => {
         },
         messaging_hook: {
           url: 'http://example2.com/mms'
-        }
+        },
+        app_json : '[\
+          {\
+            "verb": "hangup",\
+            "headers": {\
+              "X-Reason" : "maximum call duration exceeded"\
+            }\
+          }\
+        ]',
+        record_all_calls: true
       }
     });
     t.ok(result.statusCode === 204, 'successfully updated application');
@@ -84,7 +136,59 @@ test('application tests', async(t) => {
       auth: authAdmin,
       json: true,
     });
-    t.ok(result[0].messaging_hook.url === 'http://example2.com/mms' , 'successfully updated messaging_hook');
+    t.ok(result.messaging_hook.url === 'http://example2.com/mms' , 'successfully updated messaging_hook');
+    app_json = JSON.parse(result.app_json);
+    t.ok(app_json[0].verb === 'hangup', 'successfully updated app_json from application')
+    t.ok(result.record_all_calls === 1, 'successfully updated record_all_calls from application')
+
+    /* remove applications app_json*/
+    result = await request.put(`/Applications/${sid}`, {
+      auth: authAdmin,
+      json: true,
+      resolveWithFullResponse: true,
+      body: {
+        call_hook: {
+          url: 'http://example2.com'
+        },
+        messaging_hook: {
+          url: 'http://example2.com/mms'
+        },
+        app_json : null
+      }
+    });
+    t.ok(result.statusCode === 204, 'successfully updated application');
+
+    /* validate messaging hook was updated */
+    result = await request.get(`/Applications/${sid}`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.app_json == undefined, 'successfully removed app_json from application')
+
+    /* Update invalid applications app_json*/
+    result = await request.put(`/Applications/${sid}`, {
+      auth: authAdmin,
+      json: true,
+      resolveWithFullResponse: true,
+      simple: false,
+      body: {
+        call_hook: {
+          url: 'http://example2.com'
+        },
+        messaging_hook: {
+          url: 'http://example2.com/mms'
+        },
+        app_json : '[\
+          {\
+            "verb": "play",\
+            "timeoutSecs": 10,\
+            "seekOffset": 8000,\
+            "actionHook": "/play/action"\
+        }\
+      ]'
+      }
+    });
+    t.ok(result.statusCode === 400, 'Cant update invalid application app_json');
     
     /* assign phone number to application */
     result = await request.put(`/PhoneNumbers/${phone_number_sid}`, {

test/auth.js

@@ -127,7 +127,7 @@ test('authentication tests', async(t) => {
         sip_realm: 'sip.foo.bar'
       }
     });
-    t.ok(result.statusCode === 422 && result.body.msg === 'cannot update account from different service provider',
+    t.ok(result.statusCode === 403 && result.body.msg === 'insufficient permissions',
       'service provider token B cannot be used to update account from service provider A');
 
     /* cannot delete account from different service provider */
@@ -137,7 +137,7 @@ test('authentication tests', async(t) => {
       simple: false,
       json: true,
     });
-    t.ok(result.statusCode === 422 && result.body.msg === 'cannot delete account from different service provider',
+    t.ok(result.statusCode === 403 && result.body.msg === 'insufficient permissions',
       'service provider token B cannot be used to delete account from service provider A');
 
     /* service provider token A can update account A1 */
@@ -179,7 +179,7 @@ test('authentication tests', async(t) => {
       }
     });
     //console.log(`result: ${JSON.stringify(result)}`);
-    t.ok(result.statusCode === 422 && result.body.msg === 'insufficient permissions to create accounts',
+    t.ok(result.statusCode === 403 && result.body.msg === 'insufficient permissions',
       'cannot create an account using an account-level token');
 
     /* using account token we see one account */
@@ -200,8 +200,7 @@ test('authentication tests', async(t) => {
         sip_realm: 'sip.foo.bar'
       }
     });
-    //console.log(`result: ${JSON.stringify(result)}`);
-    t.ok(result.statusCode === 422 && result.body.msg === 'insufficient privileges to update this account',
+    t.ok(result.statusCode === 403 && result.body.msg === 'insufficient permissions',
       'cannot update account A2 using auth token for account A1');
 
     /* can update an account using an appropriate account-level token */
@@ -251,7 +250,8 @@ test('authentication tests', async(t) => {
         }
       }
     });
-    t.ok(result.statusCode === 400 && result.body.msg === 'insufficient privileges to create an application under the specified account',
+    //console.log(`result: ${JSON.stringify(result)}`);
+    t.ok(result.statusCode === 403 && result.body.msg === 'insufficient privileges',
       'cannot create application for account A2 using service provider token B');
 
     result = await request.post('/Applications', {
@@ -318,7 +318,7 @@ test('authentication tests', async(t) => {
       json: true
     });
     //console.log(`result: ${JSON.stringify(result)}`);
-    t.ok(result.length === 1, 'using account token A1 we are able to retrieve application A1');
+    t.ok(result.name === "A1-app", 'using account token A1 we are able to retrieve application A1');
 
     /* cannot see app under another account using account token */
     result = await request.get(`/Applications/${appA2}`, {

test/call-test.js

@@ -0,0 +1,156 @@
+const test = require('tape');
+const jwt = require('jsonwebtoken');
+const request = require('request-promise-native').defaults({
+  baseUrl: 'http://127.0.0.1:3000/v1'
+});
+const consoleLogger = { debug: console.log, info: console.log, error: console.error }
+const {
+  createServiceProvider,
+  createAccount,
+  createGoogleSpeechCredentials,
+  getLastRequestFromFeatureServer
+} = require('./utils');
+
+test('Create Call Success With Synthesizer in Payload', async (t) => {
+  // GIVEN
+  const app = require('../app');
+  let result;
+  const service_provider_sid = await createServiceProvider(request, 'account_has_synthesizer');
+  const account_sid = await createAccount(request, service_provider_sid, 'account_has_synthesizer');
+  const token = jwt.sign({
+    account_sid,
+    scope: "account",
+    permissions: ["PROVISION_USERS", "PROVISION_SERVICES", "VIEW_ONLY"]
+  }, process.env.JWT_SECRET, { expiresIn: '1h' });
+  const authUser = { bearer: token };
+  const speech_sid = await createGoogleSpeechCredentials(request, account_sid, null, authUser, true, true)
+
+  // WHEN
+  result = await request.post(`/Accounts/${account_sid}/Calls`, {
+    resolveWithFullResponse: true,
+    auth: authUser,
+    json: true,
+    body: {
+      call_hook: "https://public-apps.jambonz.us/hello-world",
+      call_status_hook: "https://public-apps.jambonz.us/call-status",
+      from: "15083778299",
+      to: {
+        type: "phone",
+        number: "15089084809"
+      },
+      speech_synthesis_vendor: "google",
+      speech_synthesis_language: "en-US",
+      speech_synthesis_voice: "en-US-Standard-C",
+      speech_recognizer_vendor: "google",
+      speech_recognizer_language: "en-US"
+    }
+  });
+  // THEN
+  t.ok(result.statusCode === 201, 'successfully created Call without Synthesizer && application_sid');
+  const fs_request = await getLastRequestFromFeatureServer('15083778299_createCall');
+  const obj = JSON.parse(fs_request);
+  t.ok(obj.body.speech_synthesis_vendor == 'google', 'speech synthesizer successfully added')
+  t.ok(obj.body.speech_recognizer_vendor == 'google', 'speech recognizer successfully added')
+});
+
+test('Create Call Success Without Synthesizer in Payload', async (t) => {
+  // GIVEN
+  const app = require('../app');
+  let result;
+  const service_provider_sid = await createServiceProvider(request, 'account2_has_synthesizer');
+  const account_sid = await createAccount(request, service_provider_sid, 'account2_has_synthesizer');
+  const token = jwt.sign({
+    account_sid,
+    scope: "account",
+    permissions: ["PROVISION_USERS", "PROVISION_SERVICES", "VIEW_ONLY"]
+  }, process.env.JWT_SECRET, { expiresIn: '1h' });
+  const authUser = { bearer: token };
+  const speech_sid = await createGoogleSpeechCredentials(request, account_sid, null, authUser, true, true)
+
+  // WHEN
+  result = await request.post(`/Accounts/${account_sid}/Calls`, {
+    resolveWithFullResponse: true,
+    auth: authUser,
+    json: true,
+    body: {
+      call_hook: "https://public-apps.jambonz.us/hello-world",
+      call_status_hook: "https://public-apps.jambonz.us/call-status",
+      from: "15083778299",
+      to: {
+        type: "phone",
+        number: "15089084809"
+      }
+    }
+  }).then(data => { t.ok(false, 'Create Call should not be success') })
+    .catch(error => { t.ok(error.response.statusCode === 400, 'Call failed for no synthesizer') });
+});
+
+test("Create call with application sid and app_json", async(t) => {
+  const app = require('../app');
+
+  const service_provider_sid = await createServiceProvider(request, 'account3_has_synthesizer');
+  const account_sid = await createAccount(request, service_provider_sid, 'account3_has_synthesizer');
+
+  const token = jwt.sign({
+    account_sid,
+    scope: "account",
+    permissions: ["PROVISION_USERS", "PROVISION_SERVICES", "VIEW_ONLY"]
+  }, process.env.JWT_SECRET, { expiresIn: '1h' });
+  const authUser = { bearer: token };
+  const speech_sid = await createGoogleSpeechCredentials(request, account_sid, null, authUser, true, true);
+
+  // GIVEN
+/* add an application */
+
+const app_json = '[\
+  {\
+    "verb": "play",\
+    "url": "https://example.com/example.mp3",\
+    "timeoutSecs": 10,\
+    "seekOffset": 8000,\
+    "actionHook": "/play/action"\
+}\
+]';
+let result = await request.post('/Applications', {
+  resolveWithFullResponse: true,
+  auth: authUser,
+  json: true,
+  body: {
+    name: 'daveh',
+    account_sid,
+    call_hook: {
+      url: 'http://example.com'
+    },
+    call_status_hook: {
+      url: 'http://example.com/status',
+      method: 'POST'
+    },
+    messaging_hook: {
+      url: 'http://example.com/sms'
+    },
+    app_json
+  }
+});
+t.ok(result.statusCode === 201, 'successfully created application');
+const sid = result.body.sid;
+
+// WHEN
+result = await request.post(`/Accounts/${account_sid}/Calls`, {
+  resolveWithFullResponse: true,
+  auth: authUser,
+  json: true,
+  body: {
+    application_sid: sid,
+    from: "15083778299",
+    to: {
+      type: "phone",
+      number: "15089084809"
+    },
+  }
+});
+// THEN
+t.ok(result.statusCode === 201, 'successfully created Call without Synthesizer && application_sid');
+const fs_request = await getLastRequestFromFeatureServer('15083778299_createCall');
+const obj = JSON.parse(fs_request);
+t.ok(obj.body.app_json == app_json, 'app_json successfully added')
+});

test/clients.js

@@ -0,0 +1,119 @@
+const test = require('tape') ;
+const ADMIN_TOKEN = '38700987-c7a4-4685-a5bb-af378f9734de';
+const authAdmin = {bearer: ADMIN_TOKEN};
+const request = require('request-promise-native').defaults({
+  baseUrl: 'http://127.0.0.1:3000/v1'
+});
+
+
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason);
+});
+
+test('client test', async(t) => {
+  const app = require('../app');
+
+  try {
+    let result;
+    /* add a service provider */
+    result = await request.post('/ServiceProviders', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        name: 'client_sp',
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created client service provider');
+    const sp_sid = result.body.sid;
+
+    /* add an account */
+    result = await request.post('/Accounts', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        name: 'sample_account',
+        service_provider_sid: sp_sid,
+        registration_hook: {
+          url: 'http://example.com/reg',
+          method: 'get'
+        },
+        webhook_secret: 'foobar'
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created account');
+    const account_sid = result.body.sid;
+
+    /* add new entity */
+    result = await request.post('/Clients', {
+      resolveWithFullResponse: true,
+      auth: authAdmin,
+      json: true,
+      body: {
+        account_sid,
+        username: 'client1',
+        password: 'sdf12412',
+        is_active: 1
+      }
+    });
+    t.ok(result.statusCode === 201, 'successfully created Client');
+    const sid = result.body.sid;
+
+    /* query all entity */
+    result = await request.get('/Clients', {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 1 , 'successfully queried all Clients');
+
+    /* query one entity */
+    result = await request.get(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.account_sid === account_sid , 'successfully retrieved Client by sid');
+    t.ok(result.client_sid, 'successfully retrieved Client by sid');
+    t.ok(result.username ===  'client1', 'successfully retrieved Client by sid');
+    t.ok(result.is_active === 1 , 'successfully retrieved Client by sid');
+    t.ok(result.password === 'sXXXXXXX' , 'successfully retrieved Client by sid');
+
+    /* update the entity */
+    result = await request.put(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+      resolveWithFullResponse: true,
+      body: {
+        is_active: 0
+      }
+    });
+    t.ok(result.statusCode === 204, 'successfully updated Client');
+    /* query one entity */
+    result = await request.get(`/Clients/${sid}`, {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.is_active === 0 , 'successfully updated Client');
+    t.ok(result.password === 'sXXXXXXX' , 'successfully retrieved Client by sid');
+
+    /* delete Client */
+    result = await request.delete(`/Clients/${sid}`, {
+      resolveWithFullResponse: true,
+      simple: false,
+      json: true,
+      auth: authAdmin
+    });
+    t.ok(result.statusCode === 204, 'successfully deleted Clients');
+
+    /* query all entity */
+    result = await request.get('/Clients', {
+      auth: authAdmin,
+      json: true,
+    });
+    t.ok(result.length === 0 , 'successfully queried all Clients');
+    
+  } catch (err) {
+    console.error(err);
+    t.end(err);
+  }
+})

test/docker-compose-testbed.yaml

@@ -46,7 +46,6 @@ services:
   
   db:
     image: postgres:11-alpine
-    restart: always
     environment:
       POSTGRES_PASSWORD: homerSeven
       POSTGRES_USER: root
@@ -127,6 +126,21 @@ services:
     depends_on:
       db:
         condition: service_healthy
-          
-      
-      
+  
+  feature-server-test-scaffold:
+    image: jambonz/feature-server-test-scaffold:latest
+    ports:
+      - "3100:3000/tcp"
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.9
+
+  webhook-tts-scaffold:
+    image: jambonz/webhook-tts-test-scaffold:latest
+    ports:
+      - "3101:3000/tcp"
+    volumes:
+      - ./test-apps:/tmp
+    networks:
+      jambonz-api:
+        ipv4_address: 172.58.0.10

test/email_utils.js

@@ -0,0 +1,29 @@
+const test = require('tape');
+const {emailSimpleText} = require('../lib/utils/email-utils');
+const bent = require('bent');
+const getJSON = bent('json')
+const logger = {
+  debug: () =>{},
+  info: () => {}
+}
+
+test('email-test', async(t) => {
+  // Prepare env:
+  process.env.CUSTOM_EMAIL_VENDOR_URL = 'http://127.0.0.1:3101/custom_email_vendor';
+  process.env.CUSTOM_EMAIL_VENDOR_USERNAME = 'USERNAME';
+  process.env.CUSTOM_EMAIL_VENDOR_PASSWORD = 'PASSWORD';
+
+  await emailSimpleText(logger, 'test@gmail.com', 'subject', 'body text');
+
+  const obj = await getJSON(`http://127.0.0.1:3101/lastRequest/custom_email_vendor`);
+  t.ok(obj.headers['Content-Type'] == 'application/json');
+  t.ok(obj.headers.Authorization == 'Basic VVNFUk5BTUU6UEFTU1dPUkQ=');
+  t.ok(obj.body.from == 'jambonz Support <support@jambonz.org>');
+  t.ok(obj.body.to == 'test@gmail.com');
+  t.ok(obj.body.subject == 'subject');
+  t.ok(obj.body.text == 'body text');
+
+  process.env.CUSTOM_EMAIL_VENDOR_URL = null;
+  process.env.CUSTOM_EMAIL_VENDOR_USERNAME = null;
+  process.env.CUSTOM_EMAIL_VENDOR_PASSWORD = null;
+});

test/feature-server-test-scaffold/Dockerfile

@@ -0,0 +1,23 @@
+FROM --platform=linux/amd64 node:18.6.0-alpine as base
+
+RUN apk --update --no-cache add --virtual .builds-deps build-base python3
+
+WORKDIR /opt/app/
+
+FROM base as build
+
+COPY package.json package-lock.json ./
+
+RUN npm ci
+
+COPY . .
+
+FROM base
+
+COPY --from=build /opt/app /opt/app/
+
+ARG NODE_ENV
+
+ENV NODE_ENV $NODE_ENV
+
+CMD [ "node", "app.js" ]