fix jambones-sql.sql missing FOREIGN_KEY_CHECKS

* allow sip port is null

* update upgrade script

* fix review comment

.dockerignore

@@ -0,0 +1 @@
+node_modules

.eslintrc.json

@@ -8,7 +8,7 @@
       "jsx": false,
       "modules": false
     },
-    "ecmaVersion": 2018
+    "ecmaVersion": 2020
   },
   "plugins": ["promise"],
   "rules": {

.github/workflows/ci.yml

@@ -1,17 +1,15 @@
 name: CI
 
-on:
-  push:
-  workflow_dispatch:
+on: [push, pull_request, workflow_dispatch]
 
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: lts/*
       - run: npm install
       - run: npm run jslint
       - run: npm test

.github/workflows/docker-publish-dbcreate.yml

@@ -0,0 +1,55 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+
+jobs:
+  push:
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: prepare tag 
+        id: prepare_tag
+        run: |
+            IMAGE_ID=jambonz/db-create
+
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
+
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
+
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.db-create
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.github/workflows/docker-publish.yml

@@ -0,0 +1,54 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'
+
+jobs:
+  push:
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: prepare tag 
+        id: prepare_tag
+        run: |
+            IMAGE_ID=jambonz/api-server
+
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "main" ] && VERSION=latest
+
+            echo IMAGE_ID=$IMAGE_ID
+            echo VERSION=$VERSION
+
+            echo "image_id=$IMAGE_ID" >> $GITHUB_OUTPUT
+            echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Login to Docker Hub 
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.prepare_tag.outputs.image_id }}:${{ steps.prepare_tag.outputs.version }}
+          build-args: |
+            GITHUB_REPOSITORY=$GITHUB_REPOSITORY
+            GITHUB_REF=$GITHUB_REF

.gitignore

@@ -1,7 +1,7 @@
 # Logs
 logs
 *.log
-package-lock.json
+run-tests.sh
 
 # Runtime data
 pids

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+npm run jslint

Dockerfile

@@ -1,16 +1,23 @@
-FROM node:alpine as builder
-RUN apk update && apk add --no-cache python make g++
-WORKDIR /opt/app/
-COPY package.json ./
-RUN npm install
-RUN npm prune
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
 
-FROM node:alpine as app
-WORKDIR /opt/app
-COPY . /opt/app
-COPY --from=builder /opt/app/node_modules ./node_modules
+RUN apk --update --no-cache add --virtual .builds-deps build-base python3
+
+WORKDIR /opt/app/
+
+FROM base as build
+
+COPY package.json package-lock.json ./
+
+RUN npm ci
+
+COPY . .
+
+FROM base
+
+COPY --from=build /opt/app /opt/app/
 
 ARG NODE_ENV
+
 ENV NODE_ENV $NODE_ENV
 
-CMD [ "npm", "start" ]
+CMD [ "node", "app.js" ]

Dockerfile.db-create

@@ -0,0 +1,23 @@
+FROM --platform=linux/amd64 node:18.15-alpine3.16 as base
+
+RUN apk --update --no-cache add --virtual .builds-deps build-base python3
+
+WORKDIR /opt/app/
+
+FROM base as build
+
+COPY package.json package-lock.json ./
+
+RUN npm ci
+
+COPY . .
+
+FROM base
+
+COPY --from=build /opt/app /opt/app/
+
+ARG NODE_ENV
+
+ENV NODE_ENV $NODE_ENV
+
+CMD [ "npm", "run", "upgrade-db" ]

README.md

@@ -1,29 +1,46 @@
 # jambonz-api-server ![Build Status](https://github.com/jambonz/jambonz-api-server/workflows/CI/badge.svg)
 
-Jambones REST API server.
+Jambones REST API server of the jambones platform.
 
 ## Configuration
 
-This process requires the following environment variables to be set.
+Configuration is provided via environment variables:
 
-```
-JAMBONES_MYSQL_HOST
-JAMBONES_MYSQL_USER
-JAMBONES_MYSQL_PASSWORD
-JAMBONES_MYSQL_DATABASE
-JAMBONES_MYSQL_CONNECTION_LIMIT   # defaults to 10
-JAMBONES_REDIS_HOST
-JAMBONES_REDIS_PORT
-JAMBONES_LOGLEVEL                 # defaults to info
-JAMBONES_API_VERSION              # defaults to v1
-HTTP_PORT                         # defaults to 3000
-```
+| variable | meaning | required?|
+|----------|----------|---------|
+|JWT_SECRET| secret for signing JWT token |yes|
+|JWT_EXPIRES_IN| expiration time for JWT token(in minutes) |no|
+|ENCRYPTION_SECRET| secret for credential encryption(JWT_SECRET is deprecated) |yes|
+|HTTP_PORT| tcp port to listen on for API requests from jambonz-api-server |no|
+|JAMBONES_LOGLEVEL| log level for application, 'info' or 'debug' |no|
+|JAMBONES_MYSQL_HOST| mysql host |yes|
+|JAMBONES_MYSQL_USER| mysql username |yes|
+|JAMBONES_MYSQL_PASSWORD|  mysql password |yes|
+|JAMBONES_MYSQL_DATABASE| mysql data |yes|
+|JAMBONES_MYSQL_PORT| mysql port |no|
+|JAMBONES_MYSQL_CONNECTION_LIMIT| mysql connection limit |no|
+|JAMBONES_REDIS_HOST| redis host |yes|
+|JAMBONES_REDIS_PORT| redis port |no|
+|RATE_LIMIT_WINDOWS_MINS| rate limit window |no|
+|RATE_LIMIT_MAX_PER_WINDOW| number of requests per window |no|
+|JAMBONES_TRUST_PROXY| trust proxies, must be a number |no|
+|JAMBONES_API_VERSION| api version |no|
+|JAMBONES_TIME_SERIES_HOST| influxdb host |yes|
+|JAMBONES_CLUSTER_ID| cluster id |no|
+|HOMER_BASE_URL| HOMER URL |no|
+|HOMER_USERNAME| HOMER username |no|
+|HOMER_PASSWORD| HOMER password |no|
+|K8S| service running as kubernetes service |no|
+|K8S_FEATURE_SERVER_SERVICE_NAME| feature server name(required for K8S) |no|
+|K8S_FEATURE_SERVER_SERVICE_PORT| feature server port(required for K8S) |no|
+|JAMBONZ_RECORD_WS_USERNAME| recording websocket username|no|
+|JAMBONZ_RECORD_WS_PASSWORD| recording websocket password|no|
 
 #### Database dependency
 A mysql database is used to store long-lived objects such as Accounts, Applications, etc. To create the database schema, use or review the scripts in the 'db' folder, particularly:
-- [create_db.sql](db/create_db.sql), which creates the database and associated user (you may want to edit the username and password),
 - [jambones-sql.sql](db/jambones-sql.sql), which creates the schema,
-- [create-admin-token.sql](db/create-admin-token.sql), which creates an admin-level auth token that can be used for testing/exercising the API.
+- [seed-production-database-open-source.sql](db/seed-production-database-open-source.sql), which seeds the database with initial dataset(accounts, permissions, api keys, applications etc).
+- [create-admin-user.sql](db/create-admin-user.sql), which creates admin user with password set to "admin". The password will be forced to change after the first login.
 
 > Note: due to the dependency on the npmjs [mysql](https://www.npmjs.com/package/mysql) package, the mysql database must be configured to use sql [native authentication](https://medium.com/@crmcmullen/how-to-run-mysql-8-0-with-native-password-authentication-502de5bac661).
 

app.js

@@ -1,15 +1,10 @@
 const assert = require('assert');
-const opts = Object.assign({
-  timestamp: () => {
-    return `, "time": "${new Date().toISOString()}"`;
-  }
-}, {
-  level: process.env.JAMBONES_LOGLEVEL || 'info'
-});
-const logger = require('pino')(opts);
+const logger = require('./lib/logger');
 const express = require('express');
 const app = express();
-app.disable('x-powered-by');
+const helmet = require('helmet');
+const nocache = require('nocache');
+const rateLimit = require('express-rate-limit');
 const cors = require('cors');
 const passport = require('passport');
 const routes = require('./lib/routes');
@@ -18,22 +13,46 @@ assert.ok(process.env.JAMBONES_MYSQL_HOST &&
   process.env.JAMBONES_MYSQL_USER &&
   process.env.JAMBONES_MYSQL_PASSWORD &&
   process.env.JAMBONES_MYSQL_DATABASE, 'missing JAMBONES_MYSQL_XXX env vars');
-assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+if (process.env.JAMBONES_REDIS_SENTINELS) {
+  assert.ok(process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+    'missing JAMBONES_REDIS_SENTINEL_MASTER_NAME env var, JAMBONES_REDIS_SENTINEL_PASSWORD env var is optional');
+} else {
+  assert.ok(process.env.JAMBONES_REDIS_HOST, 'missing JAMBONES_REDIS_HOST env var');
+}
 assert.ok(process.env.JAMBONES_TIME_SERIES_HOST, 'missing JAMBONES_TIME_SERIES_HOST env var');
-const {queryCdrs, queryAlerts, writeCdrs, writeAlerts, AlertType} = require('@jambonz/time-series')(
+assert.ok(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET, 'missing ENCRYPTION_SECRET env var');
+assert.ok(process.env.JWT_SECRET, 'missing JWT_SECRET env var');
+
+const {
+  queryCdrs,
+  queryCdrsSP,
+  queryAlerts,
+  queryAlertsSP,
+  writeCdrs,
+  writeAlerts,
+  AlertType
+} = require('@jambonz/time-series')(
   logger, process.env.JAMBONES_TIME_SERIES_HOST
 );
 const {
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
   retrieveKey,
-  deleteKey
-} = require('@jambonz/realtimedb-helpers')({
-  host: process.env.JAMBONES_REDIS_HOST || 'localhost',
+  deleteKey,
+  incrKey,
+  JAMBONES_REDIS_SENTINELS
+} = require('./lib/helpers/realtimedb-helpers');
+const {
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache
+} = require('@jambonz/speech-utils')(JAMBONES_REDIS_SENTINELS || {
+  host: process.env.JAMBONES_REDIS_HOST,
   port: process.env.JAMBONES_REDIS_PORT || 6379
 }, logger);
 const {
@@ -54,6 +73,8 @@ const {
 }, logger);
 const PORT = process.env.HTTP_PORT || 3000;
 const authStrategy = require('./lib/auth')(logger, retrieveKey);
+const {delayLoginMiddleware} = require('./lib/middleware');
+const Websocket = require('ws');
 
 passport.use(authStrategy);
 
@@ -64,11 +85,16 @@ app.locals = {
   retrieveCall,
   deleteCall,
   listCalls,
+  listSortedSets,
   purgeCalls,
   retrieveSet,
   addKey,
+  incrKey,
   retrieveKey,
   deleteKey,
+  getTtsVoices,
+  getTtsSize,
+  purgeTtsCache,
   lookupAppBySid,
   lookupAccountBySid,
   lookupAccountByPhoneNumber,
@@ -77,7 +103,9 @@ app.locals = {
   lookupSipGatewayBySid,
   lookupSmppGatewayBySid,
   queryCdrs,
+  queryCdrsSP,
   queryAlerts,
+  queryAlertsSP,
   writeCdrs,
   writeAlerts,
   AlertType
@@ -89,9 +117,39 @@ const unless = (paths, middleware) => {
     return middleware(req, res, next);
   };
 };
+
+const limiter = rateLimit({
+  windowMs: (process.env.RATE_LIMIT_WINDOWS_MINS || 5) * 60 * 1000, // 5 minutes
+  max: process.env.RATE_LIMIT_MAX_PER_WINDOW || 600, // Limit each IP to 600 requests per `window`
+  standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
+  legacyHeaders: false, // Disable the `X-RateLimit-*` headers
+});
+
+// Setup websocket for recording audio
+const recordWsServer = require('./lib/record');
+const wsServer = new Websocket.Server({ noServer: true });
+wsServer.setMaxListeners(0);
+wsServer.on('connection', recordWsServer.bind(null, logger));
+
+if (process.env.JAMBONES_TRUST_PROXY) {
+  const proxyCount = parseInt(process.env.JAMBONES_TRUST_PROXY);
+  if (!isNaN(proxyCount) && proxyCount > 0) {
+    logger.info(`setting trust proxy to ${proxyCount} and mounting endpoint /ip`);
+    app.set('trust proxy', proxyCount);
+    app.get('/ip', (req, res) => {
+      logger.info({headers: req.headers}, 'received GET /ip');
+      res.send(req.ip);
+    });
+  }
+}
+app.use(limiter);
+app.use(helmet());
+app.use(helmet.hidePoweredBy());
+app.use(nocache());
 app.use(passport.initialize());
 app.use(cors());
 app.use(express.urlencoded({extended: true}));
+app.use(delayLoginMiddleware);
 app.use(unless(['/stripe'], express.json()));
 app.use('/v1', unless(
   [
@@ -113,7 +171,52 @@ app.use((err, req, res, next) => {
   });
 });
 logger.info(`listening for HTTP traffic on port ${PORT}`);
-app.listen(PORT);
+const server = app.listen(PORT);
+
+
+const isValidWsKey = (hdr) => {
+  const username = process.env.JAMBONZ_RECORD_WS_USERNAME || process.env.JAMBONES_RECORD_WS_USERNAME;
+  const password = process.env.JAMBONZ_RECORD_WS_PASSWORD || process.env.JAMBONES_RECORD_WS_PASSWORD;
+  if (username && password) {
+    if (!hdr) {
+      // auth header is missing
+      return false;
+    }
+    const token = Buffer.from(`${username}:${password}`).toString('base64');
+    const arr = /^Basic (.*)$/.exec(hdr);
+    if (!Array.isArray(arr)) {
+      // malformed auth header
+      return false;
+    }
+    return arr[1] === token;
+  }
+  return true;
+};
+
+server.on('upgrade', (request, socket, head) => {
+  logger.debug({
+    url: request.url,
+    headers: request.headers,
+  }, 'received upgrade request');
+
+  /* verify the path starts with /transcribe */
+  if (!request.url.includes('/record/')) {
+    logger.info(`unhandled path: ${request.url}`);
+    return socket.write('HTTP/1.1 404 Not Found \r\n\r\n', () => socket.destroy());
+  }
+
+  /* verify the api key */
+  if (!isValidWsKey(request.headers['authorization'])) {
+    logger.info(`invalid auth header: ${request.headers['authorization'] || 'authorization header missing'}`);
+    return socket.write('HTTP/1.1 403 Forbidden \r\n\r\n', () => socket.destroy());
+  }
+
+  /* complete the upgrade */
+  wsServer.handleUpgrade(request, socket, head, (ws) => {
+    logger.info(`upgraded to websocket, url: ${request.url}`);
+    wsServer.emit('connection', ws, request.url);
+  });
+});
 
 // purge old calls from active call set every 10 mins
 async function purge() {

db/add-predefined-carriers.sql

@@ -45,8 +45,6 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);

db/create-admin-user.sql

@@ -0,0 +1,10 @@
+/* hashed password is "admin" */
+insert into users (user_sid, name, email, hashed_password, force_change, provider, email_validated)
+values ('12c80508-edf9-4b22-8d09-55abd02648eb', 'admin', 'joe@foo.bar', '$argon2i$v=19$m=65536,t=3,p=4$c2FsdHNhbHRzYWx0c2FsdA$x5OO6gXFXS25oqUU2JvbYqrSgRxBujNUJBq6xv9EgjM', 1, 'local', 1);
+
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('8919e0dc-4d69-4de5-be56-a121598d9093', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc342a-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('d6fdf064-0a65-4b17-8b10-5500e956a159', '12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3a10-546a-11ed-bdc3-0242ac120002');
+insert into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+values ('f68185dd-0486-4767-a77d-a0b84c1b236e' ,'12c80508-edf9-4b22-8d09-55abd02648eb', 'ffbc3c5e-546a-11ed-bdc3-0242ac120002');

db/jambones-sql.sql

@@ -1,9 +1,10 @@
 /* SQLEditor (MySQL (2))*/
-
 SET FOREIGN_KEY_CHECKS=0;
 
 DROP TABLE IF EXISTS account_static_ips;
 
+DROP TABLE IF EXISTS account_limits;
+
 DROP TABLE IF EXISTS account_products;
 
 DROP TABLE IF EXISTS account_subscriptions;
@@ -12,12 +13,22 @@ DROP TABLE IF EXISTS beta_invite_codes;
 
 DROP TABLE IF EXISTS call_routes;
 
+DROP TABLE IF EXISTS clients;
+
 DROP TABLE IF EXISTS dns_records;
 
+DROP TABLE IF EXISTS lcr;
+
 DROP TABLE IF EXISTS lcr_carrier_set_entry;
 
 DROP TABLE IF EXISTS lcr_routes;
 
+DROP TABLE IF EXISTS password_settings;
+
+DROP TABLE IF EXISTS user_permissions;
+
+DROP TABLE IF EXISTS permissions;
+
 DROP TABLE IF EXISTS predefined_sip_gateways;
 
 DROP TABLE IF EXISTS predefined_smpp_gateways;
@@ -28,18 +39,24 @@ DROP TABLE IF EXISTS account_offers;
 
 DROP TABLE IF EXISTS products;
 
+DROP TABLE IF EXISTS schema_version;
+
 DROP TABLE IF EXISTS api_keys;
 
 DROP TABLE IF EXISTS sbc_addresses;
 
 DROP TABLE IF EXISTS ms_teams_tenants;
 
+DROP TABLE IF EXISTS service_provider_limits;
+
 DROP TABLE IF EXISTS signup_history;
 
 DROP TABLE IF EXISTS smpp_addresses;
 
 DROP TABLE IF EXISTS speech_credentials;
 
+DROP TABLE IF EXISTS system_information;
+
 DROP TABLE IF EXISTS users;
 
 DROP TABLE IF EXISTS smpp_gateways;
@@ -67,6 +84,15 @@ private_ipv4 VARBINARY(16) NOT NULL UNIQUE ,
 PRIMARY KEY (account_static_ip_sid)
 );
 
+CREATE TABLE account_limits
+(
+account_limits_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+category ENUM('api_rate','voice_call_session', 'device','voice_call_minutes','voice_call_session_license', 'voice_call_minutes_license') NOT NULL,
+quantity INTEGER NOT NULL,
+PRIMARY KEY (account_limits_sid)
+);
+
 CREATE TABLE account_subscriptions
 (
 account_subscription_sid CHAR(36) NOT NULL UNIQUE ,
@@ -103,6 +129,16 @@ application_sid CHAR(36) NOT NULL,
 PRIMARY KEY (call_route_sid)
 ) COMMENT='a regex-based pattern match for call routing';
 
+CREATE TABLE clients
+(
+client_sid CHAR(36) NOT NULL UNIQUE ,
+account_sid CHAR(36) NOT NULL,
+is_active BOOLEAN NOT NULL DEFAULT 1,
+username VARCHAR(64),
+password VARCHAR(1024),
+PRIMARY KEY (client_sid)
+);
+
 CREATE TABLE dns_records
 (
 dns_record_sid CHAR(36) NOT NULL UNIQUE ,
@@ -115,11 +151,38 @@ PRIMARY KEY (dns_record_sid)
 CREATE TABLE lcr_routes
 (
 lcr_route_sid CHAR(36),
+lcr_sid CHAR(36) NOT NULL,
 regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
 description VARCHAR(1024),
-priority INTEGER NOT NULL UNIQUE  COMMENT 'lower priority routes are attempted first',
+priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
 PRIMARY KEY (lcr_route_sid)
-) COMMENT='Least cost routing table';
+) COMMENT='An ordered list of  digit patterns in an LCR table.  The patterns are tested in sequence until one matches';
+
+CREATE TABLE lcr
+(
+lcr_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+is_active BOOLEAN NOT NULL DEFAULT 1,
+default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+service_provider_sid CHAR(36),
+account_sid CHAR(36),
+PRIMARY KEY (lcr_sid)
+) COMMENT='An LCR (least cost routing) table that is used by a service provider or account to make decisions about routing outbound calls when multiple carriers are available.';
+
+CREATE TABLE password_settings
+(
+min_password_length INTEGER NOT NULL DEFAULT 8,
+require_digit BOOLEAN NOT NULL DEFAULT false,
+require_special_character BOOLEAN NOT NULL DEFAULT false
+);
+
+CREATE TABLE permissions
+(
+permission_sid CHAR(36) NOT NULL UNIQUE ,
+name VARCHAR(32) NOT NULL UNIQUE ,
+description VARCHAR(255),
+PRIMARY KEY (permission_sid)
+);
 
 CREATE TABLE predefined_carriers
 (
@@ -190,6 +253,11 @@ stripe_product_id VARCHAR(56) NOT NULL,
 PRIMARY KEY (account_offer_sid)
 );
 
+CREATE TABLE schema_version
+(
+version VARCHAR(16)
+);
+
 CREATE TABLE api_keys
 (
 api_key_sid CHAR(36) NOT NULL UNIQUE ,
@@ -207,7 +275,10 @@ CREATE TABLE sbc_addresses
 sbc_address_sid CHAR(36) NOT NULL UNIQUE ,
 ipv4 VARCHAR(255) NOT NULL,
 port INTEGER NOT NULL DEFAULT 5060,
+tls_port INTEGER,
+wss_port INTEGER,
 service_provider_sid CHAR(36),
+last_updated DATETIME,
 PRIMARY KEY (sbc_address_sid)
 );
 
@@ -221,6 +292,15 @@ tenant_fqdn VARCHAR(255) NOT NULL UNIQUE ,
 PRIMARY KEY (ms_teams_tenant_sid)
 ) COMMENT='A Microsoft Teams customer tenant';
 
+CREATE TABLE service_provider_limits
+(
+service_provider_limits_sid CHAR(36) NOT NULL UNIQUE ,
+service_provider_sid CHAR(36) NOT NULL,
+category ENUM('api_rate','voice_call_session', 'device','voice_call_minutes','voice_call_session_license', 'voice_call_minutes_license') NOT NULL,
+quantity INTEGER NOT NULL,
+PRIMARY KEY (service_provider_limits_sid)
+);
+
 CREATE TABLE signup_history
 (
 email VARCHAR(255) NOT NULL,
@@ -254,9 +334,17 @@ last_tested DATETIME,
 tts_tested_ok BOOLEAN,
 stt_tested_ok BOOLEAN,
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+label VARCHAR(64),
 PRIMARY KEY (speech_credential_sid)
 );
 
+CREATE TABLE system_information
+(
+domain_name VARCHAR(255),
+sip_domain_name VARCHAR(255),
+monitoring_domain_name VARCHAR(255)
+);
+
 CREATE TABLE users
 (
 user_sid CHAR(36) NOT NULL UNIQUE ,
@@ -276,6 +364,7 @@ email_activation_code VARCHAR(16),
 email_validated BOOLEAN NOT NULL DEFAULT false,
 phone_validated BOOLEAN NOT NULL DEFAULT false,
 email_content_opt_out BOOLEAN NOT NULL DEFAULT false,
+is_active BOOLEAN NOT NULL DEFAULT true,
 PRIMARY KEY (user_sid)
 );
 
@@ -303,9 +392,21 @@ smpp_password VARCHAR(64),
 smpp_enquire_link_interval INTEGER DEFAULT 0,
 smpp_inbound_system_id VARCHAR(255),
 smpp_inbound_password VARCHAR(64),
+register_from_user VARCHAR(128),
+register_from_domain VARCHAR(255),
+register_public_ip_in_contact BOOLEAN NOT NULL DEFAULT false,
+register_status VARCHAR(4096),
 PRIMARY KEY (voip_carrier_sid)
 ) COMMENT='A Carrier or customer PBX that can send or receive calls';
 
+CREATE TABLE user_permissions
+(
+user_permissions_sid CHAR(36) NOT NULL UNIQUE ,
+user_sid CHAR(36) NOT NULL,
+permission_sid CHAR(36) NOT NULL,
+PRIMARY KEY (user_permissions_sid)
+);
+
 CREATE TABLE smpp_gateways
 (
 smpp_gateway_sid CHAR(36) NOT NULL UNIQUE ,
@@ -323,7 +424,7 @@ PRIMARY KEY (smpp_gateway_sid)
 CREATE TABLE phone_numbers
 (
 phone_number_sid CHAR(36) UNIQUE ,
-number VARCHAR(32) NOT NULL UNIQUE ,
+number VARCHAR(132) NOT NULL,
 voip_carrier_sid CHAR(36),
 account_sid CHAR(36),
 application_sid CHAR(36),
@@ -336,11 +437,13 @@ CREATE TABLE sip_gateways
 sip_gateway_sid CHAR(36),
 ipv4 VARCHAR(128) NOT NULL COMMENT 'ip address or DNS name of the gateway.  For gateways providing inbound calling service, ip address is required.',
 netmask INTEGER NOT NULL DEFAULT 32,
-port INTEGER NOT NULL DEFAULT 5060 COMMENT 'sip signaling port',
+port INTEGER COMMENT 'sip signaling port',
 inbound BOOLEAN NOT NULL COMMENT 'if true, whitelist this IP to allow inbound calls from the gateway',
 outbound BOOLEAN NOT NULL COMMENT 'if true, include in least-cost routing when placing calls to the PSTN',
 voip_carrier_sid CHAR(36) NOT NULL,
 is_active BOOLEAN NOT NULL DEFAULT 1,
+pad_crypto BOOLEAN NOT NULL DEFAULT 0,
+protocol ENUM('udp','tcp','tls', 'tls/srtp') DEFAULT 'udp' COMMENT 'Outbound call protocol',
 PRIMARY KEY (sip_gateway_sid)
 ) COMMENT='A whitelisted sip gateway used for origination/termination';
 
@@ -373,12 +476,24 @@ account_sid CHAR(36) COMMENT 'account that this application belongs to (if null,
 call_hook_sid CHAR(36) COMMENT 'webhook to call for inbound calls ',
 call_status_hook_sid CHAR(36) COMMENT 'webhook to call for call status events',
 messaging_hook_sid CHAR(36) COMMENT 'webhook to call for inbound SMS/MMS ',
+app_json TEXT,
 speech_synthesis_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_synthesis_language VARCHAR(12) NOT NULL DEFAULT 'en-US',
 speech_synthesis_voice VARCHAR(64),
+speech_synthesis_label VARCHAR(64),
 speech_recognizer_vendor VARCHAR(64) NOT NULL DEFAULT 'google',
 speech_recognizer_language VARCHAR(64) NOT NULL DEFAULT 'en-US',
+speech_recognizer_label VARCHAR(64),
+use_for_fallback_speech BOOLEAN DEFAULT false,
+fallback_speech_synthesis_vendor VARCHAR(64),
+fallback_speech_synthesis_language VARCHAR(12),
+fallback_speech_synthesis_voice VARCHAR(64),
+fallback_speech_synthesis_label VARCHAR(64),
+fallback_speech_recognizer_vendor VARCHAR(64),
+fallback_speech_recognizer_language VARCHAR(64),
+fallback_speech_recognizer_label VARCHAR(64),
 created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
 PRIMARY KEY (application_sid)
 ) COMMENT='A defined set of behaviors to be applied to phone calls ';
 
@@ -411,6 +526,14 @@ disable_cdrs BOOLEAN NOT NULL DEFAULT 0,
 trial_end_date DATETIME,
 deactivated_reason VARCHAR(255),
 device_to_call_ratio INTEGER NOT NULL DEFAULT 5,
+subspace_client_id VARCHAR(255),
+subspace_client_secret VARCHAR(255),
+subspace_sip_teleport_id VARCHAR(255),
+subspace_sip_teleport_destinations VARCHAR(255),
+siprec_hook_sid CHAR(36),
+record_all_calls BOOLEAN NOT NULL DEFAULT false,
+record_format VARCHAR(16) NOT NULL DEFAULT 'mp3',
+bucket_credential VARCHAR(8192) COMMENT 'credential used to authenticate with storage service',
 PRIMARY KEY (account_sid)
 ) COMMENT='An enterprise that uses the platform for comm services';
 
@@ -418,19 +541,34 @@ CREATE INDEX account_static_ip_sid_idx ON account_static_ips (account_static_ip_
 CREATE INDEX account_sid_idx ON account_static_ips (account_sid);
 ALTER TABLE account_static_ips ADD FOREIGN KEY account_sid_idxfk (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX account_sid_idx ON account_limits (account_sid);
+ALTER TABLE account_limits ADD FOREIGN KEY account_sid_idxfk_1 (account_sid) REFERENCES accounts (account_sid) ON DELETE CASCADE;
+
 CREATE INDEX account_subscription_sid_idx ON account_subscriptions (account_subscription_sid);
 CREATE INDEX account_sid_idx ON account_subscriptions (account_sid);
-ALTER TABLE account_subscriptions ADD FOREIGN KEY account_sid_idxfk_1 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE account_subscriptions ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX invite_code_idx ON beta_invite_codes (invite_code);
 CREATE INDEX call_route_sid_idx ON call_routes (call_route_sid);
-ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE call_routes ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE call_routes ADD FOREIGN KEY application_sid_idxfk (application_sid) REFERENCES applications (application_sid);
 
-CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
-ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_3 (account_sid) REFERENCES accounts (account_sid);
+CREATE INDEX client_sid_idx ON clients (client_sid);
+ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid);
 
+CREATE INDEX dns_record_sid_idx ON dns_records (dns_record_sid);
+ALTER TABLE dns_records ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
+
+CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid);
+ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid);
+
+CREATE INDEX lcr_sid_idx ON lcr (lcr_sid);
+ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid);
+
+CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid);
+CREATE INDEX account_sid_idx ON lcr (account_sid);
+CREATE INDEX permission_sid_idx ON permissions (permission_sid);
 CREATE INDEX predefined_carrier_sid_idx ON predefined_carriers (predefined_carrier_sid);
 CREATE INDEX predefined_sip_gateway_sid_idx ON predefined_sip_gateways (predefined_sip_gateway_sid);
 CREATE INDEX predefined_carrier_sid_idx ON predefined_sip_gateways (predefined_carrier_sid);
@@ -449,14 +587,14 @@ ALTER TABLE account_products ADD FOREIGN KEY product_sid_idxfk (product_sid) REF
 
 CREATE INDEX account_offer_sid_idx ON account_offers (account_offer_sid);
 CREATE INDEX account_sid_idx ON account_offers (account_sid);
-ALTER TABLE account_offers ADD FOREIGN KEY account_sid_idxfk_4 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE account_offers ADD FOREIGN KEY account_sid_idxfk_5 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX product_sid_idx ON account_offers (product_sid);
 ALTER TABLE account_offers ADD FOREIGN KEY product_sid_idxfk_1 (product_sid) REFERENCES products (product_sid);
 
 CREATE INDEX api_key_sid_idx ON api_keys (api_key_sid);
 CREATE INDEX account_sid_idx ON api_keys (account_sid);
-ALTER TABLE api_keys ADD FOREIGN KEY account_sid_idxfk_5 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE api_keys ADD FOREIGN KEY account_sid_idxfk_6 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX service_provider_sid_idx ON api_keys (service_provider_sid);
 ALTER TABLE api_keys ADD FOREIGN KEY service_provider_sid_idxfk (service_provider_sid) REFERENCES service_providers (service_provider_sid);
@@ -470,59 +608,68 @@ ALTER TABLE sbc_addresses ADD FOREIGN KEY service_provider_sid_idxfk_1 (service_
 CREATE INDEX ms_teams_tenant_sid_idx ON ms_teams_tenants (ms_teams_tenant_sid);
 ALTER TABLE ms_teams_tenants ADD FOREIGN KEY service_provider_sid_idxfk_2 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
-ALTER TABLE ms_teams_tenants ADD FOREIGN KEY account_sid_idxfk_6 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE ms_teams_tenants ADD FOREIGN KEY account_sid_idxfk_7 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE ms_teams_tenants ADD FOREIGN KEY application_sid_idxfk_1 (application_sid) REFERENCES applications (application_sid);
 
 CREATE INDEX tenant_fqdn_idx ON ms_teams_tenants (tenant_fqdn);
+CREATE INDEX service_provider_sid_idx ON service_provider_limits (service_provider_sid);
+ALTER TABLE service_provider_limits ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) REFERENCES service_providers (service_provider_sid) ON DELETE CASCADE;
+
 CREATE INDEX email_idx ON signup_history (email);
 CREATE INDEX smpp_address_sid_idx ON smpp_addresses (smpp_address_sid);
 CREATE INDEX service_provider_sid_idx ON smpp_addresses (service_provider_sid);
-ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
-
-CREATE UNIQUE INDEX speech_credentials_idx_1 ON speech_credentials (vendor,account_sid);
+ALTER TABLE smpp_addresses ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX speech_credential_sid_idx ON speech_credentials (speech_credential_sid);
 CREATE INDEX service_provider_sid_idx ON speech_credentials (service_provider_sid);
-ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_4 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE speech_credentials ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX account_sid_idx ON speech_credentials (account_sid);
-ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_7 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE speech_credentials ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX user_sid_idx ON users (user_sid);
 CREATE INDEX email_idx ON users (email);
 CREATE INDEX phone_idx ON users (phone);
 CREATE INDEX account_sid_idx ON users (account_sid);
-ALTER TABLE users ADD FOREIGN KEY account_sid_idxfk_8 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE users ADD FOREIGN KEY account_sid_idxfk_9 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX service_provider_sid_idx ON users (service_provider_sid);
-ALTER TABLE users ADD FOREIGN KEY service_provider_sid_idxfk_5 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE users ADD FOREIGN KEY service_provider_sid_idxfk_6 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX email_activation_code_idx ON users (email_activation_code);
 CREATE INDEX voip_carrier_sid_idx ON voip_carriers (voip_carrier_sid);
 CREATE INDEX account_sid_idx ON voip_carriers (account_sid);
-ALTER TABLE voip_carriers ADD FOREIGN KEY account_sid_idxfk_9 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE voip_carriers ADD FOREIGN KEY account_sid_idxfk_10 (account_sid) REFERENCES accounts (account_sid);
 
 CREATE INDEX service_provider_sid_idx ON voip_carriers (service_provider_sid);
-ALTER TABLE voip_carriers ADD FOREIGN KEY service_provider_sid_idxfk_6 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE voip_carriers ADD FOREIGN KEY service_provider_sid_idxfk_7 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 ALTER TABLE voip_carriers ADD FOREIGN KEY application_sid_idxfk_2 (application_sid) REFERENCES applications (application_sid);
 
+CREATE INDEX user_permissions_sid_idx ON user_permissions (user_permissions_sid);
+CREATE INDEX user_sid_idx ON user_permissions (user_sid);
+ALTER TABLE user_permissions ADD FOREIGN KEY user_sid_idxfk (user_sid) REFERENCES users (user_sid) ON DELETE CASCADE;
+
+ALTER TABLE user_permissions ADD FOREIGN KEY permission_sid_idxfk (permission_sid) REFERENCES permissions (permission_sid);
+
 CREATE INDEX smpp_gateway_sid_idx ON smpp_gateways (smpp_gateway_sid);
 CREATE INDEX voip_carrier_sid_idx ON smpp_gateways (voip_carrier_sid);
 ALTER TABLE smpp_gateways ADD FOREIGN KEY voip_carrier_sid_idxfk (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
+CREATE UNIQUE INDEX phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid);
+
 CREATE INDEX phone_number_sid_idx ON phone_numbers (phone_number_sid);
 CREATE INDEX number_idx ON phone_numbers (number);
 CREATE INDEX voip_carrier_sid_idx ON phone_numbers (voip_carrier_sid);
 ALTER TABLE phone_numbers ADD FOREIGN KEY voip_carrier_sid_idxfk_1 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid);
 
-ALTER TABLE phone_numbers ADD FOREIGN KEY account_sid_idxfk_10 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE phone_numbers ADD FOREIGN KEY account_sid_idxfk_11 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE phone_numbers ADD FOREIGN KEY application_sid_idxfk_3 (application_sid) REFERENCES applications (application_sid);
 
 CREATE INDEX service_provider_sid_idx ON phone_numbers (service_provider_sid);
-ALTER TABLE phone_numbers ADD FOREIGN KEY service_provider_sid_idxfk_7 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE phone_numbers ADD FOREIGN KEY service_provider_sid_idxfk_8 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX sip_gateway_idx_hostport ON sip_gateways (ipv4,port);
 
@@ -538,10 +685,10 @@ CREATE UNIQUE INDEX applications_idx_name ON applications (account_sid,name);
 
 CREATE INDEX application_sid_idx ON applications (application_sid);
 CREATE INDEX service_provider_sid_idx ON applications (service_provider_sid);
-ALTER TABLE applications ADD FOREIGN KEY service_provider_sid_idxfk_8 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE applications ADD FOREIGN KEY service_provider_sid_idxfk_9 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 CREATE INDEX account_sid_idx ON applications (account_sid);
-ALTER TABLE applications ADD FOREIGN KEY account_sid_idxfk_11 (account_sid) REFERENCES accounts (account_sid);
+ALTER TABLE applications ADD FOREIGN KEY account_sid_idxfk_12 (account_sid) REFERENCES accounts (account_sid);
 
 ALTER TABLE applications ADD FOREIGN KEY call_hook_sid_idxfk (call_hook_sid) REFERENCES webhooks (webhook_sid);
 
@@ -557,7 +704,7 @@ ALTER TABLE service_providers ADD FOREIGN KEY registration_hook_sid_idxfk (regis
 CREATE INDEX account_sid_idx ON accounts (account_sid);
 CREATE INDEX sip_realm_idx ON accounts (sip_realm);
 CREATE INDEX service_provider_sid_idx ON accounts (service_provider_sid);
-ALTER TABLE accounts ADD FOREIGN KEY service_provider_sid_idxfk_9 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
+ALTER TABLE accounts ADD FOREIGN KEY service_provider_sid_idxfk_10 (service_provider_sid) REFERENCES service_providers (service_provider_sid);
 
 ALTER TABLE accounts ADD FOREIGN KEY registration_hook_sid_idxfk_1 (registration_hook_sid) REFERENCES webhooks (webhook_sid);
 
@@ -565,4 +712,5 @@ ALTER TABLE accounts ADD FOREIGN KEY queue_event_hook_sid_idxfk (queue_event_hoo
 
 ALTER TABLE accounts ADD FOREIGN KEY device_calling_application_sid_idxfk (device_calling_application_sid) REFERENCES applications (application_sid);
 
-SET FOREIGN_KEY_CHECKS=1;
+ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid);
+SET FOREIGN_KEY_CHECKS=1;

db/reset_admin_password.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 const {promisePool} = require('../lib/db');
-const uuidv4 = require('uuid/v4');
+const { v4: uuidv4 } = require('uuid');
 const {generateHashedPassword} = require('../lib/utils/password-utils');
 const sqlInsert = `INSERT into users 
 (user_sid, name, email, hashed_password, force_change, provider, email_validated) 
@@ -12,6 +12,9 @@ values (?, ?)`;
 const sqlQueryAccount = 'SELECT * from accounts LEFT JOIN api_keys ON api_keys.account_sid = accounts.account_sid';
 const sqlAddAccountToken = `INSERT into api_keys (api_key_sid, token, account_sid) 
 VALUES (?, ?, ?)`;
+const sqlInsertPermissions = `
+INSERT into user_permissions (user_permissions_sid, user_sid, permission_sid) 
+VALUES (?,?,?)`;
 
 const password = process.env.JAMBONES_ADMIN_INITIAL_PASSWORD || 'admin';
 console.log(`reset_admin_password, initial admin password is ${password}`);
@@ -21,6 +24,7 @@ const doIt = async() => {
   const sid = uuidv4();
   await promisePool.execute('DELETE from users where name = "admin"');
   await promisePool.execute('DELETE from api_keys where account_sid is null and service_provider_sid is null');
+
   await promisePool.execute(sqlInsert,
     [
       sid,
@@ -34,6 +38,12 @@ const doIt = async() => {
   );
   await promisePool.execute(sqlInsertAdminToken, [uuidv4(), uuidv4()]);
 
+  /* assign all permissions to the admin user */
+  const [p] = await promisePool.query('SELECT * from permissions');
+  for (const perm of p) {
+    await promisePool.execute(sqlInsertPermissions, [uuidv4(), sid, perm.permission_sid]);
+  }
+
   /* create admin token for single account */
   const [r] = await promisePool.query({sql: sqlQueryAccount, nestTables: true});
   if (1 === r.length && r[0].api_keys.api_key_sid === null) {

db/seed-integration-test.sql

@@ -1,5 +1,12 @@
 SET FOREIGN_KEY_CHECKS=0;
 
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
 insert into sbc_addresses (sbc_address_sid, ipv4, port) 
 values('f6567ae1-bf97-49af-8931-ca014b689995', '52.55.111.178', 5060);
 insert into sbc_addresses (sbc_address_sid, ipv4, port) 
@@ -15,17 +22,25 @@ values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9
 
 -- create one service provider and one account
 insert into service_providers (service_provider_sid, name, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.us');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'default service provider', 'sip.jambonz.cloud');
 
 insert into accounts (account_sid, service_provider_sid, name, webhook_secret) 
 values ('9351f46a-678c-43f5-b8a6-d4eb58d131af','2708b1b3-2736-40ea-b502-c53d8396247f', 'default account', 'wh_secret_cJqgtMDPzDhhnjmaJH6Mtk');
 
+-- create account level api key
+insert into api_keys (api_key_sid, token, service_provider_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fa', '38700987-c7a4-4685-a5bb-af378f9734da', '9351f46a-678c-43f5-b8a6-d4eb58d131af');
+
+-- create SP level api key
+insert into api_keys (api_key_sid, token, account_sid) 
+values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36fs', '38700987-c7a4-4685-a5bb-af378f9734ds', '2708b1b3-2736-40ea-b502-c53d8396247f');
+
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES
@@ -72,6 +87,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways
@@ -91,10 +107,8 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
 

db/seed-production-database-open-source.sql

@@ -1,5 +1,12 @@
 SET FOREIGN_KEY_CHECKS=0;
 
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
 -- create one service provider and account
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
@@ -17,10 +24,9 @@ values ('09e92f3c-9d73-4303-b63f-3668574862ce', '1cf2f4f4-64c4-4249-9a3e-5bb4cb5
 -- create two applications
 insert into webhooks(webhook_sid, url, method)
 values 
-('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.us/call-status', 'POST'),
-('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.us/hello-world', 'POST'),
-('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.us/dial-time', 'POST');
-
+('84e3db00-b172-4e46-b54b-a503fdb19e4a', 'https://public-apps.jambonz.cloud/call-status', 'POST'),
+('d31568d0-b193-4a05-8ff6-778369bc6efe', 'https://public-apps.jambonz.cloud/hello-world', 'POST'),
+('81844b05-714d-4295-8bf3-3b0640a4bf02', 'https://public-apps.jambonz.cloud/dial-time', 'POST');
 insert into applications (application_sid, account_sid, name, call_hook_sid, call_status_hook_sid, speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 VALUES
 ('7087fe50-8acb-4f3b-b820-97b573723aab', '9351f46a-678c-43f5-b8a6-d4eb58d131af', 'hello world', 'd31568d0-b193-4a05-8ff6-778369bc6efe', '84e3db00-b172-4e46-b54b-a503fdb19e4a', 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US'),
@@ -66,6 +72,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways
@@ -85,10 +92,8 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
 SET FOREIGN_KEY_CHECKS=1;

db/seed-production-database.sql

@@ -1,6 +1,15 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- create standard permissions 
+insert into permissions (permission_sid, name, description)
+values 
+('ffbc342a-546a-11ed-bdc3-0242ac120002', 'VIEW_ONLY', 'Can view data but not make changes'),
+('ffbc3a10-546a-11ed-bdc3-0242ac120002', 'PROVISION_SERVICES', 'Can provision services'),
+('ffbc3c5e-546a-11ed-bdc3-0242ac120002', 'PROVISION_USERS', 'Can provision users');
+
 -- create one service provider
 insert into service_providers (service_provider_sid, name, description, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'sip.jambonz.us', 'jambonz.us service provider', 'sip.jambonz.us');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'sip.jambonz.xyz', 'jambonz.xyz service provider', 'sip.jambonz.xyz');
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
 
@@ -16,9 +25,9 @@ insert into predefined_carriers (predefined_carrier_sid, name, requires_static_i
 requires_register, register_username, register_password, 
 register_sip_realm, tech_prefix, inbound_auth_username, inbound_auth_password, diversion)
 VALUES
+('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
 ('7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', 'Twilio', 0, 1, 0, '<your-twilio-credential-username>', '<your-twilio-credential-password>', NULL, NULL, NULL, NULL, NULL),
 ('032d90d5-39e8-41c0-b807-9c88cffba65c', 'Voxbone', 0, 1, 0, '<your-voxbone-outbound-username>', '<your-voxbone-outbound-password>', NULL, NULL, NULL, NULL, '<your-voxbone-DID>'),
-('17479288-bb9f-421a-89d1-f4ac57af1dca', 'TelecomsXChange', 0, 0, 0, NULL, NULL, NULL, 'your-tech-prefix', NULL, NULL, NULL),
 ('e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'Simwood', 0, 1, 0, '<your-simwood-auth-trunk-username>',  '<your-simwood-auth-trunk-password>', NULL, NULL, NULL, NULL, NULL);
 
 -- TelecomXchange gateways
@@ -44,6 +53,7 @@ VALUES
 ('81a0c8cb-a33e-42da-8f20-99083da6f02f', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.252.254.64', 30, 5060, 1, 0),
 ('eeeef07a-46b8-4ffe-a4f2-04eb32ca889e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '54.169.127.128', 30, 5060, 1, 0),
 ('fbb6c194-4b68-4dff-9b42-52412be1c39e', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '177.71.206.192', 30, 5060, 1, 0),
+('973e7824-0cf3-4645-88e4-d2460ddb8577', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '168.86.128.0', 18, 5060, 1, 0),
 ('3ed1dd12-e1a7-44ff-811a-3cc5dc13dc72', '7d509a18-bbff-4c5d-b21e-b99bf8f8c49a', '<your-domain>.pstn.twilio.com', 32, 5060, 0, 1);
 
 -- voxbone gateways
@@ -63,9 +73,8 @@ VALUES
 ('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.77', 32, 5060, 1, 0),
 ('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.140.77', 32, 5060, 1, 0),
 ('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.77', 32, 5060, 1, 0),
-('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.141.77', 32, 5060, 1, 0),
-('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.77', 32, 5060, 1, 0),
-('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.34', 32, 5060, 1, 0),
-('e7447e7e-2c7d-4738-ab53-097c187236ff', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.143.66', 32, 5060, 1, 0),
+('38d8520a-527f-4f8e-8456-f9dfca742561', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.77', 32, 5060, 1, 0),
+('834f8b0c-d4c2-4f3e-93d9-cf307995eedd', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.88', 32, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
+SET FOREIGN_KEY_CHECKS=1;

db/upgrade-jambonz-db.js

@@ -0,0 +1,258 @@
+#!/usr/bin/env node
+/* eslint-disable max-len */
+const assert = require('assert');
+const mysql = require('mysql2/promise');
+const {readFile} = require('fs/promises');
+const {execSync} = require('child_process');
+const {version:desiredVersion} = require('../package.json');
+const logger = require('pino')();
+
+logger.info(`upgrade-jambonz-db: desired version ${desiredVersion}`);
+
+assert.ok(process.env.JAMBONES_MYSQL_HOST, 'missing env JAMBONES_MYSQL_HOST');
+assert.ok(process.env.JAMBONES_MYSQL_DATABASE, 'missing env JAMBONES_MYSQL_DATABASE');
+assert.ok(process.env.JAMBONES_MYSQL_PASSWORD, 'missing env JAMBONES_MYSQL_PASSWORD');
+assert.ok(process.env.JAMBONES_MYSQL_USER, 'missing env JAMBONES_MYSQL_USER');
+
+const opts = {
+  host: process.env.JAMBONES_MYSQL_HOST,
+  user: process.env.JAMBONES_MYSQL_USER,
+  password: process.env.JAMBONES_MYSQL_PASSWORD,
+  database: process.env.JAMBONES_MYSQL_DATABASE,
+  port: process.env.JAMBONES_MYSQL_PORT || 3306,
+  multipleStatements: true
+};
+
+const sql = {
+  '7006': [
+    'ALTER TABLE `accounts` ADD COLUMN `siprec_hook_sid` CHAR(36)',
+    'ALTER TABLE accounts ADD FOREIGN KEY siprec_hook_sid_idxfk (siprec_hook_sid) REFERENCES applications (application_sid)'
+  ],
+  '7007': [
+    `CREATE TABLE service_provider_limits 
+    (service_provider_limits_sid CHAR(36) NOT NULL UNIQUE,
+    service_provider_sid CHAR(36) NOT NULL,
+    category ENUM('api_rate','voice_call_session', 'device') NOT NULL,
+    quantity INTEGER NOT NULL,
+    PRIMARY KEY (service_provider_limits_sid)
+    )`,
+    `CREATE TABLE account_limits
+    (
+    account_limits_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    category ENUM('api_rate','voice_call_session', 'device') NOT NULL,
+    quantity INTEGER NOT NULL,
+    PRIMARY KEY (account_limits_sid)
+    )`,
+    'CREATE INDEX service_provider_sid_idx ON service_provider_limits (service_provider_sid)',
+    `ALTER TABLE service_provider_limits 
+    ADD FOREIGN KEY service_provider_sid_idxfk_3 (service_provider_sid) 
+    REFERENCES service_providers (service_provider_sid) 
+    ON DELETE CASCADE`,
+    'CREATE INDEX account_sid_idx ON account_limits (account_sid)',
+    `ALTER TABLE account_limits 
+    ADD FOREIGN KEY account_sid_idxfk_2 (account_sid) 
+    REFERENCES accounts (account_sid) 
+    ON DELETE CASCADE`,
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_from_user` VARCHAR(128)',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_from_domain` VARCHAR(256)',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_public_ip_in_contact` BOOLEAN NOT NULL DEFAULT false'
+  ],
+  '8000': [
+    'ALTER TABLE `applications` ADD COLUMN `app_json` TEXT',
+    'ALTER TABLE voip_carriers CHANGE register_public_domain_in_contact register_public_ip_in_contact BOOLEAN',
+    'alter table phone_numbers modify number varchar(132) NOT NULL UNIQUE',
+    `CREATE TABLE permissions
+    (
+    permission_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(32) NOT NULL UNIQUE ,
+    description VARCHAR(255),
+    PRIMARY KEY (permission_sid)
+    )`,
+    `CREATE TABLE user_permissions
+    (
+    user_permissions_sid CHAR(36) NOT NULL UNIQUE ,
+    user_sid CHAR(36) NOT NULL,
+    permission_sid CHAR(36) NOT NULL,
+    PRIMARY KEY (user_permissions_sid)
+    )`,
+    `CREATE TABLE password_settings
+    (
+    min_password_length INTEGER NOT NULL DEFAULT 8,
+    require_digit BOOLEAN NOT NULL DEFAULT false,
+    require_special_character BOOLEAN NOT NULL DEFAULT false
+    )`,
+    'CREATE INDEX user_permissions_sid_idx ON user_permissions (user_permissions_sid)',
+    'CREATE INDEX user_sid_idx ON user_permissions (user_sid)',
+    'ALTER TABLE user_permissions ADD FOREIGN KEY user_sid_idxfk (user_sid) REFERENCES users (user_sid) ON DELETE CASCADE',
+    'ALTER TABLE user_permissions ADD FOREIGN KEY permission_sid_idxfk (permission_sid) REFERENCES permissions (permission_sid)',
+    'ALTER TABLE `users` ADD COLUMN `is_active` BOOLEAN NOT NULL default true',
+  ],
+  8003: [
+    'SET FOREIGN_KEY_CHECKS=0',
+    'ALTER TABLE `voip_carriers` ADD COLUMN `register_status` VARCHAR(4096)',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `last_updated` DATETIME',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `tls_port` INTEGER',
+    'ALTER TABLE `sbc_addresses` ADD COLUMN `wss_port` INTEGER',
+    `CREATE TABLE system_information
+    (
+    domain_name VARCHAR(255),
+    sip_domain_name VARCHAR(255),
+    monitoring_domain_name VARCHAR(255)
+    )`,
+    'DROP TABLE IF EXISTS `lcr_routes`',
+    'DROP TABLE IF EXISTS `lcr_carrier_set_entry`',
+    `CREATE TABLE lcr_routes
+    (
+    lcr_route_sid CHAR(36),
+    lcr_sid CHAR(36) NOT NULL,
+    regex VARCHAR(32) NOT NULL COMMENT 'regex-based pattern match against dialed number, used for LCR routing of PSTN calls',
+    description VARCHAR(1024),
+    priority INTEGER NOT NULL COMMENT 'lower priority routes are attempted first',
+    PRIMARY KEY (lcr_route_sid)
+    )`,
+    `CREATE TABLE lcr
+    (
+    lcr_sid CHAR(36) NOT NULL UNIQUE ,
+    name VARCHAR(64) COMMENT 'User-assigned name for this LCR table',
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    default_carrier_set_entry_sid CHAR(36) COMMENT 'default carrier/route to use when no digit match based results are found.',
+    service_provider_sid CHAR(36),
+    account_sid CHAR(36),
+    PRIMARY KEY (lcr_sid)
+    )`,
+    `CREATE TABLE lcr_carrier_set_entry
+    (
+    lcr_carrier_set_entry_sid CHAR(36),
+    workload INTEGER NOT NULL DEFAULT 1 COMMENT 'represents a proportion of traffic to send through the associated carrier; can be used for load balancing traffic across carriers with a common priority for a destination',
+    lcr_route_sid CHAR(36) NOT NULL,
+    voip_carrier_sid CHAR(36) NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 0 COMMENT 'lower priority carriers are attempted first',
+    PRIMARY KEY (lcr_carrier_set_entry_sid)
+    )`,
+    'CREATE INDEX lcr_sid_idx ON lcr_routes (lcr_sid)',
+    'ALTER TABLE lcr_routes ADD FOREIGN KEY lcr_sid_idxfk (lcr_sid) REFERENCES lcr (lcr_sid)',
+    'CREATE INDEX lcr_sid_idx ON lcr (lcr_sid)',
+    'ALTER TABLE lcr ADD FOREIGN KEY default_carrier_set_entry_sid_idxfk (default_carrier_set_entry_sid) REFERENCES lcr_carrier_set_entry (lcr_carrier_set_entry_sid)',
+    'CREATE INDEX service_provider_sid_idx ON lcr (service_provider_sid)',
+    'CREATE INDEX account_sid_idx ON lcr (account_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY lcr_route_sid_idxfk (lcr_route_sid) REFERENCES lcr_routes (lcr_route_sid)',
+    'ALTER TABLE lcr_carrier_set_entry ADD FOREIGN KEY voip_carrier_sid_idxfk_3 (voip_carrier_sid) REFERENCES voip_carriers (voip_carrier_sid)',
+    'SET FOREIGN_KEY_CHECKS=1',
+  ],
+  8004: [
+    'alter table accounts add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table accounts add column bucket_credential VARCHAR(8192)',
+    'alter table accounts add column record_format VARCHAR(16) NOT NULL DEFAULT \'mp3\'',
+    'alter table applications add column record_all_calls BOOLEAN NOT NULL DEFAULT false',
+    'alter table phone_numbers DROP INDEX number',
+    'create unique index phone_numbers_unique_idx_voip_carrier_number ON phone_numbers (number,voip_carrier_sid)',
+    `CREATE TABLE clients
+    (
+    client_sid CHAR(36) NOT NULL UNIQUE ,
+    account_sid CHAR(36) NOT NULL,
+    is_active BOOLEAN NOT NULL DEFAULT 1,
+    username VARCHAR(64),
+    password VARCHAR(1024),
+    PRIMARY KEY (client_sid)
+    )`,
+    'CREATE INDEX client_sid_idx ON clients (client_sid)',
+    'ALTER TABLE clients ADD CONSTRAINT account_sid_idxfk_13 FOREIGN KEY account_sid_idxfk_13 (account_sid) REFERENCES accounts (account_sid)',
+    'ALTER TABLE sip_gateways ADD COLUMN protocol ENUM(\'udp\',\'tcp\',\'tls\', \'tls/srtp\') DEFAULT \'udp\''
+  ],
+  8005: [
+    'DROP INDEX speech_credentials_idx_1 ON speech_credentials',
+    'ALTER TABLE speech_credentials ADD COLUMN label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN use_for_fallback_speech BOOLEAN DEFAULT false',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_language VARCHAR(12)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_voice VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_synthesis_label VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_vendor VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_language VARCHAR(64)',
+    'ALTER TABLE applications ADD COLUMN fallback_speech_recognizer_label VARCHAR(64)',
+    'ALTER TABLE sip_gateways ADD COLUMN pad_crypto BOOLEAN NOT NULL DEFAULT 0',
+    'ALTER TABLE sip_gateways MODIFY port INTEGER'
+  ]
+};
+
+const doIt = async() => {
+  let connection;
+  try {
+    logger.info({opts}, 'connecting to mysql database..');
+    connection = await mysql.createConnection(opts);
+  } catch (err) {
+    logger.error({err}, 'Error connecting to database with provided env vars');
+    process.exit(1);
+  }
+
+  try {
+    /* does the schema exist at all ? */
+    const [r] = await connection.execute('SELECT version from schema_version');
+    let errors = 0;
+    if (r.length) {
+      const {version} = r[0];
+      const arr = /v?(\d+)\.(\d+)\.(\d+)/.exec(version);
+      if (arr) {
+        const upgrades = [];
+        logger.info(`performing schema migration: ${version} => ${desiredVersion}`);
+        const val = (1000 * arr[1]) + (100 * arr[2]) + arr[3];
+        logger.info(`current schema value: ${val}`);
+
+        if (val < 7006) upgrades.push(...sql['7006']);
+        if (val < 7007) upgrades.push(...sql['7007']);
+        if (val < 8000) upgrades.push(...sql['8000']);
+        if (val < 8003) upgrades.push(...sql['8003']);
+        if (val < 8004) upgrades.push(...sql['8004']);
+        if (val < 8005) upgrades.push(...sql['8005']);
+
+        // perform all upgrades
+        logger.info({upgrades}, 'applying schema upgrades..');
+        for (const upgrade of upgrades) {
+          try {
+            await connection.execute(upgrade);
+          } catch (err) {
+            errors++;
+            logger.info({statement:upgrade, err}, 'Error applying statement');
+          }
+        }
+      }
+      if (errors === 0) await connection.execute(`UPDATE schema_version SET version = '${desiredVersion}'`);
+      await connection.end();
+      logger.info(`schema migration to ${desiredVersion} completed with ${errors} errors`);
+      return;
+    }
+  } catch (err) {
+  }
+  try {
+    await createSchema(connection);
+    await seedDatabase(connection);
+    logger.info('reset admin password..');
+    execSync(`${__dirname}/../db/reset_admin_password.js`);
+    await connection.query(`INSERT into schema_version (version) values('${desiredVersion}')`);
+    logger.info('database install/upgrade complete.');
+    await connection.end();
+  } catch (err) {
+    logger.error({err}, 'Error seeding database');
+    process.exit(1);
+  }
+};
+
+const createSchema = async(connection) => {
+  logger.info('reading schema..');
+  const sql = await readFile(`${__dirname}/../db/jambones-sql.sql`, {encoding: 'utf8'});
+  logger.info('creating schema..');
+  await connection.query(sql);
+};
+
+const seedDatabase = async(connection) => {
+  const sql = await readFile(`${__dirname}/../db/seed-production-database-open-source.sql`, {encoding: 'utf8'});
+  logger.info('seeding data..');
+  await connection.query(sql);
+};
+
+
+doIt();
+

db/webapp-tests.sql

@@ -2,7 +2,7 @@ SET FOREIGN_KEY_CHECKS=0;
 
 -- create one service provider
 insert into service_providers (service_provider_sid, name, description, root_domain) 
-values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.us', 'jambonz.us service provider', 'sip.yakeeda.com');
+values ('2708b1b3-2736-40ea-b502-c53d8396247f', 'jambonz.cloud', 'jambonz.cloud service provider', 'sip.yakeeda.com');
 insert into api_keys (api_key_sid, token) 
 values ('3f35518f-5a0d-4c2e-90a5-2407bb3b36f0', '38700987-c7a4-4685-a5bb-af378f9734de');
 
@@ -19,8 +19,8 @@ insert into sip_gateways (sip_gateway_sid, voip_carrier_sid, ipv4, port, inbound
 values ('46b727eb-c7dc-44fa-b063-96e48d408e4a', '5145b436-2f38-4029-8d4c-fd8c67831c7a', '3.3.3.3', 5060, 1, 1, 1);
 
 -- create the test application and test phone number
-insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.us/testCall', 'POST');
-insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.us/callStatus', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('d9c205c6-a129-443e-a9c0-d1bb437d4bb7', 'https://flows.jambonz.cloud/testCall', 'POST');
+insert into webhooks (webhook_sid, url, method) values ('6ac36aeb-6bd0-428a-80a1-aed95640a296', 'https://flows.jambonz.cloud/callStatus', 'POST');
 insert into applications (application_sid, name, service_provider_sid, call_hook_sid, call_status_hook_sid, 
 speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice, speech_recognizer_vendor, speech_recognizer_language)
 values ('7a489343-02ed-471e-8df0-fc5e1b98ce8f', 'Test application', '2708b1b3-2736-40ea-b502-c53d8396247f', 
@@ -85,9 +85,6 @@ VALUES
 -- simwood gateways
 insert into predefined_sip_gateways (predefined_sip_gateway_sid, predefined_carrier_sid, ipv4, netmask, port, inbound, outbound)
 VALUES
-('91cb050f-9826-4ac9-b736-84a10372a9fe', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '149.91.14.0', 24, 5060, 1, 0),
-('58700fad-98bf-4d31-b61e-888c54911b35', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '154.51.137.96', 27, 5060, 1, 0),
-('d020fd9e-7fdb-4bca-ae0d-e61b38142873', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '78.40.245.160', 27, 5060, 1, 0),
 ('441fd2e7-c845-459c-963d-6e917063ed9a', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.136.24', 29, 5060, 1, 0),
 ('1e0d3e80-9973-4184-9bec-07ae564f983f', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.136.28', 28, 5060, 1, 0),
 ('e56ec745-5f37-443f-afb4-7bbda31ae7ac', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.140.48', 28, 5060, 1, 0),
@@ -98,7 +95,7 @@ VALUES
 ('b6ae6240-55ac-4c11-892f-a71b2155ea60', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.142.0', 26, 5060, 1, 0),
 ('5a976337-164b-408e-8748-d8bfb4bd5d76', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '185.63.143.0', 26, 5060, 1, 0),
 ('ed0434ca-7f26-4624-9523-0419d0d2924d', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '178.22.139.0', 26, 5060, 1, 0),
-('d1a594c2-c14f-4ead-b621-96129bc87886', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.224.0', 24, 5060, 1, 0),
+('6bfb55e5-e248-48dc-a104-4f3eedd7d7de', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', '172.86.225.0', 24, 5060, 1, 0),
 ('5f431d42-48e4-44ce-a311-d946f0b475b6', 'e6fb301a-1af0-4fb8-a1f6-f65530c6e1c6', 'out.simwood.com', 32, 5060, 0, 1);
 
 

lib/auth/index.js

@@ -1,54 +1,56 @@
 const Strategy = require('passport-http-bearer').Strategy;
 const {getMysqlConnection} = require('../db');
-const {hashString} = require('../utils/password-utils');
 const debug = require('debug')('jambonz:api-server');
+const {cacheClient} = require('../helpers');
 const jwt = require('jsonwebtoken');
 const sql = `
   SELECT *
   FROM api_keys
   WHERE api_keys.token = ?`;
 
-function makeStrategy(logger, retrieveKey) {
+function makeStrategy(logger) {
   return new Strategy(
     async function(token, done) {
-      logger.debug(`validating with token ${token}`);
       jwt.verify(token, process.env.JWT_SECRET, async(err, decoded) => {
         if (err) {
           if (err.name === 'TokenExpiredError') {
             logger.debug('jwt expired');
             return done(null, false);
           }
-          /* its not a jwt obtained through login, check api leys */
+          /* its not a jwt obtained through login, check api keys */
           checkApiTokens(logger, token, done);
         }
         else {
-          /* validated -- make sure it is not on blacklist */
           try {
-            const s = `jwt:${hashString(token)}`;
-            const result = await retrieveKey(s);
-            if (result) {
-              debug(`result from searching for ${s}: ${result}`);
+            const {user_sid} = decoded;
+            /* Valid jwt tokens are stored in redis by hashed user_id */
+            const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+            const result = await cacheClient.get(redisKey);
+            if (result === null) {
+              debug(`result from searching for ${redisKey}: ${result}`);
               logger.info('jwt invalidated after logout');
               return done(null, false);
             }
-          } catch (err) {
+          } catch (error) {
             debug(err);
-            logger.info({err}, 'Error checking blacklist for jwt');
+            logger.info({err}, 'Error checking redis for jwt');
           }
-          const {user_sid, account_sid, email, name} = decoded;
-          //logger.debug({user_sid, account_sid}, 'successfully validated jwt');
-          const scope = ['account'];
+          const { user_sid, service_provider_sid, account_sid, email, name, scope, permissions } = decoded;
+
           const user = {
+            service_provider_sid,
             account_sid,
             user_sid,
             jwt: token,
             email,
             name,
-            hasScope: (s) => s === 'account',
-            hasAdminAuth: false,
-            hasServiceProviderAuth: false,
-            hasAccountAuth: true
+            permissions,
+            hasScope: (s) => s === scope,
+            hasAdminAuth: scope === 'admin',
+            hasServiceProviderAuth: scope === 'service_provider',
+            hasAccountAuth: scope === 'account'
           };
+          logger.debug({user}, 'successfully validated jwt');
           return done(null, user, {scope});
         }
       });
@@ -75,26 +77,30 @@ const checkApiTokens = (logger, token, done) => {
       }
 
       // found api key
-      const scope = [];
+      let scope;
+      //const scope = [];
       if (results[0].account_sid === null && results[0].service_provider_sid === null) {
-        scope.push.apply(scope, ['admin', 'service_provider', 'account']);
+        //scope.push.apply(scope, ['admin', 'service_provider', 'account']);
+        scope = 'admin';
       }
       else if (results[0].service_provider_sid) {
-        scope.push.apply(scope, ['service_provider', 'account']);
+        //scope.push.apply(scope, ['service_provider', 'account']);
+        scope = 'service_provider';
       }
       else {
-        scope.push('account');
+        //scope.push('account');
+        scope = 'account';
       }
 
       const user = {
         account_sid: results[0].account_sid,
         service_provider_sid: results[0].service_provider_sid,
-        hasScope: (s) => scope.includes(s),
-        hasAdminAuth: scope.length === 3,
-        hasServiceProviderAuth: scope.includes('service_provider'),
-        hasAccountAuth: scope.includes('account') && !scope.includes('service_provider')
+        hasScope: (s) => s === scope,
+        hasAdminAuth: scope === 'admin',
+        hasServiceProviderAuth: scope === 'service_provider',
+        hasAccountAuth: scope === 'account'
       };
-      logger.info(user, `successfully validated with scope ${scope}`);
+      logger.debug({user}, `successfully validated with scope ${scope}`);
       return done(null, user, {scope});
     });
   });

lib/helpers/cache-client.js

@@ -0,0 +1,82 @@
+const {
+  addKey: addKeyRedis,
+  deleteKey: deleteKeyRedis,
+  retrieveKey: retrieveKeyRedis,
+} = require('./realtimedb-helpers');
+const { hashString } = require('../utils/password-utils');
+const logger = require('../logger');
+
+class CacheClient {
+  constructor() { }
+
+  async set(params) {
+    const {
+      redisKey,
+      value = '1',
+      time = 3600,
+    } = params || {};
+
+    try {
+      await addKeyRedis(redisKey, value, time);
+
+    } catch (err) {
+      logger.error('CacheClient.get set', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        ...params
+      });
+    }
+  }
+
+  async get(redisKey) {
+    try {
+      const result = await retrieveKeyRedis(redisKey);
+      return result;
+    } catch (err) {
+      logger.error('CacheClient.get error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        redisKey
+      });
+    }
+  }
+
+
+  async delete(key) {
+    try {
+      await deleteKeyRedis(key);
+      logger.debug('CacheClient.delete key from redis', { key });
+    } catch (err) {
+      logger.error('CacheClient.delete error', {
+        error: {
+          message: err.message,
+          name: err.name
+        },
+        key
+      });
+    }
+  }
+
+  generateRedisKey(type, key, version) {
+    let suffix = '';
+    if (version) {
+      suffix = `:version:${version}`;
+    }
+
+    switch (type) {
+      case 'reset-link':
+        return `reset-link:${key}`;
+      case 'jwt':
+      default:
+        return `jwt:${hashString(key)}${suffix}`;
+    }
+  }
+}
+
+const cacheClient = new CacheClient();
+
+module.exports = { cacheClient };

lib/helpers/index.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require('./cache-client'),
+  ...require('./realtimedb-helpers'),
+};

lib/helpers/realtimedb-helpers.js

@@ -0,0 +1,54 @@
+const logger = require('../logger');
+
+const JAMBONES_REDIS_SENTINELS = process.env.JAMBONES_REDIS_SENTINELS ? {
+  sentinels: process.env.JAMBONES_REDIS_SENTINELS.split(',').map((sentinel) => {
+    let host, port = 26379;
+    if (sentinel.includes(':')) {
+      const arr = sentinel.split(':');
+      host = arr[0];
+      port = parseInt(arr[1], 10);
+    } else {
+      host = sentinel;
+    }
+    return {host, port};
+  }),
+  name: process.env.JAMBONES_REDIS_SENTINEL_MASTER_NAME,
+  ...(process.env.JAMBONES_REDIS_SENTINEL_PASSWORD && {
+    password: process.env.JAMBONES_REDIS_SENTINEL_PASSWORD
+  }),
+  ...(process.env.JAMBONES_REDIS_SENTINEL_USERNAME && {
+    username: process.env.JAMBONES_REDIS_SENTINEL_USERNAME
+  })
+} : null;
+
+const {
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  incrKey,
+  client: redisClient,
+} = require('@jambonz/realtimedb-helpers')(JAMBONES_REDIS_SENTINELS || {
+  host: process.env.JAMBONES_REDIS_HOST || 'localhost',
+  port: process.env.JAMBONES_REDIS_PORT || 6379
+}, logger);
+
+module.exports = {
+  retrieveCall,
+  deleteCall,
+  listCalls,
+  listSortedSets,
+  purgeCalls,
+  retrieveSet,
+  addKey,
+  retrieveKey,
+  deleteKey,
+  redisClient,
+  incrKey,
+  JAMBONES_REDIS_SENTINELS
+};

lib/logger.js

@@ -0,0 +1,7 @@
+const opts = {
+  level: process.env.JAMBONES_LOGLEVEL || 'info'
+};
+const pino = require('pino');
+const logger = pino(opts, pino.destination(1, {sync: false}));
+
+module.exports = logger;

lib/middleware.js

@@ -0,0 +1,32 @@
+const logger = require('./logger');
+
+function delayLoginMiddleware(req, res, next) {
+  if (req.path.includes('/login') || req.path.includes('/signin')) {
+    const min = 200;
+    const max = 1000;
+    /* Random delay between 200 - 1000ms */
+    const sendStatusDelay = Math.floor(Math.random() * (max - min + 1)) + min;
+
+    /* the res.json take longer, we decrease the max delay slightly to 0-800ms */
+    const jsonDelay = Math.floor(Math.random() * 800);
+    logger.debug(`delayLoginMiddleware: sendStatus ${sendStatusDelay} - json ${jsonDelay}`);
+    const sendStatus = res.sendStatus;
+    const json = res.json;
+
+    res.sendStatus = function(status) {
+      setTimeout(() => {
+        sendStatus.call(res, status);
+      }, sendStatusDelay);
+    };
+    res.json = function(body) {
+      setTimeout(() => {
+        json.call(res, body);
+      }, jsonDelay);
+    };
+  }
+  next();
+}
+
+module.exports = {
+  delayLoginMiddleware
+};

lib/models/account-limits.js

@@ -0,0 +1,41 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sql = 'SELECT * FROM account_limits WHERE account_sid = ?';
+
+class AccountLimits extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieve(account_sid) {
+    const [rows] = await promisePool.query(sql, [account_sid]);
+    return rows;
+  }
+
+}
+
+AccountLimits.table = 'account_limits';
+AccountLimits.fields = [
+  {
+    name: 'account_limits_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'category',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'quantity',
+    type: 'number',
+    required: true
+  }
+];
+
+module.exports = AccountLimits;

lib/models/account.js

@@ -2,8 +2,9 @@ const debug = require('debug')('jambonz:api-server');
 const Model = require('./model');
 const {getMysqlConnection} = require('../db');
 const {promisePool} = require('../db');
-const uuid = require('uuid').v4;
-const {encrypt} = require('../utils/encrypt-decrypt');
+const { v4: uuid } = require('uuid');
+
+const {encrypt, decrypt} = require('../utils/encrypt-decrypt');
 
 const retrieveSql = `SELECT * from accounts acc
 LEFT JOIN webhooks AS rh
@@ -33,7 +34,7 @@ AND effective_end_date IS NULL
 AND pending=0`;
 
 const updatePaymentInfoSql = `UPDATE account_subscriptions 
-SET last4 = ?, exp_month = ?, exp_year = ?, card_type = ? 
+SET last4 = ?, stripe_payment_method_id=?, exp_month = ?, exp_year = ?, card_type = ? 
 WHERE account_sid = ? 
 AND effective_end_date IS NULL`;
 
@@ -54,6 +55,13 @@ WHERE account_sid = ?
 AND effective_end_date IS NULL 
 AND pending = 0`;
 
+const extractBucketCredential = (obj) => {
+  const {bucket_credential} = obj;
+  if (bucket_credential) {
+    obj.bucket_credential = JSON.parse(decrypt(bucket_credential));
+  }
+};
+
 function transmogrifyResults(results) {
   return results.map((row) => {
     const obj = row.acc;
@@ -74,6 +82,8 @@ function transmogrifyResults(results) {
     else obj.queue_event_hook = null;
     delete obj.queue_event_hook_sid;
 
+    extractBucketCredential(obj);
+
     return obj;
   });
 }
@@ -196,10 +206,10 @@ class Account extends Model {
   }
 
   static async updatePaymentInfo(logger, account_sid, pm) {
-    const {card} = pm;
+    const {id, card} = pm;
     const last4_encrypted = encrypt(card.last4);
     await promisePool.execute(updatePaymentInfoSql,
-      [last4_encrypted, card.exp_month, card.exp_year, card.brand, account_sid]);
+      [last4_encrypted, id, card.exp_month, card.exp_year, card.brand, account_sid]);
   }
 
   static async provisionPendingSubscription(logger, account_sid, products, payment_method, subscription_id) {
@@ -237,7 +247,6 @@ class Account extends Model {
     }));
     return account_subscription_sid;
   }
-
 }
 
 Account.table = 'accounts';
@@ -296,6 +305,38 @@ Account.fields = [
   {
     name: 'disable_cdrs',
     type: 'number',
+  },
+  {
+    name: 'subspace_client_id',
+    type: 'string',
+  },
+  {
+    name: 'subspace_client_secret',
+    type: 'string',
+  },
+  {
+    name: 'subspace_sip_teleport_id',
+    type: 'string',
+  },
+  {
+    name: 'subspace_sip_teleport_destinations',
+    type: 'string',
+  },
+  {
+    name: 'siprec_hook_sid',
+    type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number'
+  },
+  {
+    name: 'record_format',
+    type: 'string'
+  },
+  {
+    name: 'bucket_credential',
+    type: 'string'
   }
 ];
 

lib/models/api-key.js

@@ -27,6 +27,25 @@ class ApiKey extends Model {
     });
   }
 
+  /**
+  * list all api keys for a service provider
+  */
+  static retrieveAllForSP(service_provider_sid) {
+    const sql = 'SELECT * from api_keys WHERE service_provider_sid = ?';
+    const args = [service_provider_sid];
+
+    return new Promise((resolve, reject) => {
+      getMysqlConnection((err, conn) => {
+        if (err) return reject(err);
+        conn.query(sql, args, (err, results) => {
+          conn.release();
+          if (err) return reject(err);
+          resolve(results);
+        });
+      });
+    });
+  }
+
   /**
   * update last_used api key for an account
   */

lib/models/application.js

@@ -120,6 +120,10 @@ Application.fields = [
   {
     name: 'messaging_hook_sid',
     type: 'string',
+  },
+  {
+    name: 'record_all_calls',
+    type: 'number',
   }
 ];
 

lib/models/client.js

@@ -0,0 +1,58 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Client extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(service_provider_sid) {
+    const sql = `SELECT c.client_sid, c.account_sid, c.is_active, c.username, c.hashed_password
+      FROM ${this.table} AS c LEFT JOIN accounts AS acc ON c.account_sid = acc.account_sid
+      LEFT JOIN service_providers AS sp ON sp.service_provider_sid = accs.service_provider_sid
+      WHERE sp.service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, service_provider_sid);
+    return rows;
+  }
+
+  static async retrieveByAccountSidAndUserName(account_sid, username) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ? AND username = ?`;
+    const [rows] = await promisePool.query(sql, [account_sid, username]);
+    return rows;
+  }
+}
+
+Client.table = 'clients';
+Client.fields = [
+  {
+    name: 'client_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+  {
+    name: 'username',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'password',
+    type: 'string'
+  }
+];
+
+module.exports = Client;

lib/models/lcr-carrier-set-entry.js

@@ -0,0 +1,47 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class LcrCarrierSetEntry extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrRouteSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_route_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrRouteSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_route_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+}
+
+LcrCarrierSetEntry.table = 'lcr_carrier_set_entry';
+LcrCarrierSetEntry.fields = [
+  {
+    name: 'lcr_carrier_set_entry_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'workload',
+    type: 'number'
+  },
+  {
+    name: 'lcr_route_sid',
+    type: 'string'
+  },
+  {
+    name: 'voip_carrier_sid',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrCarrierSetEntry;

lib/models/lcr-route.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+
+class LcrRoutes extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByLcrSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE lcr_sid = ? ORDER BY priority`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async deleteByLcrSid(sid) {
+    const sql = `DELETE FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.affectedRows;
+  }
+
+  static async countAllByLcrSid(sid) {
+    const sql = `SELECT COUNT(*) AS count FROM ${this.table} WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows.length ? rows[0].count : 0;
+  }
+}
+
+LcrRoutes.table = 'lcr_routes';
+LcrRoutes.fields = [
+  {
+    name: 'lcr_route_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
+  },
+  {
+    name: 'regex',
+    type: 'string'
+  },
+  {
+    name: 'description',
+    type: 'string'
+  },
+  {
+    name: 'priority',
+    type: 'number'
+  }
+];
+
+module.exports = LcrRoutes;

lib/models/lcr.js

@@ -0,0 +1,54 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+
+class Lcr extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAllByAccountSid(account_sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE account_sid = ?`;
+    const [rows] = await promisePool.query(sql, account_sid);
+    return rows;
+  }
+
+  static async retrieveAllByServiceProviderSid(sid) {
+    const sql = `SELECT * FROM ${this.table} WHERE service_provider_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+
+  static async releaseDefaultEntry(sid) {
+    const sql = `UPDATE ${this.table} SET default_carrier_set_entry_sid = null WHERE lcr_sid = ?`;
+    const [rows] = await promisePool.query(sql, sid);
+    return rows;
+  }
+}
+
+Lcr.table = 'lcr';
+Lcr.fields = [
+  {
+    name: 'lcr_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'account_sid',
+    type: 'string'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  },
+  {
+    name: 'default_carrier_set_entry_sid',
+    type: 'string'
+  }
+];
+
+module.exports = Lcr;

lib/models/model.js

@@ -1,5 +1,5 @@
 const Emitter = require('events');
-const uuidv4 = require('uuid/v4');
+const { v4: uuidv4 } = require('uuid');
 const assert = require('assert');
 const {getMysqlConnection} = require('../db');
 const {DbErrorBadRequest} = require('../utils/errors');
@@ -107,7 +107,7 @@ class Model extends Emitter {
       if (pk.name in obj) throw new DbErrorBadRequest(`primary key ${pk.name} is immutable`);
       getMysqlConnection((err, conn) => {
         if (err) return reject(err);
-        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = '${sid}'`, obj, (err, results, fields) => {
+        conn.query(`UPDATE ${this.table} SET ? WHERE ${pk.name} = ?`, [obj, sid], (err, results, fields) => {
           conn.release();
           if (err) return reject(err);
           resolve(results.affectedRows);

lib/models/password-settings.js

@@ -0,0 +1,70 @@
+const {promisePool} = require('../db');
+
+class PasswordSettings {
+
+  /**
+   * Retrieve object from database
+   */
+  static async retrieve() {
+    const [r] = await promisePool.execute(`SELECT * FROM ${this.table}`);
+    return r;
+  }
+
+  /**
+  * Update object into the database
+  */
+  static async update(obj) {
+    let sql = `UPDATE ${this.table} SET `;
+    const values = [];
+    const keys = Object.keys(obj);
+    this.fields.forEach(({name}) => {
+      if (keys.includes(name)) {
+        sql = sql + `${name} = ?,`;
+        values.push(obj[name]);
+      }
+    });
+    if (values.length) {
+      sql = sql.slice(0, -1);
+      await promisePool.execute(sql, values);
+    }
+  }
+
+  /**
+     * insert object into the database
+     */
+  static async make(obj) {
+    let params = '', marks = '';
+    const values = [];
+    const keys = Object.keys(obj);
+    this.fields.forEach(({name}) => {
+      if (keys.includes(name)) {
+        params = params + `${name},`;
+        marks = marks + '?,';
+        values.push(obj[name]);
+      }
+    });
+    if (values.length) {
+      params = `(${params.slice(0, -1)})`;
+      marks = `values(${marks.slice(0, -1)})`;
+      return await promisePool.execute(`INSERT into ${this.table} ${params} ${marks}`, values);
+    }
+  }
+}
+
+
+PasswordSettings.table = 'password_settings';
+PasswordSettings.fields = [
+  {
+    name: 'min_password_length',
+    type: 'number'
+  },
+  {
+    name: 'require_digit',
+    type: 'number'
+  },
+  {
+    name: 'require_special_character',
+    type: 'number'
+  }
+];
+module.exports = PasswordSettings;

lib/models/phone-number.js

@@ -1,6 +1,6 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
-const sql = 'SELECT * from phone_numbers WHERE account_sid = ?';
+const sql = 'SELECT * from phone_numbers WHERE account_sid = ? ORDER BY number';
 const sqlSP = `SELECT * 
 FROM phone_numbers 
 WHERE account_sid IN 
@@ -8,7 +8,7 @@ WHERE account_sid IN
   SELECT account_sid 
   FROM accounts 
   WHERE service_provider_sid = ?
-)`;
+) ORDER BY number`;
 
 class PhoneNumber extends Model {
   constructor() {
@@ -16,7 +16,7 @@ class PhoneNumber extends Model {
   }
 
   static async retrieveAll(account_sid) {
-    if (!account_sid) return super.retrieveAll();
+    if (!account_sid) return await super.retrieveAll();
     const [rows] = await promisePool.query(sql, account_sid);
     return rows;
   }

lib/models/service-provider-limits.js

@@ -0,0 +1,39 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sql = 'SELECT * FROM service_provider_limits WHERE service_provider_sid = ?';
+
+class ServiceProviderLimits extends Model {
+  constructor() {
+    super();
+  }
+  static async retrieve(service_provider_sid) {
+    const [rows] = await promisePool.query(sql, [service_provider_sid]);
+    return rows;
+  }
+}
+
+ServiceProviderLimits.table = 'service_provider_limits';
+ServiceProviderLimits.fields = [
+  {
+    name: 'service_provider_limits_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'category',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'quantity',
+    type: 'number',
+    required: true
+  }
+];
+
+module.exports = ServiceProviderLimits;

lib/models/service-provider.js

@@ -89,6 +89,10 @@ ServiceProvider.fields = [
   {
     name: 'ms_teams_fqdn',
     type: 'string',
+  },
+  {
+    name: 'lcr_sid',
+    type: 'string'
   }
 
 ];

lib/models/sip-gateway.js

@@ -51,6 +51,10 @@ SipGateway.fields = [
     name: 'is_active',
     type: 'number'
   },
+  {
+    name: 'pad_crypto',
+    type: 'number'
+  },
   {
     name: 'account_sid',
     type: 'string'
@@ -58,6 +62,10 @@ SipGateway.fields = [
   {
     name: 'application_sid',
     type: 'string'
+  },
+  {
+    name: 'protocol',
+    type: 'string'
   }
 ];
 

lib/models/speech-credential.js

@@ -1,7 +1,7 @@
 const Model = require('./model');
 const {promisePool} = require('../db');
 const retrieveSql = 'SELECT * from speech_credentials WHERE account_sid = ?';
-const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ?';
+const retrieveSqlForSP = 'SELECT * from speech_credentials WHERE service_provider_sid = ? and account_sid is null';
 
 class SpeechCredential extends Model {
   constructor() {
@@ -20,6 +20,17 @@ class SpeechCredential extends Model {
     return rows;
   }
 
+  static async isAvailableVendorAndLabel(service_provider_sid, account_sid, vendor, label) {
+    let sql;
+    if (account_sid) {
+      sql = 'SELECT * FROM speech_credentials WHERE account_sid = ? AND vendor = ? AND label = ?';
+    } else {
+      sql = 'SELECT * FROM speech_credentials WHERE service_provider_sid = ? AND vendor = ? AND label = ?';
+    }
+    const [rows] = await promisePool.query(sql, [account_sid ? account_sid : service_provider_sid, vendor, label]);
+    return rows;
+  }
+
   static async disableStt(account_sid) {
     await promisePool.execute('UPDATE speech_credentials SET use_for_stt = 0 WHERE account_sid = ?', [account_sid]);
   }
@@ -86,6 +97,10 @@ SpeechCredential.fields = [
   {
     name: 'last_tested',
     type: 'date'
+  },
+  {
+    name: 'label',
+    type: 'string'
   }
 ];
 

lib/models/system-information.js

@@ -0,0 +1,38 @@
+const Model = require('./model');
+const { promisePool } = require('../db');
+class SystemInformation extends Model {
+  constructor() {
+    super();
+  }
+
+  static async add(body) {
+    let [sysInfo] = await this.retrieveAll();
+    if (sysInfo) {
+      const sql = `UPDATE ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    } else {
+      const sql = `INSERT INTO ${this.table} SET ?`;
+      await promisePool.query(sql, body);
+    }
+    [sysInfo] = await this.retrieveAll();
+    return sysInfo;
+  }
+}
+
+SystemInformation.table = 'system_information';
+SystemInformation.fields = [
+  {
+    name: 'domain_name',
+    type: 'string',
+  },
+  {
+    name: 'sip_domain_name',
+    type: 'string',
+  },
+  {
+    name: 'monitoring_domain_name',
+    type: 'string',
+  },
+];
+
+module.exports = SystemInformation;

lib/models/user.js

@@ -0,0 +1,117 @@
+const Model = require('./model');
+const {promisePool} = require('../db');
+const sqlAll = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+`;
+const sqlAccount = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+WHERE u.account_sid = ? 
+`;
+const sqlSP = `
+SELECT u.user_sid, u.name, u.email, u.account_sid, u.service_provider_sid, u.is_active, 
+u.force_change, u.phone, u.pending_email, u.provider, u.provider_userid, 
+u.email_activation_code, u.email_validated, 
+sp.name as service_provider_name, acc.name as account_name  
+FROM users u   
+LEFT JOIN service_providers as sp ON u.service_provider_sid = sp.service_provider_sid 
+LEFT JOIN accounts acc ON u.account_sid = acc.account_sid  
+WHERE u.service_provider_sid = ? 
+`;
+
+class User extends Model {
+  constructor() {
+    super();
+  }
+
+  static async retrieveAll() {
+    const [rows] = await promisePool.query(sqlAll);
+    return rows;
+  }
+
+  static async retrieveAllForAccount(account_sid) {
+    const [rows] = await promisePool.query(sqlAccount, [account_sid]);
+    return rows;
+  }
+
+  static async retrieveAllForServiceProvider(service_provider_sid) {
+    const [rows] = await promisePool.query(sqlSP, [service_provider_sid]);
+    return rows;
+  }
+}
+
+User.table = 'users';
+User.fields = [
+  {
+    name: 'user_sid',
+    type: 'string',
+    primaryKey: true
+  },
+  {
+    name: 'name',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'email',
+    type: 'string',
+    required: true
+  },
+  {
+    name: 'pending_email',
+    type: 'string'
+  },
+  {
+    name: 'phone',
+    type: 'string'
+  },
+  {
+    name: 'hashed_password',
+    type: 'string'
+  },
+  {
+    name: 'account_sid',
+    type: 'string'
+  },
+  {
+    name: 'service_provider_sid',
+    type: 'string'
+  },
+  {
+    name: 'force_change',
+    type: 'number'
+  },
+  {
+    name: 'provider',
+    type: 'string'
+  },
+  {
+    name: 'provider_userid',
+    type: 'string'
+  },
+  {
+    name: 'email_activation_code',
+    type: 'string'
+  },
+  {
+    name: 'email_validated',
+    type: 'number'
+  },
+  {
+    name: 'is_active',
+    type: 'number'
+  },
+];
+
+module.exports = User;

lib/models/voip-carrier.js

@@ -11,10 +11,16 @@ class VoipCarrier extends Model {
   static async retrieveAll(account_sid) {
     if (!account_sid) return super.retrieveAll();
     const [rows] = await promisePool.query(retrieveSql, account_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
     return rows;
   }
   static async retrieveAllForSP(service_provider_sid) {
     const [rows] = await promisePool.query(retrieveSqlForSP, service_provider_sid);
+    if (rows) {
+      rows.map((r) => r.register_status = JSON.parse(r.register_status || '{}'));
+    }
     return rows;
   }
 }
@@ -111,6 +117,22 @@ VoipCarrier.fields = [
     name: 'smpp_system_id',
     type: 'string'
   },
+  {
+    name: 'register_from_user',
+    type: 'string'
+  },
+  {
+    name: 'register_from_domain',
+    type: 'string'
+  },
+  {
+    name: 'register_public_ip_in_contact',
+    type: 'number'
+  },
+  {
+    name: 'register_status',
+    type: 'string'
+  }
 ];
 
 module.exports = VoipCarrier;

lib/record/azure-storage.js

@@ -0,0 +1,41 @@
+const { Writable } = require('stream');
+const { BlobServiceClient } = require('@azure/storage-blob');
+const { v4: uuidv4 } = require('uuid');
+
+class AzureStorageUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+    this.blockBlobClient = blobServiceClient.getContainerClient(opts.bucketName).getBlockBlobClient(opts.Key);
+    this.metadata = opts.metadata;
+    this.blocks = [];
+  }
+
+  async _write(chunk, encoding, callback) {
+    const blockID = uuidv4().replace(/-/g, '');
+    this.blocks.push(blockID);
+    try {
+      await this.blockBlobClient.stageBlock(blockID, chunk, chunk.length);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this.blockBlobClient.commitBlockList(this.blocks);
+      // remove all null/undefined props
+      const filteredObj = Object.entries(this.metadata).reduce((acc, [key, val]) => {
+        if (val !== undefined && val !== null) acc[key] = val;
+        return acc;
+      }, {});
+      await this.blockBlobClient.setMetadata(filteredObj);
+      callback();
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = AzureStorageUploadStream;

lib/record/encoder.js

@@ -0,0 +1,61 @@
+const { Transform } = require('stream');
+const lamejs = require('@jambonz/lamejs');
+
+class PCMToMP3Encoder extends Transform {
+  constructor(options, logger) {
+    super(options);
+
+    const channels = options.channels || 1;
+    const sampleRate = options.sampleRate || 8000;
+    const bitRate = options.bitRate || 128;
+
+    this.encoder = new lamejs.Mp3Encoder(channels, sampleRate, bitRate);
+    this.channels = channels;
+    this.logger = logger;
+  }
+
+  _transform(chunk, encoding, callback) {
+    try {
+      // Convert chunk buffer into Int16Array for lamejs
+      const samples = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.length / 2);
+
+      // Split input samples into left and right channel arrays if stereo
+      let leftChannel, rightChannel;
+      if (this.channels === 2) {
+        leftChannel = new Int16Array(samples.length / 2);
+        rightChannel = new Int16Array(samples.length / 2);
+
+        for (let i = 0; i < samples.length; i += 2) {
+          leftChannel[i / 2] = samples[i];
+          rightChannel[i / 2] = samples[i + 1];
+        }
+      } else {
+        leftChannel = samples;
+      }
+
+      // Encode the input data
+      const mp3Data = this.encoder.encodeBuffer(leftChannel, rightChannel);
+
+      if (mp3Data.length > 0) {
+        this.push(Buffer.from(mp3Data));
+      }
+      callback();
+    } catch (err) {
+      this.logger.error(
+        { err },
+        'Error while mp3 transform');
+    }
+  }
+
+  _flush(callback) {
+    // Finalize encoding and flush the internal buffers
+    const mp3Data = this.encoder.flush();
+
+    if (mp3Data.length > 0) {
+      this.push(Buffer.from(mp3Data));
+    }
+    callback();
+  }
+}
+
+module.exports = PCMToMP3Encoder;

lib/record/google-storage.js

@@ -0,0 +1,41 @@
+const { Storage } = require('@google-cloud/storage');
+const { Writable } = require('stream');
+
+class GoogleStorageUploadStream extends Writable {
+
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.metadata = opts.metadata;
+
+    const storage = new Storage(opts.bucketCredential);
+    this.gcsFile = storage.bucket(opts.bucketName).file(opts.Key);
+    this.writeStream = this.gcsFile.createWriteStream();
+
+    this.writeStream.on('error', (err) => this.logger.error(err));
+    this.writeStream.on('finish', () => {
+      this.logger.info('google storage Upload completed.');
+      this._addMetadata();
+    });
+  }
+
+  _write(chunk, encoding, callback) {
+    this.writeStream.write(chunk, encoding, callback);
+  }
+
+  _final(callback) {
+    this.writeStream.end();
+    this.writeStream.once('finish', callback);
+  }
+
+  async _addMetadata() {
+    try {
+      await this.gcsFile.setMetadata({metadata: this.metadata});
+      this.logger.info('Google storage Upload and metadata setting completed.');
+    } catch (err) {
+      this.logger.error(err, 'Google storage  An error occurred while setting metadata');
+    }
+  }
+}
+
+module.exports = GoogleStorageUploadStream;

lib/record/index.js

@@ -0,0 +1,6 @@
+
+async function record(logger, socket) {
+  return require('./upload')(logger, socket);
+}
+
+module.exports = record;

lib/record/s3-multipart-upload-stream.js

@@ -0,0 +1,103 @@
+const { Writable } = require('stream');
+const {
+  S3Client,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+  CompleteMultipartUploadCommand,
+} = require('@aws-sdk/client-s3');
+
+class S3MultipartUploadStream extends Writable {
+  constructor(logger, opts) {
+    super(opts);
+    this.logger = logger;
+    this.bucketName = opts.bucketName;
+    this.objectKey = opts.objectKey;
+    this.uploadId = null;
+    this.partNumber = 1;
+    this.multipartETags = [];
+    this.buffer = Buffer.alloc(0);
+    this.minPartSize = 5 * 1024 * 1024; // 5 MB
+    this.s3 = new S3Client(opts.bucketCredential);
+    this.metadata = opts.metadata;
+  }
+
+  async _initMultipartUpload() {
+    const command = new CreateMultipartUploadCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      Metadata: this.metadata
+    });
+    const response = await this.s3.send(command);
+    return response.UploadId;
+  }
+
+  async _uploadBuffer() {
+    const uploadPartCommand = new UploadPartCommand({
+      Bucket: this.bucketName,
+      Key: this.objectKey,
+      PartNumber: this.partNumber,
+      UploadId: this.uploadId,
+      Body: this.buffer,
+    });
+
+    const uploadPartResponse = await this.s3.send(uploadPartCommand);
+    this.multipartETags.push({
+      ETag: uploadPartResponse.ETag,
+      PartNumber: this.partNumber,
+    });
+    this.partNumber += 1;
+  }
+
+  async _write(chunk, encoding, callback) {
+    try {
+      if (!this.uploadId) {
+        this.uploadId = await this._initMultipartUpload();
+      }
+
+      this.buffer = Buffer.concat([this.buffer, chunk]);
+
+      if (this.buffer.length >= this.minPartSize) {
+        await this._uploadBuffer();
+        this.buffer = Buffer.alloc(0);
+      }
+
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+
+  async _finalize(err) {
+    try {
+      if (this.buffer.length > 0) {
+        await this._uploadBuffer();
+      }
+
+      const completeMultipartUploadCommand = new CompleteMultipartUploadCommand({
+        Bucket: this.bucketName,
+        Key: this.objectKey,
+        MultipartUpload: {
+          Parts: this.multipartETags.sort((a, b) => a.PartNumber - b.PartNumber),
+        },
+        UploadId: this.uploadId,
+      });
+
+      await this.s3.send(completeMultipartUploadCommand);
+      this.logger.info('Finished upload to S3');
+    } catch (error) {
+      this.logger.error('Error completing multipart upload:', error);
+      throw error;
+    }
+  }
+
+  async _final(callback) {
+    try {
+      await this._finalize();
+      callback(null);
+    } catch (error) {
+      callback(error);
+    }
+  }
+}
+
+module.exports = S3MultipartUploadStream;

lib/record/upload.js

@@ -0,0 +1,99 @@
+const Account = require('../models/account');
+const Websocket = require('ws');
+const PCMToMP3Encoder = require('./encoder');
+const wav = require('wav');
+const { getUploader } = require('./utils');
+
+async function upload(logger, socket) {
+  socket._recvInitialMetadata = false;
+  socket.on('message', async function(data, isBinary) {
+    try {
+      if (!isBinary && !socket._recvInitialMetadata) {
+        socket._recvInitialMetadata = true;
+        logger.debug(`initial metadata: ${data}`);
+        const obj = JSON.parse(data.toString());
+        logger.info({ obj }, 'received JSON message from jambonz');
+        const { sampleRate, accountSid, callSid, direction, from, to,
+          callId, applicationSid, originatingSipIp, originatingSipTrunkName } = obj;
+        const account = await Account.retrieve(accountSid);
+        if (account && account.length && account[0].bucket_credential) {
+          const obj = account[0].bucket_credential;
+          // add tags to metadata
+          const metadata = {
+            accountSid,
+            callSid,
+            direction,
+            from,
+            to,
+            callId,
+            applicationSid,
+            originatingSipIp,
+            originatingSipTrunkName,
+            sampleRate: `${sampleRate}`
+          };
+          if (obj.tags && obj.tags.length) {
+            obj.tags.forEach((tag) => {
+              metadata[tag.Key] = tag.Value;
+            });
+          }
+          // create S3 path
+          const day = new Date();
+          let key = `${day.getFullYear()}/${(day.getMonth() + 1).toString().padStart(2, '0')}`;
+          key += `/${day.getDate().toString().padStart(2, '0')}/${callSid}.${account[0].record_format}`;
+
+          // Uploader
+          const uploadStream = getUploader(key, metadata, obj, logger);
+          if (!uploadStream) {
+            logger.info('There is no available record uploader, close the socket.');
+            socket.close();
+          }
+
+          /**encoder */
+          let encoder;
+          if (account[0].record_format === 'wav') {
+            encoder = new wav.Writer({ channels: 2, sampleRate, bitDepth: 16 });
+          } else {
+            // default is mp3
+            encoder = new PCMToMP3Encoder({
+              channels: 2,
+              sampleRate: sampleRate,
+              bitrate: 128
+            }, logger);
+          }
+          const handleError = (err, streamType) => {
+            logger.error(
+              { err },
+              `Error while streaming for vendor: ${obj.vendor}, pipe: ${streamType}: ${err.message}`
+            );
+          };
+
+          /* start streaming data */
+          const duplex = Websocket.createWebSocketStream(socket);
+          duplex
+            .on('error', (err) => handleError(err, 'duplex'))
+            .pipe(encoder)
+            .on('error', (err) => handleError(err, 'encoder'))
+            .pipe(uploadStream)
+            .on('error', (err) => handleError(err, 'uploadStream'));
+
+        } else {
+          logger.info(`account ${accountSid} does not have any bucket credential, close the socket`);
+          socket.close();
+        }
+      }
+    } catch (err) {
+      logger.error({ err, data }, 'error parsing message during connection');
+    }
+  });
+  socket.on('error', function(err) {
+    logger.error({ err }, 'record upload: error');
+  });
+  socket.on('close', (data) => {
+    logger.info({ data }, 'record upload: close');
+  });
+  socket.on('end', function(err) {
+    logger.error({ err }, 'record upload: socket closed from jambonz');
+  });
+}
+
+module.exports = upload;

lib/record/utils.js

@@ -0,0 +1,58 @@
+const AzureStorageUploadStream = require('./azure-storage');
+const GoogleStorageUploadStream = require('./google-storage');
+const S3MultipartUploadStream = require('./s3-multipart-upload-stream');
+
+const getUploader = (key, metadata, bucket_credential, logger) => {
+  const uploaderOpts = {
+    bucketName: bucket_credential.name,
+    objectKey: key,
+    metadata
+  };
+  try {
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+        uploaderOpts.bucketCredential = {
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: bucket_credential.region || 'us-east-1'
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 's3_compatible':
+        uploaderOpts.bucketCredential = {
+          endpoint: bucket_credential.endpoint,
+          credentials: {
+            accessKeyId: bucket_credential.access_key_id,
+            secretAccessKey: bucket_credential.secret_access_key,
+          },
+          region: 'us-east-1',
+          forcePathStyle: true
+        };
+        return new S3MultipartUploadStream(logger, uploaderOpts);
+      case 'google':
+        const serviceKey = JSON.parse(bucket_credential.service_key);
+        uploaderOpts.bucketCredential = {
+          projectId: serviceKey.project_id,
+          credentials: {
+            client_email: serviceKey.client_email,
+            private_key: serviceKey.private_key
+          }
+        };
+        return new GoogleStorageUploadStream(logger, uploaderOpts);
+      case 'azure':
+        uploaderOpts.connection_string = bucket_credential.connection_string;
+        return new AzureStorageUploadStream(logger, uploaderOpts);
+      default:
+        logger.error(`unknown bucket vendor: ${bucket_credential.vendor}`);
+        break;
+    }
+  } catch (err) {
+    logger.error(`Error creating uploader, vendor: ${bucket_credential.vendor}, reason: ${err.message}`);
+  }
+  return null;
+};
+
+module.exports = {
+  getUploader
+};

lib/routes/api/accounts.js

@@ -1,6 +1,7 @@
 const router = require('express').Router();
+const assert = require('assert');
 const request = require('request');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden, DbErrorUnprocessableRequest} = require('../../utils/errors');
 const Account = require('../../models/account');
 const Application = require('../../models/application');
 const Webhook = require('../../models/webhook');
@@ -8,21 +9,74 @@ const ApiKey = require('../../models/api-key');
 const ServiceProvider = require('../../models/service-provider');
 const {deleteDnsRecords} = require('../../utils/dns-utils');
 const {deleteCustomer} = require('../../utils/stripe-utils');
-const uuidv4 = require('uuid/v4');
+const { v4: uuidv4 } = require('uuid');
 const snakeCase = require('../../utils/snake-case');
 const sysError = require('../error');
 const {promisePool} = require('../../db');
-const {hasAccountPermissions, parseAccountSid} = require('./utils');
+const {
+  hasAccountPermissions,
+  parseAccountSid,
+  parseCallSid,
+  enableSubspace,
+  disableSubspace,
+  parseVoipCarrierSid
+} = require('./utils');
 const short = require('short-uuid');
 const VoipCarrier = require('../../models/voip-carrier');
+const { encrypt } = require('../../utils/encrypt-decrypt');
+const { testS3Storage, testGoogleStorage, testAzureStorage } = require('../../utils/storage-utils');
 const translator = short();
 
 let idx = 0;
 
-const stripPort = (hostport) => {
-  const arr = /^(.*):(.*)$/.exec(hostport);
-  if (arr) return arr[1];
-  return hostport;
+const getFsUrl = async(logger, retrieveSet, setName) => {
+  if (process.env.K8S) {
+    const port = process.env.K8S_FEATURE_SERVER_SERVICE_PORT || 3000;
+    return `http://${process.env.K8S_FEATURE_SERVER_SERVICE_NAME}:${port}/v1/createCall`;
+  }
+
+  try {
+    const fs = await retrieveSet(setName);
+    if (0 === fs.length) {
+      logger.info('No available feature servers to handle createCall API request');
+      return ;
+    }
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/createCall`;
+  } catch (err) {
+    logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
+  }
+};
+
+const validateRequest = async(req, account_sid) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('account')) {
+      if (account_sid === req.user.account_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+      );
+
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  } catch (error) {
+    throw error;
+  }
 };
 
 router.use('/:sid/SpeechCredentials', hasAccountPermissions, require('./speech-credentials'));
@@ -31,10 +85,13 @@ router.use('/:sid/Alerts', hasAccountPermissions, require('./alerts'));
 router.use('/:sid/Charges', hasAccountPermissions, require('./charges'));
 router.use('/:sid/SipRealms', hasAccountPermissions, require('./sip-realm'));
 router.use('/:sid/PredefinedCarriers', hasAccountPermissions, require('./add-from-predefined-carrier'));
+router.use('/:sid/Limits', hasAccountPermissions, require('./limits'));
+router.use('/:sid/TtsCache', hasAccountPermissions, require('./tts-cache'));
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const results = await Application.retrieveAll(null, account_sid);
     res.status(200).json(results);
   } catch (err) {
@@ -45,17 +102,39 @@ router.get('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const results = await VoipCarrier.retrieveAll(account_sid);
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
   }
 });
+router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const sid = parseVoipCarrierSid(req);
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
+    const rowsAffected = await VoipCarrier.update(sid, req.body);
+    if (rowsAffected === 0) {
+      return res.sendStatus(404);
+    }
+
+    return res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 router.post('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   const payload = req.body;
   try {
     const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
     logger.debug({payload}, 'POST /:sid/VoipCarriers');
     const uuid = await VoipCarrier.make({
       account_sid,
@@ -97,9 +176,13 @@ function validateUpdateCall(opts) {
     'child_call_hook',
     'call_status',
     'listen_status',
+    'transcribe_status',
     'conf_hold_status',
     'conf_mute_status',
-    'mute_status']
+    'mute_status',
+    'sip_request',
+    'record'
+  ]
     .reduce((acc, prop) => (opts[prop] ? ++acc : acc), 0);
 
   switch (count) {
@@ -133,6 +216,16 @@ function validateUpdateCall(opts) {
   if (opts.conf_mute_status && !['mute', 'unmute'].includes(opts.conf_mute_status)) {
     throw new DbErrorBadRequest('invalid conf_mute_status');
   }
+  if (opts.sip_request &&
+    (!opts.sip_request.method || !opts.sip_request.content_type ||  !opts.sip_request.content)) {
+    throw new DbErrorBadRequest('sip_request requires method, content_type and content properties');
+  }
+  if (opts.record && !opts.record.action) {
+    throw new DbErrorBadRequest('record requires action property');
+  }
+  if ('startCallRecording' === opts.record?.action && !opts.record.siprecServerURL) {
+    throw new DbErrorBadRequest('record requires siprecServerURL property when starting recording');
+  }
 }
 
 function validateTo(to) {
@@ -152,6 +245,7 @@ function validateTo(to) {
   }
   throw new DbErrorBadRequest(`missing or invalid to property: ${JSON.stringify(to)}`);
 }
+
 async function validateCreateCall(logger, sid, req) {
   const {lookupAppBySid} = req.app.locals;
   const obj = req.body;
@@ -168,6 +262,7 @@ async function validateCreateCall(logger, sid, req) {
       const application = await lookupAppBySid(obj.application_sid);
       Object.assign(obj, {
         call_hook: application.call_hook,
+        app_json: application.app_json,
         call_status_hook: application.call_status_hook,
         speech_synthesis_vendor: application.speech_synthesis_vendor,
         speech_synthesis_language: application.speech_synthesis_language,
@@ -183,15 +278,14 @@ async function validateCreateCall(logger, sid, req) {
   }
   else {
     delete obj.application_sid;
-
-    // TODO: these should be retrieved from account, using account_sid if provided
-    Object.assign(obj, {
-      speech_synthesis_vendor: 'google',
-      speech_synthesis_voice: 'en-US-Wavenet-C',
-      speech_synthesis_language: 'en-US',
-      speech_recognizer_vendor: 'google',
-      speech_recognizer_language: 'en-US'
-    });
+    if (!obj.speech_synthesis_vendor ||
+      !obj.speech_synthesis_language ||
+      !obj.speech_synthesis_voice ||
+      !obj.speech_recognizer_vendor ||
+      !obj.speech_recognizer_language)
+      throw new DbErrorBadRequest('either application_sid or set of' +
+        ' speech_synthesis_vendor, speech_synthesis_language, speech_synthesis_voice,' +
+        ' speech_recognizer_vendor, speech_recognizer_language required');
   }
 
   if (!obj.call_hook && !obj.application_sid) {
@@ -217,10 +311,10 @@ async function validateCreateCall(logger, sid, req) {
   if (typeof obj.call_status_hook === 'object' && typeof obj.call_status_hook.url != 'string') {
     throw new DbErrorBadRequest('call_status_hook must be string or an object containing a url property');
   }
-  if (obj.call_hook && !/^https?:/.test(obj.call_hook.url)) {
+  if (obj.call_hook && !/^https?:/.test(obj.call_hook.url) && !/^wss?:/.test(obj.call_hook.url)) {
     throw new DbErrorBadRequest('call_hook url be an absolute url');
   }
-  if (obj.call_status_hook && !/^https?:/.test(obj.call_status_hook.url)) {
+  if (obj.call_status_hook && !/^https?:/.test(obj.call_status_hook.url) && !/^wss?:/.test(obj.call_status_hook.url)) {
     throw new DbErrorBadRequest('call_status_hook url be an absolute url');
   }
 }
@@ -253,7 +347,7 @@ async function validateCreateMessage(logger, sid, req) {
 async function validateAdd(req) {
   /* account-level token can not be used to add accounts */
   if (req.user.hasAccountAuth) {
-    throw new DbErrorUnprocessableRequest('insufficient permissions to create accounts');
+    throw new DbErrorForbidden('insufficient permissions');
   }
   if (req.user.hasServiceProviderAuth && req.user.service_provider_sid) {
     /* service providers can only create accounts under themselves */
@@ -272,9 +366,10 @@ async function validateAdd(req) {
     throw new DbErrorBadRequest('\'queue_event_hook\' must be an object when adding an account');
   }
 }
+
 async function validateUpdate(req, sid) {
   if (req.user.hasAccountAuth && req.user.account_sid !== sid) {
-    throw new DbErrorUnprocessableRequest('insufficient privileges to update this account');
+    throw new DbErrorForbidden('insufficient privileges');
   }
   if (req.user.hasAccountAuth && req.body.sip_realm) {
     throw new DbErrorBadRequest('use POST /Accounts/:sid/sip_realm/:realm to set or change the sip realm');
@@ -282,12 +377,23 @@ async function validateUpdate(req, sid) {
 
   if (req.user.service_provider_sid && !req.user.hasScope('admin')) {
     const result = await Account.retrieve(sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${sid}`);
+    }
     if (result[0].service_provider_sid !== req.user.service_provider_sid) {
-      throw new DbErrorUnprocessableRequest('cannot update account from different service provider');
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
+  if (req.user.hasScope('admin')) {
+    /* check to be sure that the account_sid exists */
+    const result = await Account.retrieve(sid);
+    if (!result || result.length === 0) {
+      throw new DbErrorBadRequest(`account not found for sid ${sid}`);
     }
   }
   if (req.body.service_provider_sid) throw new DbErrorBadRequest('service_provider_sid may not be modified');
 }
+
 async function validateDelete(req, sid) {
   if (req.user.hasAccountAuth && req.user.account_sid !== sid) {
     throw new DbErrorUnprocessableRequest('insufficient privileges to update this account');
@@ -295,12 +401,11 @@ async function validateDelete(req, sid) {
   if (req.user.service_provider_sid && !req.user.hasScope('admin')) {
     const result = await Account.retrieve(sid);
     if (result[0].service_provider_sid !== req.user.service_provider_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete account from different service provider');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
 }
 
-
 /* add */
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
@@ -342,8 +447,10 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     return res.status(200).json(results[0]);
   }
@@ -355,13 +462,15 @@ router.get('/:sid', async(req, res) => {
 router.get('/:sid/WebhookSecret', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
-    const results = await Account.retrieve(req.params.sid, service_provider_sid);
+    const results = await Account.retrieve(account_sid, service_provider_sid);
     if (results.length === 0) return res.status(404).end();
     let {webhook_secret} = results[0];
     if (req.query.regenerate) {
       const secret = `wh_secret_${translator.generate()}`;
-      await Account.update(req.params.sid, {webhook_secret: secret});
+      await Account.update(account_sid, {webhook_secret: secret});
       webhook_secret = secret;
     }
     return res.status(200).json({webhook_secret});
@@ -371,11 +480,120 @@ router.get('/:sid/WebhookSecret', async(req, res) => {
   }
 });
 
-/* update */
-router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
+router.post('/:sid/SubspaceTeleport', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const account_sid = parseAccountSid(req);
+    const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
+    const results = await Account.retrieve(account_sid, service_provider_sid);
+    if (results.length === 0) return res.status(404).end();
+    const {subspace_client_id, subspace_client_secret} = results[0];
+    const {destination} = req.body;
+    const arr = /^(.*):\d+$/.exec(destination);
+    const dest = arr ? `sip:${arr[1]}` : `sip:${destination}`;
+
+    const teleport = await enableSubspace({
+      subspace_client_id,
+      subspace_client_secret,
+      destination: dest
+    });
+    logger.info({destination, teleport}, 'SubspaceTeleport - create teleport');
+    await Account.update(account_sid, {
+      subspace_sip_teleport_id: teleport.id,
+      subspace_sip_teleport_destinations: JSON.stringify(teleport.teleport_entry_points)//hacky
+    });
+
+    return res.status(200).json({
+      subspace_sip_teleport_id: teleport.id,
+      subspace_sip_teleport_destinations: teleport.teleport_entry_points
+    });
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.delete('/:sid/SubspaceTeleport', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const account_sid = parseAccountSid(req);
+    const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
+    const results = await Account.retrieve(account_sid, service_provider_sid);
+    if (results.length === 0) return res.status(404).end();
+    const {subspace_client_id, subspace_client_secret, subspace_sip_teleport_id} = results[0];
+
+    await disableSubspace({subspace_client_id, subspace_client_secret, subspace_sip_teleport_id});
+    await Account.update(account_sid, {
+      subspace_sip_teleport_id: null,
+      subspace_sip_teleport_destinations: null
+    });
+    return res.sendStatus(204);
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+function encryptBucketCredential(obj) {
+  if (!obj.bucket_credential) return;
+  const {
+    vendor,
+    region,
+    name,
+    access_key_id,
+    secret_access_key,
+    tags,
+    service_key,
+    connection_string,
+    endpoint
+  } = obj.bucket_credential;
+
+  switch (vendor) {
+    case 'aws_s3':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(region, 'invalid aws bucket region: region is required');
+      const awsData = JSON.stringify({vendor, region, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(awsData);
+      break;
+    case 's3_compatible':
+      assert(access_key_id, 'invalid aws S3 bucket credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws S3 bucket credential: secret_access_key is required');
+      assert(name, 'invalid aws bucket name: name is required');
+      assert(endpoint, 'invalid endpoint uri: endpoint is required');
+      const s3Data = JSON.stringify({vendor, endpoint, name, access_key_id,
+        secret_access_key, tags});
+      obj.bucket_credential = encrypt(s3Data);
+      break;
+    case 'google':
+      assert(service_key, 'invalid google cloud storage credential: service_key is required');
+      const googleData = JSON.stringify({vendor, name, service_key, tags});
+      obj.bucket_credential = encrypt(googleData);
+      break;
+    case 'azure':
+      assert(name, 'invalid azure container name: name is required');
+      assert(connection_string, 'invalid azure cloud storage credential: connection_string is required');
+      const azureData = JSON.stringify({vendor, name, connection_string, tags});
+      obj.bucket_credential = encrypt(azureData);
+      break;
+    case 'none':
+      obj.bucket_credential = null;
+      break;
+    default:
+      throw new DbErrorBadRequest(`unknown storage vendor: ${vendor}`);
+  }
+}
+
+/**
+ * update
+ */
+router.put('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
 
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
@@ -414,6 +632,8 @@ router.put('/:sid', async(req, res) => {
       delete obj.registration_hook;
       delete obj.queue_event_hook;
 
+      encryptBucketCredential(obj);
+
       const rowsAffected = await Account.update(sid, obj);
       if (rowsAffected === 0) {
         return res.status(404).end();
@@ -435,16 +655,18 @@ router.put('/:sid', async(req, res) => {
 
 /* delete */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   const sqlDeleteGateways = `DELETE from sip_gateways 
     WHERE voip_carrier_sid IN
     (SELECT voip_carrier_sid from voip_carriers where account_sid = ?)`;
+
   try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
     await validateDelete(req, sid);
 
     const [account] = await promisePool.query('SELECT * FROM accounts WHERE account_sid = ?', sid);
-    const {sip_realm, stripe_customer_id} = account[0];
+    const {sip_realm, stripe_customer_id, registration_hook_sid} = account[0];
     /* remove dns records */
     if (process.env.NODE_ENV !== 'test' || process.env.DME_API_KEY) {
 
@@ -485,6 +707,15 @@ account_subscriptions WHERE account_sid = ?)
     await promisePool.execute('DELETE from applications where account_sid = ?', [sid]);
     await promisePool.execute('DELETE from accounts where account_sid = ?', [sid]);
 
+    if (registration_hook_sid) {
+      /* remove registration hook if only used by this account */
+      const sql = 'SELECT COUNT(*) as count FROM accounts WHERE registration_hook_sid = ?';
+      const [r] = await promisePool.query(sql, registration_hook_sid);
+      if (r[0]?.count === 0) {
+        await promisePool.execute('DELETE from webhooks where webhook_sid = ?', [registration_hook_sid]);
+      }
+    }
+
     if (stripe_customer_id) {
       const response = await deleteCustomer(logger, stripe_customer_id);
       logger.info({response}, `deleted stripe customer_id ${stripe_customer_id} for account_si ${sid}`);
@@ -495,13 +726,57 @@ account_subscriptions WHERE account_sid = ?)
   }
 });
 
-/* retrieve account level api keys */
-router.get('/:sid/ApiKeys', async(req, res) => {
+/* Test Bucket credential Keys */
+router.post('/:sid/BucketCredentialTest', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await ApiKey.retrieveAll(req.params.sid);
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+    const {vendor, name, region, access_key_id, secret_access_key, service_key, connection_string, endpoint} = req.body;
+    const ret = {
+      status: 'not tested'
+    };
+
+    switch (vendor) {
+      case 'aws_s3':
+        await testS3Storage(logger, {vendor, name, region, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 's3_compatible':
+        await testS3Storage(logger, {vendor, name, endpoint, access_key_id, secret_access_key});
+        ret.status = 'ok';
+        break;
+      case 'google':
+        await testGoogleStorage(logger, {vendor, name, service_key});
+        ret.status = 'ok';
+        break;
+      case 'azure':
+        await testAzureStorage(logger, {vendor, name, connection_string});
+        ret.status = 'ok';
+        break;
+      default:
+        throw new DbErrorBadRequest(`Does not support test for ${vendor}`);
+    }
+    return res.status(200).json(ret);
+  }
+  catch (err) {
+    return res.status(200).json({status: 'failed', reason: err.message});
+  }
+});
+
+/**
+ * retrieve account level api keys
+ */
+router.get('/:sid/ApiKeys', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
+
+    const results = await ApiKey.retrieveAll(sid);
     res.status(200).json(results);
-    updateLastUsed(logger, req.params.sid, req).catch((err) => {});
+    updateLastUsed(logger, sid, req).catch((err) => {});
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -511,22 +786,19 @@ router.get('/:sid/ApiKeys', async(req, res) => {
  * create a new Call
  */
 router.post('/:sid/Calls', async(req, res) => {
-  const sid = req.params.sid;
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const {retrieveSet, logger} = req.app.locals;
+  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
+  const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
+
+  if (!serviceUrl) {
+    return res.status(480).json({msg: 'no available feature servers at this time'});
+  }
 
   try {
-    const fs = await retrieveSet(setName);
-    if (0 === fs.length) {
-      logger.info('No available feature servers to handle createCall API request');
-      return res.json({msg: 'no available feature servers at this time'}).status(500);
-    }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createCall API request, selecting ${ip}`);
-    const serviceUrl = `http://${ip}:3000/v1/createCall`;
-    await validateCreateCall(logger, sid, req);
+    const sid = parseAccountSid(req);
+    await validateRequest(req, sid);
 
-    logger.debug({payload: req.body}, `sending createCall API request to to ${ip}`);
+    await validateCreateCall(logger, sid, req);
     updateLastUsed(logger, sid, req).catch((err) => {});
     request({
       url: serviceUrl,
@@ -535,14 +807,16 @@ router.post('/:sid/Calls', async(req, res) => {
       body: Object.assign(req.body, {account_sid: sid})
     }, (err, response, body) => {
       if (err) {
-        logger.error(err, `Error sending createCall POST to ${ip}`);
+        logger.error(err, `Error sending createCall POST to ${serviceUrl}`);
         return res.sendStatus(500);
       }
+
       if (response.statusCode !== 201) {
-        logger.error({statusCode: response.statusCode}, `Non-success response returned by createCall ${ip}`);
+        logger.error({statusCode: response.statusCode}, `Non-success response returned by createCall ${serviceUrl}`);
         return res.sendStatus(500);
       }
-      res.status(201).json(body);
+
+      return res.status(201).json(body);
     });
   } catch (err) {
     sysError(logger, res, err);
@@ -553,11 +827,18 @@ router.post('/:sid/Calls', async(req, res) => {
  * retrieve info for a group of calls under an account
  */
 router.get('/:sid/Calls', async(req, res) => {
-  const accountSid = req.params.sid;
   const {logger, listCalls} = req.app.locals;
-
+  const {direction, from, to, callStatus} = req.query || {};
   try {
-    const calls = await listCalls(accountSid);
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const calls = await listCalls({
+      accountSid,
+      direction,
+      from,
+      to,
+      callStatus
+    });
     logger.debug(`retrieved ${calls.length} calls for account sid ${accountSid}`);
     res.status(200).json(coerceNumbers(snakeCase(calls)));
     updateLastUsed(logger, accountSid, req).catch((err) => {});
@@ -570,11 +851,12 @@ router.get('/:sid/Calls', async(req, res) => {
  * retrieve single call
  */
 router.get('/:sid/Calls/:callSid', async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, retrieveCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     const callInfo = await retrieveCall(accountSid, callSid);
     if (callInfo) {
       logger.debug(callInfo, `retrieved call info for call sid ${callSid}`);
@@ -594,11 +876,12 @@ router.get('/:sid/Calls/:callSid', async(req, res) => {
  * delete call
  */
 router.delete('/:sid/Calls/:callSid', async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, deleteCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     const result = await deleteCall(accountSid, callSid);
     if (result) {
       logger.debug(`successfully deleted call ${callSid}`);
@@ -618,11 +901,12 @@ router.delete('/:sid/Calls/:callSid', async(req, res) => {
  * update a call
  */
 const updateCall = async(req, res) => {
-  const accountSid = req.params.sid;
-  const callSid = req.params.callSid;
   const {logger, retrieveCall} = req.app.locals;
 
   try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const callSid = parseCallSid(req);
     validateUpdateCall(req.body);
     const call = await retrieveCall(accountSid, callSid);
     if (call) {
@@ -649,6 +933,7 @@ const updateCall = async(req, res) => {
 router.post('/:sid/Calls/:callSid', async(req, res) => {
   await updateCall(req, res);
 });
+
 router.put('/:sid/Calls/:callSid', async(req, res) => {
   await updateCall(req, res);
 });
@@ -657,19 +942,15 @@ router.put('/:sid/Calls/:callSid', async(req, res) => {
  * create a new Message
  */
 router.post('/:sid/Messages', async(req, res) => {
-  const account_sid = parseAccountSid(req);
-  const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:active-fs`;
   const {retrieveSet, logger} = req.app.locals;
 
   try {
-    const fs = await retrieveSet(setName);
-    if (0 === fs.length) {
-      logger.info('No available feature servers to handle createMessage API request');
-      return res.json({msg: 'no available feature servers at this time'}).status(500);
-    }
-    const ip = stripPort(fs[idx++ % fs.length]);
-    logger.info({fs}, `feature servers available for createMessage API request, selecting ${ip}`);
-    const serviceUrl = `http://${ip}:3000/v1/createMessage/${account_sid}`;
+    const account_sid = parseAccountSid(req);
+    await validateRequest(req, account_sid);
+
+    const setName = `${(process.env.JAMBONES_CLUSTER_ID || 'default')}:fs-service-url`;
+    const serviceUrl = await getFsUrl(logger, retrieveSet, setName);
+    if (!serviceUrl) res.json({msg: 'no available feature servers at this time'}).status(480);
     await validateCreateMessage(logger, account_sid, req);
 
     const payload = {
@@ -677,7 +958,7 @@ router.post('/:sid/Messages', async(req, res) => {
       account_sid,
       ...req.body
     };
-    logger.debug({payload}, `sending createMessage API request to to ${ip}`);
+    logger.debug({payload}, `sending createMessage API request to to ${serviceUrl}`);
     updateLastUsed(logger, account_sid, req).catch(() => {});
     request({
       url: serviceUrl,
@@ -686,7 +967,7 @@ router.post('/:sid/Messages', async(req, res) => {
       body: payload
     }, (err, response, body) => {
       if (err) {
-        logger.error(err, `Error sending createMessage POST to ${ip}`);
+        logger.error(err, `Error sending createMessage POST to ${serviceUrl}`);
         return res.sendStatus(500);
       }
       if (response.statusCode !== 200) {
@@ -700,4 +981,22 @@ router.post('/:sid/Messages', async(req, res) => {
   }
 });
 
+/**
+ * retrieve info for a group of queues under an account
+ */
+router.get('/:sid/Queues', async(req, res) => {
+  const {logger, listSortedSets} = req.app.locals;
+  const { search } = req.query || {};
+  try {
+    const accountSid = parseAccountSid(req);
+    await validateRequest(req, accountSid);
+    const queues = search ? await listSortedSets(accountSid, search) : await listSortedSets(accountSid);
+    logger.debug(`retrieved ${queues.length} queues for account sid ${accountSid}`);
+    res.status(200).json(queues);
+    updateLastUsed(logger, accountSid, req).catch((err) => {});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
 module.exports = router;

lib/routes/api/add-from-predefined-carrier.js

@@ -1,16 +1,14 @@
 const router = require('express').Router();
-const {DbErrorBadRequest} = require('../../utils/errors');
 const PredefinedCarrier = require('../../models/predefined-carrier');
 const VoipCarrier = require('../../models/voip-carrier');
 const SipGateway = require('../../models/sip-gateway');
 const SmppGateway = require('../../models/smpp-gateway');
 const {parseServiceProviderSid} = require('./utils');
+const short = require('short-uuid');
 const {promisePool} = require('../../db');
 const sysError = require('../error');
 
-const sqlSelectCarrierByName = `SELECT * FROM voip_carriers 
-WHERE account_sid = ? 
-AND name = ?`;
+
 const sqlSelectCarrierByNameForSP = `SELECT * FROM voip_carriers 
 WHERE service_provider_sid = ? 
 AND name = ?`;
@@ -25,26 +23,23 @@ router.post('/:sid', async(req, res) => {
   const {sid } = req.params;
   let service_provider_sid;
   const {account_sid} = req.user;
-  if (!account_sid) {
-    if (!req.user.hasScope('service_provider')) {
-      logger.error({user: req.user}, 'invalid creds');
-      return res.sendStatus(403);
-    }
-    service_provider_sid = parseServiceProviderSid(req);
-  }
+
   try {
+    if (!account_sid) {
+      service_provider_sid = parseServiceProviderSid(req);
+    } else {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
     const [template] = await PredefinedCarrier.retrieve(sid);
     logger.debug({template}, `Retrieved template carrier for sid ${sid}`);
     if (!template) return res.sendStatus(404);
 
     /* make sure not to add the same carrier twice */
-    const [r2] = account_sid ?
-      await promisePool.query(sqlSelectCarrierByName, [account_sid, template.name]) :
-      await promisePool.query(sqlSelectCarrierByNameForSP, [service_provider_sid, template.name]);
+    const [r2] = await promisePool.query(sqlSelectCarrierByNameForSP, [service_provider_sid, template.name]);
 
     if (r2.length > 0) {
-      logger.info({account_sid}, `Failed to add carrier with name ${template.name}, carrier of that name exists`);
-      throw new DbErrorBadRequest(`A carrier with name ${template.name} already exists`);
+      template.name =  `${template.name}-${short.generate()}`;
     }
 
     /* retrieve all the sip gateways */

lib/routes/api/alerts.js

@@ -1,6 +1,7 @@
 const router = require('express').Router();
 const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
+const { parseServiceProviderSid } = require('./utils');
 
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
@@ -8,25 +9,41 @@ const parseAccountSid = (url) => {
 };
 
 router.get('/', async(req, res) => {
-  const {logger, queryAlerts} = req.app.locals;
+  const {logger, queryAlerts, queryAlertsSP} = req.app.locals;
   try {
     logger.debug({opts: req.query}, 'GET /Alerts');
     const account_sid = parseAccountSid(req.originalUrl);
+    const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
     const {page, count, alert_type, days, start, end} = req.query || {};
     if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
     if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
 
-    const data = await queryAlerts({
-      account_sid,
-      page,
-      page_size: count,
-      alert_type,
-      days,
-      start: days ? undefined : start,
-      end: days ? undefined : end,
-    });
+    if (account_sid) {
+      const data = await queryAlerts({
+        account_sid,
+        page,
+        page_size: count,
+        alert_type,
+        days,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+      });
 
-    res.status(200).json(data);
+      res.status(200).json(data);
+    }
+    else {
+      const data = await queryAlertsSP({
+        service_provider_sid,
+        page,
+        page_size: count,
+        alert_type,
+        days,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+      });
+
+      res.status(200).json(data);
+    }
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/api-keys.js

@@ -3,7 +3,7 @@ const {DbErrorBadRequest} = require('../../utils/errors');
 const ApiKey = require('../../models/api-key');
 const Account = require('../../models/account');
 const decorate = require('./decorate');
-const uuidv4 = require('uuid/v4');
+const { v4: uuidv4 } = require('uuid');
 const sysError = require('../error');
 const preconditions = {
   'add': validateAddToken,

lib/routes/api/applications.js

@@ -1,14 +1,46 @@
 const router = require('express').Router();
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const Application = require('../../models/application');
 const Account = require('../../models/account');
 const Webhook = require('../../models/webhook');
+const {promisePool} = require('../../db');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const { validate } = require('@jambonz/verb-specifications');
+const { parseApplicationSid } = require('./utils');
 const preconditions = {
   'add': validateAdd,
-  'update': validateUpdate,
-  'delete': validateDelete
+  'update': validateUpdate
+};
+
+const validateRequest = async(req, account_sid) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('account')) {
+      if (account_sid === req.user.account_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+      );
+
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  } catch (error) {
+    throw error;
+  }
 };
 
 /* only user-level tokens can add applications */
@@ -21,7 +53,7 @@ async function validateAdd(req) {
     if (!req.body.account_sid) throw new DbErrorBadRequest('missing required field: \'account_sid\'');
     const result = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
     if (result.length === 0) {
-      throw new DbErrorBadRequest('insufficient privileges to create an application under the specified account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
@@ -33,12 +65,26 @@ async function validateAdd(req) {
 }
 
 async function validateUpdate(req, sid) {
-  if (req.user.account_sid) {
-    const app = await Application.retrieve(sid);
-    if (!app || !app.length || app[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorBadRequest('you may not update or delete an application associated with a different account');
+  const app = await Application.retrieve(sid);
+  if (req.user.hasAccountAuth) {
+    if (!app || 0 === app.length || app[0].account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
+
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [app[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  }
+
+
   if (req.body.call_hook && typeof req.body.call_hook !== 'object') {
     throw new DbErrorBadRequest('\'call_hook\' must be an object when updating an application');
   }
@@ -48,18 +94,29 @@ async function validateUpdate(req, sid) {
 }
 
 async function validateDelete(req, sid) {
+  const result = await Application.retrieve(sid);
   if (req.user.hasAccountAuth) {
-    const result = await Application.retrieve(sid);
     if (!result || 0 === result.length) throw new DbErrorBadRequest('application does not exist');
     if (result[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete application owned by a different account');
+      throw new DbErrorUnprocessableRequest('insufficient permissions');
     }
   }
+  if (req.user.hasServiceProviderAuth) {
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [result[0].account_sid]
+    );
+
+    if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  }
   const assignedPhoneNumbers = await Application.getForeignKeyReferences('phone_numbers.application_sid', sid);
   if (assignedPhoneNumbers > 0) throw new DbErrorUnprocessableRequest('cannot delete application with phone numbers');
 }
 
-decorate(router, Application, ['delete'], preconditions);
+decorate(router, Application, [], preconditions);
 
 /* add */
 router.post('/', async(req, res) => {
@@ -76,6 +133,16 @@ router.post('/', async(req, res) => {
       }
     }
 
+    // validate app json if required
+    if (obj['app_json']) {
+      const app_json = JSON.parse(obj['app_json']);
+      try {
+        validate(logger, app_json);
+      } catch (err) {
+        throw new DbErrorBadRequest(err);
+      }
+    }
+
     const uuid = await Application.make(obj);
     res.status(201).json({sid: uuid});
   } catch (err) {
@@ -100,22 +167,65 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const application_sid = parseApplicationSid(req);
     const service_provider_sid = req.user.hasServiceProviderAuth ? req.user.service_provider_sid : null;
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await Application.retrieve(req.params.sid, service_provider_sid, account_sid);
+    const results = await Application.retrieve(application_sid, service_provider_sid, account_sid);
     if (results.length === 0) return res.status(404).end();
-    return res.status(200).json(results);
+    await validateRequest(req, results[0].account_sid);
+    return res.status(200).json(results[0]);
   }
   catch (err) {
     sysError(logger, res, err);
   }
 });
 
-/* update */
-router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
+/* delete */
+router.delete('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parseApplicationSid(req);
+    await validateDelete(req, sid);
+
+    const [application] = await promisePool.query('SELECT * FROM applications WHERE application_sid = ?', sid);
+    const {call_hook_sid, call_status_hook_sid, messaging_hook_sid} = application[0];
+    logger.info({call_hook_sid, call_status_hook_sid, messaging_hook_sid, sid}, 'deleting application');
+    await promisePool.execute('DELETE from applications where application_sid = ?', [sid]);
+
+    if (call_hook_sid) {
+      /* remove call hook if only used by this app */
+      const sql = 'SELECT COUNT(*) as count FROM applications WHERE call_hook_sid = ?';
+      const [r] = await promisePool.query(sql, call_hook_sid);
+      if (r[0]?.count === 0) {
+        await promisePool.execute('DELETE from webhooks where webhook_sid = ?', [call_hook_sid]);
+      }
+    }
+    if (call_status_hook_sid) {
+      const sql = 'SELECT COUNT(*) as count FROM applications WHERE call_status_hook_sid = ?';
+      const [r] = await promisePool.query(sql, call_status_hook_sid);
+      if (r[0]?.count === 0) {
+        await promisePool.execute('DELETE from webhooks where webhook_sid = ?', [call_status_hook_sid]);
+      }
+    }
+    if (messaging_hook_sid) {
+      const sql = 'SELECT COUNT(*) as count FROM applications WHERE messaging_hook_sid = ?';
+      const [r] = await promisePool.query(sql, messaging_hook_sid);
+      if (r[0]?.count === 0) {
+        await promisePool.execute('DELETE from webhooks where webhook_sid = ?', [messaging_hook_sid]);
+      }
+    }
+
+    res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/* update */
+router.put('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const sid = parseApplicationSid(req);
     await validateUpdate(req, sid);
 
     // create webhooks if provided
@@ -138,6 +248,16 @@ router.put('/:sid', async(req, res) => {
       delete obj[prop];
     }
 
+    // validate app json if required
+    if (obj['app_json']) {
+      const app_json = JSON.parse(obj['app_json']);
+      try {
+        validate(logger, app_json);
+      } catch (err) {
+        throw new DbErrorBadRequest(err);
+      }
+    }
+
     const rowsAffected = await Application.update(sid, obj);
     if (rowsAffected === 0) {
       return res.status(404).end();

lib/routes/api/change-password.js

@@ -1,15 +1,15 @@
 const router = require('express').Router();
-//const debug = require('debug')('jambonz:api-server');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {generateHashedPassword, verifyPassword} = require('../../utils/password-utils');
 const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 const sqlUpdatePassword = `UPDATE users 
 SET hashed_password= ? 
 WHERE user_sid = ?`;
 
 router.post('/', async(req, res) => {
-  const {logger, retrieveKey, deleteKey} = req.app.locals;
+  const {logger} = req.app.locals;
   const {user_sid} = req.user;
   const {old_password, new_password} = req.body;
   try {
@@ -26,10 +26,10 @@ router.post('/', async(req, res) => {
 
       const isCorrect = await verifyPassword(r[0].hashed_password, old_password);
       if (!isCorrect) {
-        const key = `reset-link:${old_password}`;
-        const user_sid = await retrieveKey(key);
+        const key = cacheClient.generateRedisKey('reset-link', old_password);
+        const user_sid = await cacheClient.get(key);
         if (!user_sid) throw new DbErrorBadRequest('old_password is incorrect');
-        await deleteKey(key);
+        await cacheClient.delete(key);
       }
     }
 
@@ -42,6 +42,9 @@ router.post('/', async(req, res) => {
     sysError(logger, res, err);
     return;
   }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+  await cacheClient.delete(redisKey);
 });
 
 module.exports = router;

lib/routes/api/clients.js

@@ -0,0 +1,88 @@
+const router = require('express').Router();
+const decorate = require('./decorate');
+const sysError = require('../error');
+const Client = require('../../models/client');
+const Account = require('../../models/account');
+const { DbErrorBadRequest, DbErrorForbidden } = require('../../utils/errors');
+const { encrypt, decrypt, obscureKey } = require('../../utils/encrypt-decrypt');
+
+const commonCheck = async(req) => {
+  if (req.user.hasAccountAuth) {
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth && req.body.account_sid) {
+    const accounts = await Account.retrieve(req.body.account_sid, req.user.service_provider_sid);
+    if (accounts.length === 0) {
+      throw new DbErrorForbidden('insufficient permissions');
+    }
+  }
+
+  if (req.body.password) {
+    req.body.password = encrypt(req.body.password);
+  }
+};
+
+const validateAdd = async(req) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await commonCheck(req);
+
+  const clients = await Client.retrieveByAccountSidAndUserName(req.body.account_sid, req.body.username);
+  if (clients.length && clients[0].client_sid !== sid) {
+    throw new DbErrorBadRequest('the client\'s username already exists');
+  }
+};
+
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, Client, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Client.retrieveAll() : req.user.hasAccountAuth ?
+        await Client.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Client.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+    const ret = results.map((c) => {
+      c.password = obscureKey(decrypt(c.password), 1);
+      return c;
+    });
+    res.status(200).json(ret);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Client.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const client = results[0];
+    client.password = obscureKey(decrypt(client.password), 1);
+    if (req.user.hasAccountAuth && client.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth) {
+      const accounts = await Account.retrieve(client.account_sid, req.user.service_provider_sid);
+      if (!accounts.length) {
+        return res.sendStatus(404);
+      }
+    }
+    return res.status(200).json(client);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/error.js

@@ -1,6 +1,10 @@
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const { BadRequestError, DbErrorBadRequest, DbErrorUnprocessableRequest } = require('../../utils/errors');
 
 function sysError(logger, res, err) {
+  if (err instanceof BadRequestError) {
+    logger.info(err, err.message);
+    return res.status(400).json({msg: 'Bad request'});
+  }
   if (err instanceof DbErrorBadRequest) {
     logger.info(err, 'invalid client request');
     return res.status(400).json({msg: err.message});

lib/routes/api/forgot-password.js

@@ -4,7 +4,9 @@ const short = require('short-uuid');
 const translator = short();
 const {validateEmail, emailSimpleText} = require('../../utils/email-utils');
 const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
+const assert = require('assert');
 const sql = `SELECT * from users user 
 LEFT JOIN accounts AS acc 
 ON acc.account_sid = user.account_sid 
@@ -25,7 +27,8 @@ function createOauthEmailText(provider) {
 }
 
 function createResetEmailText(link) {
-  const baseUrl = 'http://localhost:3001';
+  assert(process.env.JAMBONZ_BASE_URL, 'process.env.JAMBONZ_BASE_URL is missing');
+  const baseUrl = process.env.JAMBONZ_BASE_URL;
 
   return `Hi there!
 
@@ -45,6 +48,7 @@ function createResetEmailText(link) {
 router.post('/', async(req, res) => {
   const {logger, addKey} = req.app.locals;
   const {email} = req.body;
+
   let obj;
   try {
     if (!email || !validateEmail(email)) {
@@ -53,11 +57,16 @@ router.post('/', async(req, res) => {
 
     const [r] = await promisePool.query({sql, nestTables: true}, email);
     if (0 === r.length) {
-      return res.status(400).json({error: 'email does not exist'});
+      logger.info('user not found');
+      return res.status(400).json({error: 'failed to reset your password'});
     }
     obj = r[0];
-    if (!obj.acc.is_active) {
-      return res.status(400).json({error: 'you may not reset the password of an inactive account'});
+    if (!obj.user.is_active) {
+      logger.info(obj.user.name, 'user is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
+    } else if (obj.acc.account_sid !== null && !obj.acc.is_active) {
+      logger.info(obj.acc.account_sid, 'account is inactive');
+      return res.status(400).json({error: 'failed to reset your password'});
     }
     res.sendStatus(204);
   } catch (err) {
@@ -73,10 +82,14 @@ router.post('/', async(req, res) => {
   else {
     /* generate a link for this user to reset, send email */
     const link = translator.generate();
-    addKey(`reset-link:${link}`, obj.user.user_sid, 3600)
+    const redisKey = cacheClient.generateRedisKey('reset-link', link);
+    addKey(redisKey, obj.user.user_sid, 3600)
       .catch((err) => logger.error({err}, 'Error adding reset link to redis'));
     emailSimpleText(logger, email, 'Reset password request', createResetEmailText(link));
   }
+
+  const redisKey = cacheClient.generateRedisKey('jwt', obj.user.user_sid, 'v2');
+  await cacheClient.delete(redisKey);
 });
 
 module.exports = router;

lib/routes/api/index.js

@@ -7,9 +7,18 @@ const isAdminScope = (req, res, next) => {
     message: 'insufficient privileges'
   });
 };
+// const isAdminOrSPScope = (req, res, next) => {
+//   if (req.user.hasScope('admin') || req.user.hasScope('service_provider')) return next();
+//   res.status(403).json({
+//     status: 'fail',
+//     message: 'insufficient privileges'
+//   });
+// };
 
 api.use('/BetaInviteCodes', isAdminScope, require('./beta-invite-codes'));
-api.use('/ServiceProviders', isAdminScope, require('./service-providers'));
+api.use('/SystemInformation', isAdminScope, require('./system-information'));
+api.use('/TtsCache', isAdminScope, require('./tts-cache'));
+api.use('/ServiceProviders', require('./service-providers'));
 api.use('/VoipCarriers', require('./voip-carriers'));
 api.use('/Webhooks', require('./webhooks'));
 api.use('/SipGateways', require('./sip-gateways'));
@@ -37,6 +46,12 @@ api.use('/Subscriptions', require('./subscriptions'));
 api.use('/Invoices', require('./invoices'));
 api.use('/InviteCodes', require('./invite-codes'));
 api.use('/PredefinedCarriers', require('./predefined-carriers'));
+api.use('/PasswordSettings', require('./password-settings'));
+// Least Cost Routing
+api.use('/Lcrs', require('./lcrs'));
+api.use('/LcrRoutes', require('./lcr-routes'));
+api.use('/LcrCarrierSetEntries', require('./lcr-carrier-set-entries'));
+api.use('/Clients', require('./clients'));
 
 // messaging
 api.use('/Smpps', require('./smpps'));   // our smpp server info

lib/routes/api/lcr-carrier-set-entries.js

@@ -0,0 +1,65 @@
+const router = require('express').Router();
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoute = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!req.body.lcr_route_sid) {
+    throw new DbErrorBadRequest('missing lcr_route_sid');
+  }
+  // check lcr_route_sid is exist
+  const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+  if (lcrRoute.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_route_sid');
+  }
+  // check voip_carrier_sid is exist
+  if (!req.body.voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+  const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+  if (!carrier) {
+    throw new DbErrorBadRequest('unknown voip_carrier_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (req.body.lcr_route_sid) {
+    const lcrRoute = await LcrRoute.retrieve(req.body.lcr_route_sid);
+    if (lcrRoute.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_route_sid');
+    }
+  }
+
+  // check voip_carrier_sid is exist
+  if (req.body.voip_carrier_sid) {
+    const carrier = await lookupCarrierBySid(req.body.voip_carrier_sid);
+    if (!carrier) {
+      throw new DbErrorBadRequest('unknown voip_carrier_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+};
+
+decorate(router, LcrCarrierSetEntry, ['add', 'retrieve', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.query.lcr_route_sid;
+  try {
+    const results = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(lcr_route_sid);
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcr-routes.js

@@ -0,0 +1,96 @@
+const router = require('express').Router();
+const LcrRoute = require('../../models/lcr-route');
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+
+const validateAdd = async(req) => {
+  // check if lcr sid is available
+  if (!req.body.lcr_sid) {
+    throw new DbErrorBadRequest('missing parameter lcr_sid');
+  }
+
+  const lcr = await Lcr.retrieve(req.body.lcr_sid);
+  if (lcr.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+};
+
+const validateUpdate = async(req) => {
+  if (req.body.lcr_sid) {
+    const lcr = await Lcr.retrieve(req.body.lcr_sid);
+    if (lcr.length === 0) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateDelete = async(req, sid) => {
+  // delete all lcr carrier set entries
+  await LcrCarrierSetEntry.deleteByLcrRouteSid(sid);
+};
+
+const checkUserScope = async(req, lcr_sid) => {
+  if (!lcr_sid) {
+    throw new DbErrorBadRequest('missing lcr_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+
+  const lcrList = await Lcr.retrieve(lcr_sid);
+  if (lcrList.length === 0) throw new DbErrorBadRequest('unknown lcr_sid');
+  const lcr = lcrList[0];
+  if (req.user.hasAccountAuth) {
+    if (!lcr.account_sid || lcr.account_sid !== req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+
+  if (req.user.hasServiceProviderAuth) {
+    if (!lcr.service_provider_sid  || lcr.service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete,
+};
+
+decorate(router, LcrRoute, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_sid = req.query.lcr_sid;
+  try {
+    await checkUserScope(req, lcr_sid);
+    const results = await LcrRoute.retrieveAllByLcrSid(lcr_sid);
+    for (const r of results) {
+      r.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(r.lcr_route_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const lcr_route_sid = req.params.sid;
+  try {
+    const results = await LcrRoute.retrieve(lcr_route_sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const route = results[0];
+    route.lcr_carrier_set_entries = await LcrCarrierSetEntry.retrieveAllByLcrRouteSid(route.lcr_route_sid);
+    res.status(200).json(route);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+
+module.exports = router;

lib/routes/api/lcrs.js

@@ -0,0 +1,138 @@
+const router = require('express').Router();
+const Lcr = require('../../models/lcr');
+const LcrCarrierSetEntry = require('../../models/lcr-carrier-set-entry');
+const LcrRoutes = require('../../models/lcr-route');
+const decorate = require('./decorate');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const sysError = require('../error');
+const ServiceProvider = require('../../models/service-provider');
+
+const validateAssociatedTarget = async(req, sid) => {
+  const {lookupAccountBySid} = req.app.locals;
+  if (req.body.account_sid) {
+    // Add only for account
+    req.body.service_provider_sid = null;
+    const account = await lookupAccountBySid(req.body.account_sid);
+    if (!account) throw new DbErrorBadRequest('unknown account_sid');
+    const lcr = await Lcr.retrieveAllByAccountSid(req.body.account_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Account: ${account.name}  already has an active call routing table.`);
+    }
+  } else if (req.body.service_provider_sid) {
+    const serviceProviders = await ServiceProvider.retrieve(req.body.service_provider_sid);
+    if (serviceProviders.length === 0) throw new DbErrorBadRequest('unknown service_provider_sid');
+    const serviceProvider = serviceProviders[0];
+    const lcr = await Lcr.retrieveAllByServiceProviderSid(req.body.service_provider_sid);
+    if (lcr.length > 0 && (!sid || sid !== lcr[0].lcr_sid)) {
+      throw new DbErrorBadRequest(`Service Provider: ${serviceProvider.name} already 
+      has an active call routing table.`);
+    }
+  }
+};
+
+const validateAdd = async(req) => {
+  if (req.user.hasAccountAuth) {
+    // Account just create LCR for himself
+    req.body.account_sid = req.user.account_sid;
+  } else if (req.user.hasServiceProviderAuth) {
+    // SP just can create LCR for himself
+    req.body.service_provider_sid = req.user.service_provider_sid;
+    req.body.account_sid = null;
+  }
+
+  await validateAssociatedTarget(req);
+  // check if lcr_carrier_set_entry is available
+  if (req.body.lcr_carrier_set_entry) {
+    const e = await LcrCarrierSetEntry.retrieve(req.body.lcr_carrier_set_entry);
+    if (e.length === 0) throw new DbErrorBadRequest('unknown lcr_carrier_set_entry');
+  }
+
+};
+
+const validateUserPermissionForExistingEntity = async(req, sid) => {
+  const r = await Lcr.retrieve(sid);
+  if (r.length === 0) {
+    throw new DbErrorBadRequest('unknown lcr_sid');
+  }
+  const lcr = r[0];
+  if (req.user.hasAccountAuth) {
+    if (lcr.account_sid != req.user.account_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  } else if (req.user.hasServiceProviderAuth) {
+    if (lcr.service_provider_sid != req.user.service_provider_sid) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+};
+
+const validateUpdate = async(req, sid) => {
+  await validateUserPermissionForExistingEntity(req, sid);
+  await validateAssociatedTarget(req, sid);
+};
+
+const validateDelete = async(req, sid) => {
+  if (req.user.hasAccountAuth) {
+    /* can only delete Lcr for the user's account */
+    const r = await Lcr.retrieve(sid);
+    const lcr = r.length > 0 ? r[0] : null;
+    if (!lcr || (req.user.account_sid && lcr.account_sid != req.user.account_sid)) {
+      throw new DbErrorBadRequest('unknown lcr_sid');
+    }
+  }
+  await Lcr.releaseDefaultEntry(sid);
+  // fetch lcr route
+  const lcr_routes = await LcrRoutes.retrieveAllByLcrSid(sid);
+  // delete all lcr carrier set entries
+  for (const e of lcr_routes) {
+    await LcrCarrierSetEntry.deleteByLcrRouteSid(e.lcr_route_sid);
+  }
+
+  // delete all lcr routes
+  await LcrRoutes.deleteByLcrSid(sid);
+};
+
+const preconditions = {
+  add: validateAdd,
+  update: validateUpdate,
+  delete: validateDelete
+};
+
+decorate(router, Lcr, ['add', 'update', 'delete'], preconditions);
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = req.user.hasAdminAuth ?
+      await Lcr.retrieveAll() : req.user.hasAccountAuth ?
+        await Lcr.retrieveAllByAccountSid(req.user.hasAccountAuth ? req.user.account_sid : null) :
+        await Lcr.retrieveAllByServiceProviderSid(req.user.service_provider_sid);
+
+    for (const lcr of results) {
+      lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    }
+    res.status(200).json(results);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const results = await Lcr.retrieve(req.params.sid);
+    if (results.length === 0) return res.sendStatus(404);
+    const lcr = results[0];
+    if (req.user.hasAccountAuth && lcr.account_sid !== req.user.account_sid) {
+      return res.sendStatus(404);
+    } else if (req.user.hasServiceProviderAuth && lcr.service_provider_sid !== req.user.service_provider_sid) {
+      return res.sendStatus(404);
+    }
+    lcr.number_routes = await LcrRoutes.countAllByLcrSid(lcr.lcr_sid);
+    return res.status(200).json(lcr);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/limits.js

@@ -0,0 +1,133 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const AccountLimits = require('../../models/account-limits');
+const ServiceProviderLimits = require('../../models/service-provider-limits');
+const {parseAccountSid, parseServiceProviderSid} = require('./utils');
+const {promisePool} = require('../../db');
+const sqlDeleteSPLimits = `
+DELETE FROM service_provider_limits 
+WHERE service_provider_sid = ? 
+`;
+const sqlDeleteSPLimitsByCategory = `
+DELETE FROM service_provider_limits 
+WHERE service_provider_sid = ? 
+AND category = ?
+`;
+const sqlDeleteAccountLimits = `
+DELETE FROM account_limits 
+WHERE account_sid = ? 
+`;
+const sqlDeleteAccountLimitsByCategory = `
+DELETE FROM account_limits 
+WHERE account_sid = ? 
+AND category = ?
+`;
+
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const {
+    category,
+    quantity
+  } = req.body;
+
+  try {
+    let service_provider_sid;
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) {
+      if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
+        logger.error('POST /SpeechCredentials invalid credentials');
+        return res.sendStatus(403);
+      }
+      service_provider_sid = parseServiceProviderSid(req);
+    }
+
+    let uuid;
+    if (account_sid) {
+      const existing = (await AccountLimits.retrieve(account_sid) || [])
+        .find((el) => el.category === category);
+      if (existing) {
+        uuid = existing.account_limits_sid;
+        await AccountLimits.update(uuid, {category, quantity});
+      }
+      else {
+        uuid = await AccountLimits.make({
+          account_sid,
+          category,
+          quantity
+        });
+      }
+    }
+    else {
+      const existing = (await ServiceProviderLimits.retrieve(service_provider_sid) || [])
+        .find((el) => el.category === category);
+      if (existing) {
+        uuid = existing.service_provider_limits_sid;
+        await ServiceProviderLimits.update(uuid, {category, quantity});
+      }
+      else {
+        uuid = await ServiceProviderLimits.make({
+          service_provider_sid,
+          category,
+          quantity
+        });
+      }
+    }
+    res.status(201).json({sid: uuid});
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+/**
+ * retrieve all limits for an account or service provider
+ */
+router.get('/', async(req, res) => {
+  let service_provider_sid;
+  const logger = req.app.locals.logger;
+
+  try {
+    const account_sid = parseAccountSid(req);
+    if (!account_sid) service_provider_sid = parseServiceProviderSid(req);
+    const limits = account_sid ?
+      await AccountLimits.retrieve(account_sid) :
+      await ServiceProviderLimits.retrieve(service_provider_sid);
+
+    if (req.query?.category) {
+      return res.status(200).json(limits.filter((el) => el.category === req.query.category));
+    }
+    res.status(200).json(limits);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.delete('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const account_sid = parseAccountSid(req);
+    const {category} = req.query;
+    const service_provider_sid = parseServiceProviderSid(req);
+    if (account_sid) {
+      if (category) {
+        await promisePool.execute(sqlDeleteAccountLimitsByCategory, [account_sid, category]);
+      }
+      else {
+        await promisePool.execute(sqlDeleteAccountLimits, [account_sid]);
+      }
+    }
+    else {
+      if (category) {
+        await promisePool.execute(sqlDeleteSPLimitsByCategory, [service_provider_sid, category]);
+      }
+      else {
+        await promisePool.execute(sqlDeleteSPLimits, [service_provider_sid]);
+      }
+    }
+    res.status(204).end();
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+module.exports = router;

lib/routes/api/login.js

@@ -1,61 +1,116 @@
 const router = require('express').Router();
-const {getMysqlConnection} = require('../../db');
+const jwt = require('jsonwebtoken');
 const {verifyPassword} = require('../../utils/password-utils');
-
+const {promisePool} = require('../../db');
+const {cacheClient} = require('../../helpers');
+const Account = require('../../models/account');
+const ServiceProvider = require('../../models/service-provider');
+const sysError = require('../error');
+const retrievePemissionsSql = `
+SELECT p.name 
+FROM permissions p, user_permissions up 
+WHERE up.permission_sid = p.permission_sid 
+AND up.user_sid = ? 
+`;
 const retrieveSql = 'SELECT * from users where name = ?';
 const tokenSql = 'SELECT token from api_keys where account_sid IS NULL AND service_provider_sid IS NULL';
 
 
-router.post('/', (req, res) => {
-  const logger = req.app.locals.logger;
+router.post('/', async(req, res) => {
+  const {logger, incrKey, retrieveKey} = req.app.locals;
   const {username, password} = req.body;
   if (!username || !password) {
     logger.info('Bad POST to /login is missing username or password');
     return res.sendStatus(400);
   }
 
-  getMysqlConnection((err, conn) => {
-    if (err) {
-      logger.error({err}, 'Error getting db connection');
+  try {
+    const [r] = await promisePool.query(retrieveSql, username);
+    if (r.length === 0) {
+      logger.info(`Failed login attempt for user ${username}`);
+      return res.sendStatus(403);
+    }
+    logger.info({r}, 'successfully retrieved user account');
+
+    const maxLoginAttempts = process.env.LOGIN_ATTEMPTS_MAX_RETRIES || 6;
+    const loginAttempsBlocked = await retrieveKey(`login:${r[0].user_sid}`) >= maxLoginAttempts;
+
+    if (loginAttempsBlocked) {
+      logger.info(`User ${r[0].user_sid} was blocked due to excessive login attempts with incorrect credentials.`);
+      return res.status(403)
+        .json({error: 'Maximum login attempts reached. Please try again later or reset your password.'});
+    }
+
+    const isCorrect = await verifyPassword(r[0].hashed_password, password);
+    if (!isCorrect) {
+      const attempTime = process.env.LOGIN_ATTEMPTS_TIME || 1800;
+      const newAttempt = await incrKey(`login:${r[0].user_sid}`, attempTime)
+        .catch((err) => logger.error({err}, 'Error adding logging attempt to redis'));
+      if (newAttempt >= maxLoginAttempts) {
+        logger.info(`User ${r[0].user_sid} is now blocked due to excessive login attempts with incorrect credentials.`);
+        return res.status(403)
+          .json({error: `Maximum login attempts reached. Please try again in ${attempTime} seconds.`});
+      }
+      return res.sendStatus(403);
+    }
+    const force_change = !!r[0].force_change;
+    const [t] = await promisePool.query(tokenSql);
+    if (t.length === 0) {
+      logger.error('Database has no admin token provisioned...run reset_admin_password');
       return res.sendStatus(500);
     }
-    conn.query(retrieveSql, [username], async(err, results) => {
-      conn.release();
-      if (err) {
-        logger.error({err}, 'Error getting db connection');
-        return res.sendStatus(500);
-      }
-      if (0 === results.length) {
-        logger.info(`Failed login attempt for user ${username}`);
-        return res.sendStatus(403);
-      }
 
-      logger.info({results}, 'successfully retrieved account');
-      const isCorrect = await verifyPassword(results[0].hashed_password, password);
-      if (!isCorrect) return res.sendStatus(403);
+    const [p] = await promisePool.query(retrievePemissionsSql, r[0].user_sid);
+    const permissions = p.map((x) => x.name);
+    const obj = {user_sid: r[0].user_sid, scope: 'admin', force_change, permissions};
+    if (r[0].service_provider_sid && r[0].account_sid) {
+      const account = await Account.retrieve(r[0].account_sid);
+      const service_provider = await ServiceProvider.retrieve(r[0].service_provider_sid);
+      obj.scope = 'account';
+      obj.service_provider_sid = r[0].service_provider_sid;
+      obj.account_sid = r[0].account_sid;
+      obj.account_name = account[0].name;
+      obj.service_provider_name = service_provider[0].name;
+    }
+    else if (r[0].service_provider_sid) {
+      const service_provider = await ServiceProvider.retrieve(r[0].service_provider_sid);
+      obj.scope = 'service_provider';
+      obj.service_provider_sid = r[0].service_provider_sid;
+      obj.service_provider_name = service_provider[0].name;
+    }
+    const payload = {
+      scope: obj.scope,
+      permissions,
+      ...(obj.service_provider_sid && {
+        service_provider_sid: obj.service_provider_sid,
+        service_provider_name: obj.service_provider_name
+      }),
+      ...(obj.account_sid && {
+        account_sid: obj.account_sid,
+        account_name: obj.account_name,
+        service_provider_name: obj.service_provider_name
+      }),
+      user_sid: obj.user_sid
+    };
 
-      const force_change = !!results[0].force_change;
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60;
+    const token = jwt.sign(
+      payload,
+      process.env.JWT_SECRET,
+      { expiresIn }
+    );
+    res.json({token, ...obj});
 
-      getMysqlConnection((err, conn) => {
-        if (err) {
-          logger.error({err}, 'Error getting db connection');
-          return res.sendStatus(500);
-        }
-        conn.query(tokenSql, (err, tokenResults) => {
-          conn.release();
-          if (err) {
-            logger.error({err}, 'Error getting db connection');
-            return res.sendStatus(500);
-          }
-          if (0 === tokenResults.length) {
-            logger.error('Database has no admin token provisioned...run reset_admin_password');
-            return res.sendStatus(500);
-          }
-          res.json({user_sid: results[0].user_sid, force_change, token: tokenResults[0].token});
-        });
-      });
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', obj.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
     });
-  });
+
+  } catch (err) {
+    sysError(logger, res, err);
+  }
 });
 
 

lib/routes/api/logout.js

@@ -1,19 +1,18 @@
 const router = require('express').Router();
 const debug = require('debug')('jambonz:api-server');
-const {hashString} = require('../../utils/password-utils');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 
 router.post('/', async(req, res) => {
-  const {logger, addKey} = req.app.locals;
-  const {jwt} = req.user;
+  const {logger} = req.app.locals;
+  const {user_sid} = req.user;
 
-  debug(`adding jwt to blacklist: ${jwt}`);
+  debug(`logout user and invalidate jwt token for user: ${user_sid}`);
 
   try {
-    /* add key to blacklist */
-    const s = `jwt:${hashString(jwt)}`;
-    const result = await addKey(s, '1', 3600);
-    debug(`result from adding ${s}: ${result}`);
+    const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+    await cacheClient.delete(redisKey);
+
     res.sendStatus(204);
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/password-settings.js

@@ -0,0 +1,45 @@
+const router = require('express').Router();
+const sysError = require('../error');
+const PasswordSettings = require('../../models/password-settings');
+const { DbErrorBadRequest } = require('../../utils/errors');
+
+const validate = (obj) => {
+  if (obj.min_password_length && (
+    obj.min_password_length < 8 ||
+    obj.min_password_length > 20
+  )) {
+    throw new DbErrorBadRequest('invalid min_password_length property: should be between 8-20');
+  }
+};
+
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    if (!req.user.hasAdminAuth) {
+      return res.sendStatus(403);
+    }
+    validate(req.body);
+    const [existing] = (await PasswordSettings.retrieve() || []);
+    if (existing) {
+      await PasswordSettings.update(req.body);
+    } else {
+      await PasswordSettings.make(req.body);
+    }
+    res.status(201).json({});
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  try {
+    const [results] = (await PasswordSettings.retrieve() || []);
+    return res.status(200).json(results || {min_password_length: 8});
+  }
+  catch (err) {
+    sysError(logger, res, err);
+  }
+});
+module.exports = router;

lib/routes/api/phone-numbers.js

@@ -1,8 +1,10 @@
 const router = require('express').Router();
-const {DbErrorUnprocessableRequest, DbErrorBadRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
 const PhoneNumber = require('../../models/phone-number');
 const VoipCarrier = require('../../models/voip-carrier');
+const Account = require('../../models/account');
 const decorate = require('./decorate');
+const {promisePool} = require('../../db');
 const {e164} = require('../../utils/phone-number-utils');
 const preconditions = {
   'add': validateAdd,
@@ -10,6 +12,7 @@ const preconditions = {
   'update': validateUpdate
 };
 const sysError = require('../error');
+const { parsePhoneNumberSid } = require('./utils');
 
 
 /* check for required fields when adding */
@@ -20,6 +23,10 @@ async function validateAdd(req) {
       req.body.account_sid = req.user.account_sid;
     }
 
+    if (req.user.hasServiceProviderAuth) {
+      req.body.service_provider_sid = req.user.service_provider_sid;
+    }
+
     if (!req.body.number) throw new DbErrorBadRequest('number is required');
     const formattedNumber = e164(req.body.number);
     req.body.number = formattedNumber;
@@ -41,11 +48,11 @@ async function checkInUse(req, sid) {
   const phoneNumber = await PhoneNumber.retrieve(sid);
   if (req.user.hasAccountAuth) {
     if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot delete a phone number that belongs to another account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
   if (!req.user.hasAccountAuth && phoneNumber.account_sid) {
-    throw new DbErrorUnprocessableRequest('cannot delete phone number that is assigned to an account');
+    throw new DbErrorForbidden('insufficient privileges');
   }
 }
 
@@ -57,10 +64,23 @@ async function validateUpdate(req, sid) {
   const phoneNumber = await PhoneNumber.retrieve(sid);
   if (req.user.hasAccountAuth) {
     if (phoneNumber && phoneNumber.length && phoneNumber[0].account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('cannot operate on a phone number that belongs to another account');
+      throw new DbErrorForbidden('insufficient privileges');
     }
   }
+  if (req.user.hasServiceProviderAuth) {
+    let service_provider_sid;
 
+    if (!phoneNumber[0].service_provider_sid) {
+      const [r] = await Account.retrieve(phoneNumber[0].account_sid);
+      service_provider_sid = r.service_provider_sid;
+    } else {
+      service_provider_sid = phoneNumber[0].service_provider_sid;
+    }
+
+    if (phoneNumber && phoneNumber.length && service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('insufficient privileges');
+    }
+  }
   // TODO: if we are assigning to an account, verify it exists
 
   // TODO: if we are assigning to an application, verify it is associated to the same account
@@ -74,7 +94,9 @@ decorate(router, PhoneNumber, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
+    const results = req.user.hasServiceProviderAuth ?
+      await PhoneNumber.retrieveAllForSP(req.user.service_provider_sid) :
+      await PhoneNumber.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -85,9 +107,22 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parsePhoneNumberSid(req);
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await PhoneNumber.retrieve(req.params.sid, account_sid);
+    const results = await PhoneNumber.retrieve(sid, account_sid);
     if (results.length === 0) return res.status(404).end();
+
+    if (req.user.hasServiceProviderAuth && results.length === 1) {
+      const account_sid = results[0].account_sid;
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]);
+      if (r.length === 1 && r[0].service_provider_sid !== req.user.service_provider_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+    if (req.user.hasAccountAuth && results.length > 1) {
+      return res.status(200).json(results.filter((r) => r.phone_number_sid === sid)[0]);
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/recent-calls.js

@@ -2,33 +2,67 @@ const router = require('express').Router();
 const sysError = require('../error');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {getHomerApiKey, getHomerSipTrace, getHomerPcap} = require('../../utils/homer-utils');
+const {getJaegerTrace} = require('../../utils/jaeger-utils');
+const Account = require('../../models/account');
+const {
+  getS3Object,
+  getGoogleStorageObject,
+  getAzureStorageObject,
+  deleteS3Object,
+  deleteGoogleStorageObject,
+  deleteAzureStorageObject
+} = require('../../utils/storage-utils');
+
 const parseAccountSid = (url) => {
   const arr = /Accounts\/([^\/]*)/.exec(url);
   if (arr) return arr[1];
 };
 
+const parseServiceProviderSid = (url) => {
+  const arr = /ServiceProviders\/([^\/]*)/.exec(url);
+  if (arr) return arr[1];
+};
+
 router.get('/', async(req, res) => {
-  const {logger, queryCdrs} = req.app.locals;
+  const {logger, queryCdrs, queryCdrsSP} = req.app.locals;
   try {
     logger.debug({opts: req.query}, 'GET /RecentCalls');
     const account_sid = parseAccountSid(req.originalUrl);
-    const {page, count, trunk, direction, days, answered, start, end} = req.query || {};
+    const service_provider_sid = account_sid ? null : parseServiceProviderSid(req.originalUrl);
+    const {page, count, trunk, direction, days, answered, start, end, filter} = req.query || {};
     if (!page || page < 1) throw new DbErrorBadRequest('missing or invalid "page" query arg');
     if (!count || count < 25 || count > 500) throw new DbErrorBadRequest('missing or invalid "count" query arg');
 
-    const data = await queryCdrs({
-      account_sid,
-      page,
-      page_size: count,
-      trunk,
-      direction,
-      days,
-      answered,
-      start: days ? undefined : start,
-      end: days ? undefined : end,
-    });
-
-    res.status(200).json(data);
+    if (account_sid) {
+      const data = await queryCdrs({
+        account_sid,
+        page,
+        page_size: count,
+        trunk,
+        direction,
+        days,
+        answered,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+        filter
+      });
+      res.status(200).json(data);
+    }
+    else {
+      const data = await queryCdrsSP({
+        service_provider_sid,
+        page,
+        page_size: count,
+        trunk,
+        direction,
+        days,
+        answered,
+        start: days ? undefined : start,
+        end: days ? undefined : end,
+        filter
+      });
+      res.status(200).json(data);
+    }
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -51,12 +85,12 @@ router.get('/:call_id', async(req, res) => {
   }
 });
 
-router.get('/:call_id/pcap', async(req, res) => {
+router.get('/:call_id/:method/pcap', async(req, res) => {
   const {logger} = req.app.locals;
   try {
     const token = await getHomerApiKey(logger);
     if (!token) return res.sendStatus(400, {msg: 'getHomerApiKey: Failed to get Homer API token; check server config'});
-    const stream = await getHomerPcap(logger, token, [req.params.call_id]);
+    const stream = await getHomerPcap(logger, token, [req.params.call_id], req.params.method);
     if (!stream) {
       logger.info(`getHomerApiKey: unable to get sip traces from Homer for ${req.params.call_id}`);
       return res.sendStatus(404);
@@ -72,4 +106,100 @@ router.get('/:call_id/pcap', async(req, res) => {
   }
 });
 
+router.get('/trace/:trace_id', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {trace_id} = req.params;
+  try {
+    const obj = await getJaegerTrace(logger, trace_id);
+    if (!obj) {
+      logger.info(`/RecentCalls: unable to get spans from jaeger for ${trace_id}`);
+      return res.sendStatus(404);
+    }
+    res.status(200).json(obj.result);
+  } catch (err) {
+    logger.error({err}, `/RecentCalls error retrieving jaeger trace ${trace_id}`);
+    res.sendStatus(500);
+  }
+});
+
+router.get('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+    const getOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+    let stream;
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        stream = await getS3Object(logger, getOptions);
+        break;
+      case 'google':
+        stream = await getGoogleStorageObject(logger, getOptions);
+        break;
+      case 'azure':
+        stream = await getAzureStorageObject(logger, getOptions);
+        break;
+      default:
+        logger.error(`There is no handler for fetching record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.set({
+      'Content-Type': `audio/${format || 'mp3'}`
+    });
+    if (stream) {
+      stream.pipe(res);
+    } else {
+      return res.sendStatus(404);
+    }
+  }  catch (err) {
+    logger.error({err}, ` error retrieving recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
+router.delete('/:call_sid/record/:year/:month/:day/:format', async(req, res) => {
+  const {logger} = req.app.locals;
+  const {call_sid, year, month, day, format} = req.params;
+
+  try {
+    const account_sid = parseAccountSid(req.originalUrl);
+    const r = await Account.retrieve(account_sid);
+    if (r.length === 0 || !r[0].bucket_credential) return res.sendStatus(404);
+    const {bucket_credential} = r[0];
+
+    const deleteOptions  = {
+      ...bucket_credential,
+      key: `${year}/${month}/${day}/${call_sid}.${format || 'mp3'}`
+    };
+
+    switch (bucket_credential.vendor) {
+      case 'aws_s3':
+      case 's3_compatible':
+        await deleteS3Object(logger, deleteOptions);
+        break;
+      case 'google':
+        await deleteGoogleStorageObject(logger, deleteOptions);
+        break;
+      case 'azure':
+        await deleteAzureStorageObject(logger, deleteOptions);
+        break;
+      default:
+        logger.error(`There is no handler for deleting record from ${bucket_credential.vendor}`);
+        return res.sendStatus(500);
+    }
+    res.sendStatus(204);
+  }  catch (err) {
+    logger.error({err}, ` error deleting recording ${call_sid}`);
+    res.sendStatus(404);
+  }
+});
+
 module.exports = router;

lib/routes/api/register.js

@@ -4,7 +4,8 @@ const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/er
 const {promisePool} = require('../../db');
 const {doGithubAuth, doGoogleAuth, doLocalAuth} = require('../../utils/oauth-utils');
 const {validateEmail} = require('../../utils/email-utils');
-const uuid = require('uuid').v4;
+const {cacheClient} = require('../../helpers');
+const { v4: uuid } = require('uuid');
 const short = require('short-uuid');
 const translator = short();
 const jwt = require('jsonwebtoken');
@@ -15,8 +16,9 @@ const insertUserSql = `INSERT into users
 (user_sid, account_sid, name, email, provider, provider_userid, email_validated) 
 values (?, ?, ?, ?, ?, ?, 1)`;
 const insertUserLocalSql = `INSERT into users 
-(user_sid, account_sid, name, email, email_activation_code, email_validated, provider, hashed_password) 
-values (?, ?, ?, ?, ?, 0, 'local', ?)`;
+(user_sid, account_sid, name, email, email_activation_code, email_validated, provider,
+hashed_password, service_provider_sid) 
+values (?, ?, ?, ?, ?, 0, 'local', ?, ?)`;
 const insertAccountSql = `INSERT into accounts 
 (account_sid, service_provider_sid, name, is_active, webhook_secret, trial_end_date) 
 values (?, ?, ?, ?, ?, CURDATE() + INTERVAL 21 DAY)`;
@@ -35,7 +37,7 @@ const insertSignupHistorySql = `INSERT into signup_history
 values (?, ?)`;
 
 const addLocalUser = async(logger, user_sid, account_sid,
-  name, email, email_activation_code, passwordHash) => {
+  name, email, email_activation_code, passwordHash, service_provider_sid) => {
   const [r] = await promisePool.execute(insertUserLocalSql,
     [
       user_sid,
@@ -43,7 +45,8 @@ const addLocalUser = async(logger, user_sid, account_sid,
       name,
       email,
       email_activation_code,
-      passwordHash
+      passwordHash,
+      service_provider_sid
     ]);
   debug({r}, 'Result from adding user');
 };
@@ -144,7 +147,7 @@ router.post('/', async(req, res) => {
       const user = await doGithubAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from github');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         email_validated: user.email_validated,
         avatar_url: user.avatar_url,
@@ -156,7 +159,7 @@ router.post('/', async(req, res) => {
       const user = await doGoogleAuth(logger, req.body);
       logger.info({user}, 'retrieved user details from google');
       Object.assign(userProfile, {
-        name: user.name || user.email,
+        name: user.email || user.email,
         email: user.email,
         email_validated: user.verified_email,
         picture: user.picture,
@@ -169,7 +172,7 @@ router.post('/', async(req, res) => {
       logger.info({user}, 'retrieved user details for local provider');
       debug({user}, 'retrieved user details for local provider');
       Object.assign(userProfile, {
-        name: user.name,
+        name: user.email,
         email: user.email,
         provider: 'local',
         email_activation_code: user.email_activation_code
@@ -279,7 +282,8 @@ router.post('/', async(req, res) => {
         const passwordHash = await generateHashedPassword(req.body.password);
         debug(`hashed password: ${passwordHash}`);
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
-          userProfile.name, userProfile.email, userProfile.email_activation_code, passwordHash);
+          userProfile.name, userProfile.email, userProfile.email_activation_code,
+          passwordHash, req.body.service_provider_sid);
         debug('added local user');
       }
       else {
@@ -292,17 +296,25 @@ router.post('/', async(req, res) => {
       const callStatusSid = uuid();
       const helloWordSid = uuid();
       const dialTimeSid = uuid();
+      const echoSid = uuid();
 
-      /* 3 webhooks */
-      await promisePool.execute(insertWebookSql, [callStatusSid, 'https://public-apps.jambonz.us/call-status', 'POST']);
-      await promisePool.execute(insertWebookSql, [helloWordSid, 'https://public-apps.jambonz.us/hello-world', 'POST']);
-      await promisePool.execute(insertWebookSql, [dialTimeSid, 'https://public-apps.jambonz.us/dial-time', 'POST']);
+      /* 4 webhooks */
+      await promisePool.execute(insertWebookSql,
+        [callStatusSid, 'https://public-apps.jambonz.cloud/call-status', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [helloWordSid, 'https://public-apps.jambonz.cloud/hello-world', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [dialTimeSid, 'https://public-apps.jambonz.cloud/dial-time', 'POST']);
+      await promisePool.execute(insertWebookSql,
+        [echoSid, 'https://public-apps.jambonz.cloud/echo', 'POST']);
 
       /* 2 applications */
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'hello world',
         helloWordSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
       await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'dial time clock',
         dialTimeSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
+      await promisePool.execute(insertApplicationSql, [uuid(), userProfile.account_sid, 'simple echo test',
+        echoSid, callStatusSid, 'google', 'en-US', 'en-US-Wavenet-C', 'google', 'en-US']);
 
       Object.assign(userProfile, {
         pristine: true,
@@ -326,7 +338,7 @@ router.post('/', async(req, res) => {
 
         await addLocalUser(logger, userProfile.user_sid, userProfile.account_sid,
           userProfile.name, userProfile.email, userProfile.email_activation_code,
-          passwordHash);
+          passwordHash, req.body.service_provider_sid);
 
         /* note: we deactivate the old user once the new email is validated */
       }
@@ -338,16 +350,21 @@ router.post('/', async(req, res) => {
         /* deactivate the old/replaced user */
         const [r] = await promisePool.execute('DELETE FROM users WHERE user_sid = ?', [user_sid]);
         logger.debug({r}, 'register - removed old user');
+        const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+        await cacheClient.delete(redisKey);
       }
     }
 
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60 ;
     // generate a json web token for this user
     const token = jwt.sign({
       user_sid: userProfile.user_sid,
       account_sid: userProfile.account_sid,
+      service_provider_sid: req.body.service_provider_sid,
+      scope: 'account',
       email: userProfile.email,
       name: userProfile.name
-    }, process.env.JWT_SECRET, { expiresIn: '1h' });
+    }, process.env.JWT_SECRET, { expiresIn });
 
     logger.debug({
       user_sid: userProfile.user_sid,
@@ -356,6 +373,13 @@ router.post('/', async(req, res) => {
 
     res.json({jwt: token, ...userProfile});
 
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', userProfile.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
   } catch (err) {
     debug(err, 'Error');
     sysError(logger, res, err);

lib/routes/api/sbcs.js

@@ -2,25 +2,46 @@ const router = require('express').Router();
 const Sbc = require('../../models/sbc');
 const decorate = require('./decorate');
 const sysError = require('../error');
-//const {DbErrorBadRequest} = require('../../utils/errors');
-//const {promisePool} = require('../../db');
+const {DbErrorBadRequest} = require('../../utils/errors');
+const {promisePool} = require('../../db');
 
-decorate(router, Sbc, ['add', 'delete']);
+const validate = (req, res) => {
+  if (req.user.hasScope('admin')) return;
+  res.status(403).json({
+    status: 'fail',
+    message: 'insufficient privileges'
+  });
+};
+
+const preconditions = {
+  'add': validate,
+  'delete': validate
+};
+
+decorate(router, Sbc, ['add', 'delete'], preconditions);
 
 /* list */
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const service_provider_sid = req.query.service_provider_sid;
-    /*
+    let service_provider_sid = req.query.service_provider_sid;
+
     if (req.user.hasAccountAuth) {
       const [r] = await promisePool.query('SELECT * from accounts WHERE account_sid = ?', req.user.account_sid);
-      if (0 === r.length) throw new Error('invalid account_sid');
+      if (0 === r.length) throw new DbErrorBadRequest('invalid account_sid');
+
       service_provider_sid = r[0].service_provider_sid;
     }
-    if (!service_provider_sid) throw new DbErrorBadRequest('missing service_provider_sid in query');
-    */
-    const results = await Sbc.retrieveAll(service_provider_sid);
+
+    if (req.user.hasServiceProviderAuth) {
+      service_provider_sid = req.user.service_provider_sid;
+    }
+
+    /** generally, we have a global set of SBCs that all accounts use.
+     * However, we can have a set of SBCs that are specific for use by a service provider.
+     */
+    let results = await Sbc.retrieveAll(service_provider_sid);
+    if (results.length === 0) results = await Sbc.retrieveAll();
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/service-providers.js

@@ -1,17 +1,22 @@
 const router = require('express').Router();
 const {promisePool} = require('../../db');
-const {DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorForbidden} = require('../../utils/errors');
 const Webhook = require('../../models/webhook');
 const ServiceProvider = require('../../models/service-provider');
 const Account = require('../../models/account');
 const VoipCarrier = require('../../models/voip-carrier');
 const Application = require('../../models/application');
 const PhoneNumber = require('../../models/phone-number');
-const {hasServiceProviderPermissions, parseServiceProviderSid} = require('./utils');
+const ApiKey = require('../../models/api-key');
+const {
+  hasServiceProviderPermissions,
+  parseServiceProviderSid,
+  parseVoipCarrierSid,
+} = require('./utils');
 const sysError = require('../error');
 const decorate = require('./decorate');
 const preconditions = {
-  'delete': noActiveAccounts
+  'delete': noActiveAccountsOrUsers
 };
 const sqlDeleteSipGateways = `DELETE from sip_gateways 
 WHERE voip_carrier_sid IN (
@@ -26,37 +31,100 @@ WHERE voip_carrier_sid IN (
   WHERE service_provider_sid = ?
 )`;
 
-/* can not delete a service provider if it has any active accounts */
-async function noActiveAccounts(req, sid) {
+/* only admin users can add a service provider */
+function validateAdd(req) {
+  if (!req.user.hasAdminAuth) {
+    throw new DbErrorForbidden('only admin users can add a service provider');
+  }
+}
+
+async function validateRetrieve(req) {
+  try {
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider') || req.user.hasScope('account')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions');
+  } catch (error) {
+    throw error;
+  }
+}
+
+function validateUpdate(req) {
+  try {
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    if (req.user.hasScope('admin')) {
+      return;
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      if (service_provider_sid === req.user.service_provider_sid) return;
+    }
+
+    throw new DbErrorForbidden('insufficient permissions to update service provider');
+  } catch (error) {
+    throw error;
+  }
+}
+
+/* can not delete a service provider if it has any active accounts or users*/
+async function noActiveAccountsOrUsers(req, sid) {
+  if (!req.user.hasAdminAuth) {
+    throw new DbErrorForbidden('only admin users can delete a service provider');
+  }
   const activeAccounts = await ServiceProvider.getForeignKeyReferences('accounts.service_provider_sid', sid);
-  if (activeAccounts > 0) throw new DbErrorUnprocessableRequest('cannot delete service provider with active accounts');
+  const activeUsers = await ServiceProvider.getForeignKeyReferences('users.service_provider_sid', sid);
+  if (activeAccounts > 0 && activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
+
+  if (activeAccounts > 0) throw new DbErrorForbidden('insufficient privileges');
+  if (activeUsers > 0) throw new DbErrorForbidden('insufficient privileges');
 
   /* ok we can delete -- no active accounts.  remove carriers and speech credentials */
   await promisePool.execute('DELETE from speech_credentials WHERE service_provider_sid = ?', [sid]);
   await promisePool.query(sqlDeleteSipGateways, [sid]);
   await promisePool.query(sqlDeleteSmppGateways, [sid]);
   await promisePool.query('DELETE from voip_carriers WHERE service_provider_sid = ?', [sid]);
+  await promisePool.query('DELETE from api_keys WHERE service_provider_sid = ?', [sid]);
 }
 
 decorate(router, ServiceProvider, ['delete'], preconditions);
 
-router.use('/:sid/SpeechCredentials', hasServiceProviderPermissions, require('./speech-credentials'));
+router.use('/:sid/RecentCalls', hasServiceProviderPermissions, require('./recent-calls'));
+router.use('/:sid/Alerts', hasServiceProviderPermissions, require('./alerts'));
+router.use('/:sid/SpeechCredentials', require('./speech-credentials'));
+router.use('/:sid/Limits', hasServiceProviderPermissions, require('./limits'));
 router.use('/:sid/PredefinedCarriers', hasServiceProviderPermissions, require('./add-from-predefined-carrier'));
 router.get('/:sid/Accounts', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await Account.retrieveAll(service_provider_sid);
+    let results = await Account.retrieveAll(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
   }
 });
+
 router.get('/:sid/Applications', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await Application.retrieveAll(service_provider_sid);
+    let results = await Application.retrieveAll(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -65,8 +133,12 @@ router.get('/:sid/Applications', async(req, res) => {
 router.get('/:sid/PhoneNumbers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await PhoneNumber.retrieveAllForSP(service_provider_sid);
+    let results = await PhoneNumber.retrieveAllForSP(service_provider_sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -75,9 +147,15 @@ router.get('/:sid/PhoneNumbers', async(req, res) => {
 router.get('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    await validateRetrieve(req);
     const service_provider_sid = parseServiceProviderSid(req);
-    const results = await VoipCarrier.retrieveAllForSP(service_provider_sid);
-    res.status(200).json(results);
+    const carriers = await VoipCarrier.retrieveAllForSP(service_provider_sid);
+
+    if (req.user.hasScope('account')) {
+      return res.status(200).json(carriers.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid));
+    }
+
+    res.status(200).json(carriers);
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -85,6 +163,7 @@ router.get('/:sid/VoipCarriers', async(req, res) => {
 router.post('/:sid/VoipCarriers', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    validateUpdate(req);
     const service_provider_sid = parseServiceProviderSid(req);
     const uuid = await VoipCarrier.make({...req.body, service_provider_sid});
     res.status(201).json({sid: uuid});
@@ -95,7 +174,9 @@ router.post('/:sid/VoipCarriers', async(req, res) => {
 router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const rowsAffected = await VoipCarrier.update(req.params.voip_carrier_sid, req.body);
+    validateUpdate(req);
+    const sid = parseVoipCarrierSid(req);
+    const rowsAffected = await VoipCarrier.update(sid, req.body);
     if (rowsAffected === 0) {
       return res.sendStatus(404);
     }
@@ -104,12 +185,17 @@ router.put('/:sid/VoipCarriers/:voip_carrier_sid', async(req, res) => {
     sysError(logger, res, err);
   }
 });
-router.get(':sid/Acccounts', async(req, res) => {
+router.get('/:sid/ApiKeys', async(req, res) => {
   const logger = req.app.locals.logger;
+  const {sid} = req.params;
   try {
-    const service_provider_sid = parseServiceProviderSid(req);
-    const results = await Account.retrieveAll(service_provider_sid);
+    await validateRetrieve(req);
+    let results = await ApiKey.retrieveAllForSP(sid);
+    if (req.user.hasScope('account')) {
+      results = results.filter((r) => r.account_sid === req.user.account_sid);
+    }
     res.status(200).json(results);
+    await ApiKey.updateLastUsed(sid);
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -119,7 +205,7 @@ router.get(':sid/Acccounts', async(req, res) => {
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-
+    validateAdd(req);
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
     for (const prop of ['registration_hook']) {
@@ -142,6 +228,12 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
     const results = await ServiceProvider.retrieveAll();
+    logger.debug({results, user: req.user}, 'ServiceProvider.retrieveAll');
+    if (req.user.hasScope('service_provider') || req.user.hasScope('account')) {
+      logger.debug(`Filtering results for ${req.user.service_provider_sid}`);
+      return res.status(200).json(results.filter((e) => req.user.service_provider_sid === e.service_provider_sid));
+    }
+
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -152,7 +244,9 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await ServiceProvider.retrieve(req.params.sid);
+    await validateRetrieve(req);
+    const sid = parseServiceProviderSid(req);
+    const results = await ServiceProvider.retrieve(sid);
     if (results.length === 0) return res.status(404).end();
     return res.status(200).json(results[0]);
   }
@@ -163,9 +257,11 @@ router.get('/:sid', async(req, res) => {
 
 /* update */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    validateUpdate(req);
+    const sid = parseServiceProviderSid(req);
+
     // create webhooks if provided
     const obj = Object.assign({}, req.body);
     for (const prop of ['registration_hook']) {
@@ -174,15 +270,14 @@ router.put('/:sid', async(req, res) => {
           const sid = obj[prop]['webhook_sid'];
           delete obj[prop]['webhook_sid'];
           await Webhook.update(sid, obj[prop]);
-        }
-        else {
+        } else {
           const sid = await Webhook.make(obj[prop]);
           obj[`${prop}_sid`] = sid;
         }
-      }
-      else {
+      } else {
         obj[`${prop}_sid`] = null;
       }
+
       delete obj[prop];
     }
 
@@ -190,6 +285,7 @@ router.put('/:sid', async(req, res) => {
     if (rowsAffected === 0) {
       return res.status(404).end();
     }
+
     res.status(204).end();
   } catch (err) {
     sysError(logger, res, err);

lib/routes/api/signin.js

@@ -3,10 +3,17 @@ const router = require('express').Router();
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {promisePool} = require('../../db');
 const {verifyPassword} = require('../../utils/password-utils');
+const {cacheClient} = require('../../helpers');
 const jwt = require('jsonwebtoken');
 const sysError = require('../error');
+const retrievePermissionsSql = `
+SELECT p.name 
+FROM permissions p, user_permissions up 
+WHERE up.permission_sid = p.permission_sid 
+AND up.user_sid = ? 
+`;
 
-const validateRequest = async(req) => {
+const validateRequest = (req) => {
   const {email, password} = req.body || {};
 
   /* check required properties are there */
@@ -52,6 +59,7 @@ router.post('/', async(req, res) => {
       email: user.email,
       phone: user.phone,
       account_sid: user.account_sid,
+      service_provider_sid: a[0].service_provider_sid,
       force_change: !!user.force_change,
       provider: user.provider,
       provider_userid: user.provider_userid,
@@ -64,11 +72,22 @@ router.post('/', async(req, res) => {
       pristine: false
     });
 
+    const [p] = await promisePool.query(retrievePermissionsSql, user.user_sid);
+    const permissions = p.map((x) => x.name);
+
+    const expiresIn = parseInt(process.env.JWT_EXPIRES_IN || 60) * 60;
     // generate a json web token for this session
-    const token = jwt.sign({
+    const payload = {
+      scope: 'account',
+      permissions,
       user_sid: userProfile.user_sid,
-      account_sid: userProfile.account_sid
-    }, process.env.JWT_SECRET, { expiresIn: '1h' });
+      account_sid: userProfile.account_sid,
+      service_provider_sid: userProfile.service_provider_sid
+    };
+    const token = jwt.sign(payload,
+      process.env.JWT_SECRET,
+      { expiresIn }
+    );
 
     logger.debug({
       user_sid: userProfile.user_sid,
@@ -76,6 +95,14 @@ router.post('/', async(req, res) => {
     }, 'generated jwt');
 
     res.json({jwt: token, ...userProfile});
+
+    /* Store jwt based on user_id after successful login */
+    await cacheClient.set({
+      redisKey: cacheClient.generateRedisKey('jwt', userProfile.user_sid, 'v2'),
+      value: token,
+      time: expiresIn,
+    });
+
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/sip-gateways.js

@@ -1,11 +1,46 @@
 const router = require('express').Router();
 const SipGateway = require('../../models/sip-gateway');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
+//const {parseSipGatewaySid} = require('./utils');
 const decorate = require('./decorate');
 const sysError = require('../error');
 
+const checkUserScope = async(req, voip_carrier_sid) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+  if (req.user.hasAccountAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+
+    if ((!carrier.service_provider_sid || carrier.service_provider_sid === req.user.service_provider_sid) &&
+      (!carrier.account_sid || carrier.account_sid === req.user.account_sid)) {
+
+      if (req.method !== 'GET' && !carrier.account_sid) {
+        throw new DbErrorForbidden('insufficient privileges');
+      }
+
+      return;
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) {
+      throw new DbErrorBadRequest('invalid voip_carrier_sid');
+    }
+
+    if (carrier.service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+  }
+  throw new DbErrorForbidden('insufficient privileges');
+};
+
 const validate = async(req, sid) => {
-  const {lookupCarrierBySid, lookupSipGatewayBySid} = req.app.locals;
+  const {lookupSipGatewayBySid} = req.app.locals;
   let voip_carrier_sid;
 
   if (sid) {
@@ -17,13 +52,7 @@ const validate = async(req, sid) => {
     voip_carrier_sid = req.body.voip_carrier_sid;
     if (!voip_carrier_sid) throw new DbErrorBadRequest('missing voip_carrier_sid');
   }
-  if (req.hasAccountAuth) {
-    const carrier = await lookupCarrierBySid(voip_carrier_sid);
-    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
-    if (carrier.account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('user can not add gateway for voip_carrier belonging to other account');
-    }
-  }
+  await checkUserScope(req, voip_carrier_sid);
 };
 
 const preconditions = {
@@ -39,6 +68,7 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   const voip_carrier_sid = req.query.voip_carrier_sid;
   try {
+    await checkUserScope(req, voip_carrier_sid);
     if (!voip_carrier_sid) {
       logger.info('GET /SipGateways missing voip_carrier_sid param');
       return res.status(400).json({message: 'missing voip_carrier_sid query param'});

lib/routes/api/sip-realm.js

@@ -2,7 +2,7 @@ const router = require('express').Router();
 const {promisePool} = require('../../db');
 const {DbErrorBadRequest} = require('../../utils/errors');
 const {createDnsRecords, deleteDnsRecords} = require('../../utils/dns-utils');
-const uuid = require('uuid').v4;
+const { v4: uuid } = require('uuid');
 const sysError = require('../error');
 const insertDnsRecords = `INSERT INTO dns_records 
 (dns_record_sid, account_sid, record_type, record_id) 

lib/routes/api/smpp-gateways.js

@@ -1,11 +1,38 @@
 const router = require('express').Router();
 const SmppGateway = require('../../models/smpp-gateway');
-const {DbErrorBadRequest, DbErrorUnprocessableRequest} = require('../../utils/errors');
+const {DbErrorBadRequest, DbErrorForbidden} = require('../../utils/errors');
 const decorate = require('./decorate');
 const sysError = require('../error');
 
+const checkUserScope = async(req, voip_carrier_sid) => {
+  const {lookupCarrierBySid} = req.app.locals;
+  if (!voip_carrier_sid) {
+    throw new DbErrorBadRequest('missing voip_carrier_sid');
+  }
+
+  if (req.user.hasAdminAuth) return;
+
+  if (req.user.hasAccountAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+
+    if ((!carrier.service_provider_sid || carrier.service_provider_sid === req.user.service_provider_sid) &&
+      (!carrier.account_sid || carrier.account_sid === req.user.account_sid)) {
+      return;
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    const carrier = await lookupCarrierBySid(voip_carrier_sid);
+    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
+    if (carrier.service_provider_sid === req.user.service_provider_sid) {
+      return;
+    }
+  }
+  throw new DbErrorForbidden('insufficient privileges');
+};
+
 const validate = async(req, sid) => {
-  const {lookupCarrierBySid, lookupSmppGatewayBySid} = req.app.locals;
+  const {lookupSmppGatewayBySid} = req.app.locals;
   let voip_carrier_sid;
 
   if (sid) {
@@ -17,13 +44,8 @@ const validate = async(req, sid) => {
     voip_carrier_sid = req.body.voip_carrier_sid;
     if (!voip_carrier_sid) throw new DbErrorBadRequest('missing voip_carrier_sid');
   }
-  if (req.hasAccountAuth) {
-    const carrier = await lookupCarrierBySid(voip_carrier_sid);
-    if (!carrier) throw new DbErrorBadRequest('invalid voip_carrier_sid');
-    if (carrier.account_sid !== req.user.account_sid) {
-      throw new DbErrorUnprocessableRequest('user can not add gateway for voip_carrier belonging to other account');
-    }
-  }
+
+  await checkUserScope(req, voip_carrier_sid);
 };
 
 const preconditions = {
@@ -39,6 +61,7 @@ router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   const voip_carrier_sid = req.query.voip_carrier_sid;
   try {
+    await checkUserScope(req, voip_carrier_sid);
     if (!voip_carrier_sid) {
       logger.info('GET /SmppGateways missing voip_carrier_sid param');
       return res.status(400).json({message: 'missing voip_carrier_sid query param'});

lib/routes/api/sms-inbound.js

@@ -1,19 +1,35 @@
 const router = require('express').Router();
 const request = require('request');
 const getProvider = require('../../utils/sms-provider');
-const uuidv4 = require('uuid/v4');
+const { v4: uuidv4 } = require('uuid');
 const sysError = require('../error');
 let idx = 0;
 
+const getFsUrl = async(logger, retrieveSet, setName, provider) => {
+  if (process.env.K8S) return `http://${process.env.K8S_FEATURE_SERVER_SERVICE_NAME}:3000/v1/messaging/${provider}`;
 
-async function doSendResponse(res, respondFn, body) {
+  try {
+    const fs = await retrieveSet(setName);
+    if (0 === fs.length) {
+      logger.info('No available feature servers to handle createCall API request');
+      return ;
+    }
+    const f = fs[idx++ % fs.length];
+    logger.info({fs}, `feature servers available for createCall API request, selecting ${f}`);
+    return `${f}/v1/messaging/${provider}`;
+  } catch (err) {
+    logger.error({err}, 'getFsUrl: error retreving feature servers from redis');
+  }
+};
+
+const doSendResponse = async(res, respondFn, body) => {
   if (typeof respondFn === 'number') res.sendStatus(respondFn);
   else if (typeof respondFn !== 'function') res.sendStatus(200);
   else {
     const payload = await respondFn(body);
     res.status(200).json(payload);
   }
-}
+};
 
 router.post('/:provider', async(req, res) => {
   const provider = req.params.provider;
@@ -22,7 +38,7 @@ router.post('/:provider', async(req, res) => {
     lookupAppByPhoneNumber,
     logger
   } = req.app.locals;
-  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:active-fs`;
+  const setName = `${process.env.JAMBONES_CLUSTER_ID || 'default'}:fs-service-url`;
   logger.debug({path: req.path, body: req.body}, 'incomingSMS from carrier');
 
   // search for provider module
@@ -67,17 +83,8 @@ router.post('/:provider', async(req, res) => {
   }
 
   try {
-    const fs = await retrieveSet(setName);
-    if (0 === fs.length) {
-      logger.info('No available feature servers to handle createCall API request');
-      return res
-        .json({
-          msg: 'no available feature servers at this time'
-        })
-        .status(480);
-    }
-    const ip = fs[idx++ % fs.length];
-    const serviceUrl = `http://${ip}:3000/v1/messaging/${provider}`;
+    const serviceUrl = await getFsUrl(logger, retrieveSet, setName, provider);
+    if (!serviceUrl) res.json({msg: 'no available feature servers at this time'}).status(480);
     const messageSid = uuidv4();
     const payload = await Promise.resolve(filterFn({messageSid}, req.body));
 
@@ -113,7 +120,7 @@ router.post('/:provider', async(req, res) => {
 
     logger.debug({body: req.body, payload}, 'filtered incoming SMS');
 
-    logger.info({payload, url: serviceUrl}, `sending incomingSms API request to FS at ${ip}`);
+    logger.info({payload, url: serviceUrl}, `sending incomingSms API request to FS at ${serviceUrl}`);
 
     request({
       url: serviceUrl,
@@ -123,7 +130,7 @@ router.post('/:provider', async(req, res) => {
     },
     async(err, response, body) => {
       if (err) {
-        logger.error(err, `Error sending incomingSms POST to ${ip}`);
+        logger.error(err, `Error sending incomingSms POST to ${serviceUrl}`);
         return res.sendStatus(500);
       }
       if (200 === response.statusCode) {
@@ -131,7 +138,7 @@ router.post('/:provider', async(req, res) => {
         logger.info({body}, 'sending response to provider for incomingSMS');
         return doSendResponse(res, respondFn, body);
       }
-      logger.error({statusCode: response.statusCode}, `Non-success response returned by incomingSms ${ip}`);
+      logger.error({statusCode: response.statusCode}, `Non-success response returned by incomingSms ${serviceUrl}`);
       return res.sendStatus(500);
     });
   } catch (err) {

lib/routes/api/speech-credentials.js

@@ -1,57 +1,248 @@
 const router = require('express').Router();
+const assert = require('assert');
+const Account = require('../../models/account');
 const SpeechCredential = require('../../models/speech-credential');
 const sysError = require('../error');
-const {decrypt, encrypt} = require('../../utils/encrypt-decrypt');
-const {parseAccountSid, parseServiceProviderSid} = require('./utils');
-const {DbErrorUnprocessableRequest, DbErrorBadRequest} = require('../../utils/errors');
+const {decrypt, encrypt, obscureKey} = require('../../utils/encrypt-decrypt');
+const {parseAccountSid, parseServiceProviderSid, parseSpeechCredentialSid} = require('./utils');
+const {DbErrorUnprocessableRequest, DbErrorForbidden} = require('../../utils/errors');
 const {
   testGoogleTts,
   testGoogleStt,
   testAwsTts,
-  testAwsStt
+  testAwsStt,
+  testMicrosoftStt,
+  testMicrosoftTts,
+  testWellSaidTts,
+  testNuanceStt,
+  testNuanceTts,
+  testDeepgramStt,
+  testSonioxStt,
+  testIbmTts,
+  testIbmStt
 } = require('../../utils/speech-utils');
+const {promisePool} = require('../../db');
+
+const validateAdd = async(req) => {
+  const account_sid = parseAccountSid(req);
+  const service_provider_sid = parseServiceProviderSid(req);
+
+  if (service_provider_sid) {
+    if (req.user.hasServiceProviderAuth && service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+    if (req.user.hasAccountAuth && service_provider_sid !== req.user.service_provider_sid &&
+      req.body.account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+
+  if (account_sid) {
+    if (req.user.hasAccountAuth && account_sid !== req.user.account_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+
+    const [r] = await promisePool.execute(
+      'SELECT service_provider_sid from accounts WHERE account_sid = ?', [account_sid]
+    );
+
+    if (req.user.hasServiceProviderAuth && r[0].service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+  return;
+};
+
+const validateRetrieveUpdateDelete = async(req, speech_credentials) => {
+  if (req.user.hasServiceProviderAuth && speech_credentials[0].service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+
+  if (req.user.hasAccountAuth && speech_credentials[0].account_sid !== req.user.account_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+  return;
+};
+
+const validateRetrieveList = async(req) => {
+  const service_provider_sid = parseServiceProviderSid(req);
+
+  if (service_provider_sid) {
+    if ((req.user.hasServiceProviderAuth || req.user.hasAccountAuth) &&
+     service_provider_sid !== req.user.service_provider_sid) {
+      throw new DbErrorForbidden('Insufficient privileges');
+    }
+  }
+  return;
+};
+
+const validateTest = async(req, speech_credentials) => {
+  if (req.user.hasAdminAuth) {
+    return;
+  }
+
+  if (!req.user.hasAdminAuth && speech_credentials.service_provider_sid !== req.user.service_provider_sid) {
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+
+  if (speech_credentials.service_provider_sid === req.user.service_provider_sid) {
+    if (req.user.hasServiceProviderAuth) {
+      return;
+    }
+
+    if (req.user.hasAccountAuth && (!speech_credentials.account_sid ||
+       speech_credentials.account_sid === req.user.account_sid)) {
+      return;
+    }
+
+    throw new DbErrorForbidden('Insufficient privileges');
+  }
+};
+
+const encryptCredential = (obj) => {
+  const {
+    vendor,
+    service_key,
+    access_key_id,
+    secret_access_key,
+    aws_region,
+    api_key,
+    region,
+    client_id,
+    secret,
+    nuance_tts_uri,
+    nuance_stt_uri,
+    use_custom_tts,
+    custom_tts_endpoint,
+    custom_tts_endpoint_url,
+    use_custom_stt,
+    custom_stt_endpoint,
+    custom_stt_endpoint_url,
+    tts_api_key,
+    tts_region,
+    stt_api_key,
+    stt_region,
+    riva_server_uri,
+    instance_id,
+    custom_stt_url,
+    custom_tts_url,
+    auth_token = ''
+  } = obj;
+
+  switch (vendor) {
+    case 'google':
+      assert(service_key, 'invalid json key: service_key is required');
+      try {
+        const o = JSON.parse(service_key);
+        assert(o.client_email && o.private_key, 'invalid google service account key');
+      }
+      catch (err) {
+        assert(false, 'invalid google service account key - not JSON');
+      }
+      return encrypt(service_key);
+
+    case 'aws':
+      assert(access_key_id, 'invalid aws speech credential: access_key_id is required');
+      assert(secret_access_key, 'invalid aws speech credential: secret_access_key is required');
+      assert(aws_region, 'invalid aws speech credential: aws_region is required');
+      const awsData = JSON.stringify({aws_region, access_key_id, secret_access_key});
+      return encrypt(awsData);
+
+    case 'microsoft':
+      if (!custom_tts_endpoint_url && !custom_stt_endpoint_url) {
+        assert(region, 'invalid azure speech credential: region is required');
+        assert(api_key, 'invalid azure speech credential: api_key is required');
+      }
+      const azureData = JSON.stringify({
+        ...(region && {region}),
+        ...(api_key && {api_key}),
+        use_custom_tts,
+        custom_tts_endpoint,
+        custom_tts_endpoint_url,
+        use_custom_stt,
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
+      });
+      return encrypt(azureData);
+
+    case 'wellsaid':
+      assert(api_key, 'invalid wellsaid speech credential: api_key is required');
+      const wsData = JSON.stringify({api_key});
+      return encrypt(wsData);
+
+    case 'nuance':
+      const checked = (client_id && secret) || (nuance_tts_uri || nuance_stt_uri);
+      assert(checked, 'invalid nuance speech credential: either entered client id and\
+        secret or entered a nuance_tts_uri or nuance_stt_uri');
+      const nuanceData = JSON.stringify({client_id, secret, nuance_tts_uri, nuance_stt_uri});
+      return encrypt(nuanceData);
+
+    case 'deepgram':
+      assert(api_key, 'invalid deepgram speech credential: api_key is required');
+      const deepgramData = JSON.stringify({api_key});
+      return encrypt(deepgramData);
+
+    case 'ibm':
+      const ibmData = JSON.stringify({tts_api_key, tts_region, stt_api_key, stt_region, instance_id});
+      return encrypt(ibmData);
+
+    case 'nvidia':
+      assert(riva_server_uri, 'invalid riva server uri: riva_server_uri is required');
+      const nvidiaData = JSON.stringify({ riva_server_uri });
+      return encrypt(nvidiaData);
+
+    case 'soniox':
+      assert(api_key, 'invalid soniox speech credential: api_key is required');
+      const sonioxData = JSON.stringify({api_key});
+      return encrypt(sonioxData);
+
+    default:
+      if (vendor.startsWith('custom:')) {
+        const customData = JSON.stringify({auth_token, custom_stt_url, custom_tts_url});
+        return encrypt(customData);
+      }
+      else assert(false, `invalid or missing vendor: ${vendor}`);
+  }
+};
 
 router.post('/', async(req, res) => {
   const logger = req.app.locals.logger;
-  const {use_for_stt, use_for_tts, vendor, service_key, access_key_id, secret_access_key, aws_region} = req.body;
-  const account_sid = req.user.account_sid || req.body.account_sid;
-  let service_provider_sid;
-  if (!account_sid) {
-    if (!req.user.hasServiceProviderAuth) {
-      logger.error('POST /SpeechCredentials invalid credentials');
-      return res.send(403);
-    }
-    service_provider_sid = parseServiceProviderSid(req);
-  }
+
   try {
-    let encrypted_credential;
-    if (vendor === 'google') {
-      let obj;
-      if (!service_key) throw new DbErrorBadRequest('invalid json key: service_key is required');
-      try {
-        obj = JSON.parse(service_key);
-        if (!obj.client_email || !obj.private_key) {
-          throw new DbErrorBadRequest('invalid google service account key');
-        }
+    const {
+      use_for_stt,
+      use_for_tts,
+      vendor,
+      label
+    } = req.body;
+    const account_sid = req.user.account_sid || req.body.account_sid;
+    const service_provider_sid = req.user.service_provider_sid ||
+    req.body.service_provider_sid || parseServiceProviderSid(req);
+
+    await validateAdd(req);
+
+    if (!account_sid) {
+      if (!req.user.hasServiceProviderAuth && !req.user.hasAdminAuth) {
+        logger.error('POST /SpeechCredentials invalid credentials');
+        return res.sendStatus(403);
       }
-      catch (err) {
-        throw new DbErrorBadRequest('invalid google service account key - not JSON');
+    }
+
+    // Check if vendor and label is already used for account or SP
+    if (label) {
+      const existingSpeech = await SpeechCredential.isAvailableVendorAndLabel(
+        service_provider_sid, account_sid, vendor, label);
+      if (existingSpeech.length > 0) {
+        throw new DbErrorUnprocessableRequest(`Label ${label} is already in use for another speech credential`);
       }
-      encrypted_credential = encrypt(service_key);
     }
-    else if (vendor === 'aws') {
-      const data = JSON.stringify({
-        aws_region: aws_region || 'us-east-1',
-        access_key_id,
-        secret_access_key
-      });
-      encrypted_credential = encrypt(data);
-    }
-    else throw new DbErrorBadRequest(`invalid speech vendor ${vendor}`);
+
+    const encrypted_credential = encryptCredential(req.body);
     const uuid = await SpeechCredential.make({
       account_sid,
       service_provider_sid,
       vendor,
+      label,
       use_for_tts,
       use_for_stt,
       credential: encrypted_credential
@@ -66,24 +257,98 @@ router.post('/', async(req, res) => {
  * retrieve all speech credentials for an account
  */
 router.get('/', async(req, res) => {
-  let service_provider_sid;
-  const account_sid = parseAccountSid(req);
-  if (!account_sid) service_provider_sid = parseServiceProviderSid(req);
   const logger = req.app.locals.logger;
+
   try {
-    const creds = account_sid ?
-      await SpeechCredential.retrieveAll(account_sid) :
-      await SpeechCredential.retrieveAllForSP(service_provider_sid);
+    const account_sid = parseAccountSid(req) ? parseAccountSid(req) : req.user.account_sid;
+    const service_provider_sid = parseServiceProviderSid(req);
+
+    await validateRetrieveList(req);
+
+    const credsAccount = account_sid ? await SpeechCredential.retrieveAll(account_sid) : [];
+    const credsSP = service_provider_sid ?
+      await SpeechCredential.retrieveAllForSP(service_provider_sid) :
+      await SpeechCredential.retrieveAllForSP((await Account.retrieve(account_sid))[0].service_provider_sid);
+
+    // filter out duplicates and discard those from other non-matching accounts
+    let creds = [...new Set([...credsAccount, ...credsSP].map((c) => JSON.stringify(c)))].map((c) => JSON.parse(c));
+    if (req.user.hasScope('account')) {
+      creds = creds.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid);
+    }
 
     res.status(200).json(creds.map((c) => {
       const {credential, ...obj} = c;
+
       if ('google' === obj.vendor) {
-        obj.service_key = decrypt(credential);
+        const o = JSON.parse(decrypt(credential));
+        const key_header = '-----BEGIN PRIVATE KEY-----\n';
+        const obscured = {
+          ...o,
+          private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
+        };
+        obj.service_key = obscured;
       }
       else if ('aws' === obj.vendor) {
-        const o = decrypt(credential);
+        const o = JSON.parse(decrypt(credential));
         obj.access_key_id = o.access_key_id;
-        obj.secret_access_key = o.secret_access_key;
+        obj.secret_access_key = obscureKey(o.secret_access_key);
+        obj.aws_region = o.aws_region;
+        logger.info({obj, o}, 'retrieving aws speech credential');
+      }
+      else if ('microsoft' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.api_key = obscureKey(o.api_key);
+        obj.region = o.region;
+        obj.use_custom_tts = o.use_custom_tts;
+        obj.custom_tts_endpoint = o.custom_tts_endpoint;
+        obj.custom_tts_endpoint_url = o.custom_tts_endpoint_url;
+        obj.use_custom_stt = o.use_custom_stt;
+        obj.custom_stt_endpoint = o.custom_stt_endpoint;
+        obj.custom_stt_endpoint_url = o.custom_stt_endpoint_url;
+        logger.info({obj, o}, 'retrieving azure speech credential');
+      }
+      else if ('wellsaid' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.api_key = obscureKey(o.api_key);
+      }
+      else if ('nuance' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.client_id = o.client_id;
+        obj.secret = o.secret ? obscureKey(o.secret) : null;
+      }
+      else if ('deepgram' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.api_key = obscureKey(o.api_key);
+      }
+      else if ('ibm' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.tts_api_key = obscureKey(o.tts_api_key);
+        obj.tts_region = o.tts_region;
+        obj.stt_api_key = obscureKey(o.stt_api_key);
+        obj.stt_region = o.stt_region;
+        obj.instance_id = o.instance_id;
+      } else if ('nvidia' == obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.riva_server_uri = o.riva_server_uri;
+      }
+      else if ('soniox' === obj.vendor) {
+        const o = JSON.parse(decrypt(credential));
+        obj.api_key = obscureKey(o.api_key);
+      }
+      else if (obj.vendor.startsWith('custom:')) {
+        const o = JSON.parse(decrypt(credential));
+        obj.auth_token = obscureKey(o.auth_token);
+        obj.custom_stt_url = o.custom_stt_url;
+        obj.custom_tts_url = o.custom_tts_url;
+      }
+
+      if (req.user.hasAccountAuth && obj.account_sid === null) {
+        delete obj.api_key;
+        delete obj.secret_access_key;
+        delete obj.secret;
+        delete obj.auth_token;
+        delete obj.stt_api_key;
+        delete obj.tts_api_key;
       }
       return obj;
     }));
@@ -96,20 +361,87 @@ router.get('/', async(req, res) => {
  * retrieve a specific speech credential
  */
 router.get('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
     const cred = await SpeechCredential.retrieve(sid);
     if (0 === cred.length) return res.sendStatus(404);
+
+    await validateRetrieveUpdateDelete(req, cred);
+
     const {credential, ...obj} = cred[0];
     if ('google' === obj.vendor) {
-      obj.service_key = decrypt(credential);
+      const o = JSON.parse(decrypt(credential));
+      const key_header = '-----BEGIN PRIVATE KEY-----\n';
+      const obscured = {
+        ...o,
+        private_key: `${key_header}${obscureKey(o.private_key.slice(key_header.length, o.private_key.length))}`
+      };
+      obj.service_key = JSON.stringify(obscured);
     }
     else if ('aws' === obj.vendor) {
       const o = JSON.parse(decrypt(credential));
       obj.access_key_id = o.access_key_id;
-      obj.secret_access_key = o.secret_access_key;
+      obj.secret_access_key = obscureKey(o.secret_access_key);
+      obj.aws_region = o.aws_region;
     }
+    else if ('microsoft' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = obscureKey(o.api_key);
+      obj.region = o.region;
+      obj.use_custom_tts = o.use_custom_tts;
+      obj.custom_tts_endpoint = o.custom_tts_endpoint;
+      obj.custom_tts_endpoint_url = o.custom_tts_endpoint_url;
+      obj.use_custom_stt = o.use_custom_stt;
+      obj.custom_stt_endpoint = o.custom_stt_endpoint;
+      obj.custom_stt_endpoint_url = o.custom_stt_endpoint_url;
+    }
+    else if ('wellsaid' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = obscureKey(o.api_key);
+    }
+    else if ('nuance' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.client_id = o.client_id;
+      obj.secret = o.secret ? obscureKey(o.secret) : null;
+      obj.nuance_tts_uri = o.nuance_tts_uri;
+      obj.nuance_stt_uri = o.nuance_stt_uri;
+    }
+    else if ('deepgram' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = obscureKey(o.api_key);
+    }
+    else if ('ibm' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.tts_api_key = obscureKey(o.tts_api_key);
+      obj.tts_region = o.tts_region;
+      obj.stt_api_key = obscureKey(o.stt_api_key);
+      obj.stt_region = o.stt_region;
+      obj.instance_id = o.instance_id;
+    } else if ('nvidia' == obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.riva_server_uri = o.riva_server_uri;
+    }
+    else if ('soniox' === obj.vendor) {
+      const o = JSON.parse(decrypt(credential));
+      obj.api_key = obscureKey(o.api_key);
+    }
+    else if (obj.vendor.startsWith('custom:')) {
+      const o = JSON.parse(decrypt(credential));
+      obj.auth_token = obscureKey(o.auth_token);
+      obj.custom_stt_url = o.custom_stt_url;
+      obj.custom_tts_url = o.custom_tts_url;
+    }
+
+    if (req.user.hasAccountAuth && obj.account_sid === null) {
+      delete obj.api_key;
+      delete obj.secret_access_key;
+      delete obj.secret;
+      delete obj.auth_token;
+      delete obj.stt_api_key;
+      delete obj.tts_api_key;
+    }
+
     res.status(200).json(obj);
   } catch (err) {
     sysError(logger, res, err);
@@ -120,9 +452,11 @@ router.get('/:sid', async(req, res) => {
  * delete a speech credential
  */
 router.delete('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
+    const cred = await SpeechCredential.retrieve(sid);
+    await validateRetrieveUpdateDelete(req, cred);
     const count = await SpeechCredential.remove(sid);
     if (0 === count) return res.sendStatus(404);
     res.sendStatus(204);
@@ -136,10 +470,11 @@ router.delete('/:sid', async(req, res) => {
  * update a speech credential -- we only allow use_for_tts and use_for_stt to be updated
  */
 router.put('/:sid', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
-    const {use_for_tts, use_for_stt} = req.body;
+    const sid = parseSpeechCredentialSid(req);
+    const {use_for_tts, use_for_stt, region, aws_region, stt_region, tts_region,
+      riva_server_uri, nuance_tts_uri, nuance_stt_uri} = req.body;
     if (typeof use_for_tts === 'undefined' && typeof use_for_stt === 'undefined') {
       throw new DbErrorUnprocessableRequest('use_for_tts and use_for_stt are the only updateable fields');
     }
@@ -151,6 +486,57 @@ router.put('/:sid', async(req, res) => {
       obj.use_for_stt = use_for_stt;
     }
 
+    /* update the credential if provided */
+    try {
+      const cred = await SpeechCredential.retrieve(sid);
+
+      await validateRetrieveUpdateDelete(req, cred);
+
+      if (1 === cred.length) {
+        const {credential, vendor} = cred[0];
+        const o = JSON.parse(decrypt(credential));
+        const {
+          use_custom_tts,
+          custom_tts_endpoint,
+          custom_tts_endpoint_url,
+          use_custom_stt,
+          custom_stt_endpoint,
+          custom_stt_endpoint_url,
+          custom_stt_url,
+          custom_tts_url
+        } = req.body;
+
+        const newCred = {
+          ...o,
+          region,
+          vendor,
+          aws_region,
+          use_custom_tts,
+          custom_tts_endpoint,
+          custom_tts_endpoint_url,
+          use_custom_stt,
+          custom_stt_endpoint,
+          custom_stt_endpoint_url,
+          stt_region,
+          tts_region,
+          riva_server_uri,
+          nuance_stt_uri,
+          nuance_tts_uri,
+          custom_stt_url,
+          custom_tts_url
+        };
+        logger.info({o, newCred}, 'updating speech credential with this new credential');
+        obj.credential = encryptCredential(newCred);
+        obj.vendor = vendor;
+      }
+      else {
+        logger.info({sid}, 'speech credential not found!!');
+      }
+    } catch (err) {
+      logger.error({err}, 'error updating speech credential');
+    }
+
+    logger.info({obj}, 'updating speech credential with changes');
     const rowsAffected = await SpeechCredential.update(sid, obj);
     if (rowsAffected === 0) {
       return res.sendStatus(404);
@@ -166,12 +552,15 @@ router.put('/:sid', async(req, res) => {
  * Test a credential
  */
 router.get('/:sid/test', async(req, res) => {
-  const sid = req.params.sid;
   const logger = req.app.locals.logger;
   try {
+    const sid = parseSpeechCredentialSid(req);
     const creds = await SpeechCredential.retrieve(sid);
+
     if (!creds || 0 === creds.length) return res.sendStatus(404);
 
+    await validateTest(req, creds[0]);
+
     const cred = creds[0];
     const credential = JSON.parse(decrypt(cred.credential));
     const results = {
@@ -189,7 +578,8 @@ router.get('/:sid/test', async(req, res) => {
 
       if (cred.use_for_tts) {
         try {
-          await testGoogleTts(logger, credential);
+          const {getTtsVoices} = req.app.locals;
+          await testGoogleTts(logger, getTtsVoices, credential);
           results.tts.status = 'ok';
           SpeechCredential.ttsTestResult(sid, true);
         } catch (err) {
@@ -211,8 +601,9 @@ router.get('/:sid/test', async(req, res) => {
     }
     else if (cred.vendor === 'aws') {
       if (cred.use_for_tts) {
+        const {getTtsVoices} = req.app.locals;
         try {
-          await testAwsTts(logger, {
+          await testAwsTts(logger, getTtsVoices, {
             accessKeyId: credential.access_key_id,
             secretAccessKey: credential.secret_access_key,
             region: credential.aws_region || process.env.AWS_REGION
@@ -239,7 +630,160 @@ router.get('/:sid/test', async(req, res) => {
         }
       }
     }
+    else if (cred.vendor === 'microsoft') {
+      const {
+        api_key,
+        region,
+        use_custom_tts,
+        custom_tts_endpoint,
+        custom_tts_endpoint_url,
+        use_custom_stt,
+        custom_stt_endpoint,
+        custom_stt_endpoint_url
+      } = credential;
+      if (cred.use_for_tts) {
+        try {
+          await testMicrosoftTts(logger, {
+            api_key,
+            region,
+            use_custom_tts,
+            custom_tts_endpoint,
+            custom_tts_endpoint_url,
+            use_custom_stt,
+            custom_stt_endpoint,
+            custom_stt_endpoint_url
+          });
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+      if (cred.use_for_stt) {
+        try {
+          await testMicrosoftStt(logger, {api_key, region});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'wellsaid') {
+      const {api_key} = credential;
+      if (cred.use_for_tts) {
+        try {
+          await testWellSaidTts(logger, {api_key});
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          results.tts = {status: 'fail', reason: err.message};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'nuance') {
+      const {getTtsVoices} = req.app.locals;
+
+      const {
+        client_id,
+        secret,
+        nuance_tts_uri,
+        nuance_stt_uri
+      } = credential;
+      if (cred.use_for_tts) {
+        try {
+          await testNuanceTts(logger, getTtsVoices, {
+            client_id,
+            secret,
+            nuance_tts_uri
+          });
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          logger.error({err}, 'error testing nuance tts');
+          const reason = err.statusCode === 401 ?
+            'invalid client_id or secret' :
+            (err.message || 'error accessing nuance tts service with provided credentials');
+          results.tts = {status: 'fail', reason};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+      if (cred.use_for_stt) {
+        try {
+          await testNuanceStt(logger, {client_id, secret, nuance_stt_uri});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'deepgram') {
+      const {api_key} = credential;
+      if (cred.use_for_stt) {
+        try {
+          await testDeepgramStt(logger, {api_key});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'ibm') {
+      const {getTtsVoices} = req.app.locals;
+
+      if (cred.use_for_tts) {
+        const {tts_api_key, tts_region} = credential;
+        try {
+          await testIbmTts(logger, getTtsVoices, {
+            tts_api_key,
+            tts_region
+          });
+          results.tts.status = 'ok';
+          SpeechCredential.ttsTestResult(sid, true);
+        } catch (err) {
+          logger.error({err}, 'error testing ibm tts');
+          const reason = err.statusCode === 401 ?
+            'invalid api_key or region' :
+            (err.message || 'error accessing ibm tts service with provided credentials');
+          results.tts = {status: 'fail', reason};
+          SpeechCredential.ttsTestResult(sid, false);
+        }
+      }
+      if (cred.use_for_stt) {
+        const {stt_api_key, stt_region, instance_id} = credential;
+        try {
+          await testIbmStt(logger, {stt_region, stt_api_key, instance_id});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+    else if (cred.vendor === 'soniox') {
+      const {api_key} = credential;
+      if (cred.use_for_stt) {
+        try {
+          await testSonioxStt(logger, {api_key});
+          results.stt.status = 'ok';
+          SpeechCredential.sttTestResult(sid, true);
+        } catch (err) {
+          results.stt = {status: 'fail', reason: err.message};
+          SpeechCredential.sttTestResult(sid, false);
+        }
+      }
+    }
+
     res.status(200).json(results);
+
   } catch (err) {
     sysError(logger, res, err);
   }

lib/routes/api/system-information.js

@@ -0,0 +1,14 @@
+const router = require('express').Router();
+const SystemInformation = require('../../models/system-information');
+
+router.post('/', async(req, res) => {
+  const sysInfo = await SystemInformation.add(req.body);
+  res.status(201).json(sysInfo);
+});
+
+router.get('/', async(req, res) => {
+  const [sysInfo] = await SystemInformation.retrieveAll();
+  res.status(200).json(sysInfo || {});
+});
+
+module.exports = router;

lib/routes/api/tts-cache.js

@@ -0,0 +1,29 @@
+const router = require('express').Router();
+const {
+  parseAccountSid
+} = require('./utils');
+
+router.delete('/', async(req, res) => {
+  const {purgeTtsCache} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  if (account_sid) {
+    await purgeTtsCache({account_sid});
+  } else {
+    await purgeTtsCache();
+  }
+  res.sendStatus(204);
+});
+
+router.get('/', async(req, res) => {
+  const {getTtsSize} = req.app.locals;
+  const account_sid = parseAccountSid(req);
+  let size = 0;
+  if (account_sid) {
+    size = await getTtsSize(`tts:${account_sid}:*`);
+  } else {
+    size = await getTtsSize();
+  }
+  res.status(200).json({size});
+});
+
+module.exports = router;

lib/routes/api/users.js

@@ -1,16 +1,22 @@
-//const assert = require('assert');
-//const debug = require('debug')('jambonz:api-server');
 const router = require('express').Router();
-const {DbErrorBadRequest} = require('../../utils/errors');
+const User = require('../../models/user');
+const {DbErrorBadRequest, BadRequestError, DbErrorForbidden} = require('../../utils/errors');
 const {generateHashedPassword, verifyPassword} = require('../../utils/password-utils');
 const {promisePool} = require('../../db');
+const {validatePasswordSettings, parseUserSid} = require('./utils');
 const {decrypt} = require('../../utils/encrypt-decrypt');
+const {cacheClient} = require('../../helpers');
 const sysError = require('../error');
 const retrieveMyDetails = `SELECT * 
 FROM users user
 JOIN accounts AS account ON account.account_sid = user.account_sid 
 LEFT JOIN service_providers as sp ON account.service_provider_sid = sp.service_provider_sid  
 WHERE user.user_sid = ?`;
+const retrieveMyDetails2 = `SELECT * 
+FROM users user
+LEFT JOIN accounts AS account ON account.account_sid = user.account_sid 
+LEFT JOIN service_providers as sp ON sp.service_provider_sid = user.service_provider_sid 
+WHERE user.user_sid = ?`;
 const retrieveSql = 'SELECT * from users where user_sid = ?';
 const retrieveProducts = `SELECT * 
 FROM account_products 
@@ -22,109 +28,290 @@ AND account_subscriptions.pending=0`;
 const updateSql = 'UPDATE users set hashed_password = ?, force_change = false WHERE user_sid = ?';
 const retrieveStaticIps = 'SELECT * FROM account_static_ips WHERE account_sid = ?';
 
-const validateRequest = async(user_sid, payload) => {
-  const {old_password, new_password, name, email, email_activation_code} = payload;
+const validateRequest = async(user_sid, req) => {
+  const payload = req.body;
+  const {
+    old_password,
+    new_password,
+    initial_password,
+    name,
+    email,
+    email_activation_code,
+    force_change,
+    is_active
+  } = payload;
 
   const [r] = await promisePool.query(retrieveSql, user_sid);
-  if (r.length === 0) return null;
+  if (r.length === 0) {
+    throw new DbErrorBadRequest('Invalid request: user_sid does not exist');
+  }
   const user = r[0];
 
+  /* it is not allowed for anyone to promote a user to a higher level of authority */
+  if (null === payload.account_sid || null === payload.service_provider_sid) {
+    throw new DbErrorBadRequest('Invalid request: user may not be promoted');
+  }
+
+  if (req.user.hasAccountAuth) {
+    /* account user may not change modify account_sid or service_provider_sid */
+    if ('account_sid' in payload && payload.account_sid !== user.account_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another account');
+    }
+    if ('service_provider_sid' in payload && payload.service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another service provider');
+    }
+  }
+  if (req.user.hasServiceProviderAuth) {
+    if ('service_provider_sid' in payload && payload.service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be promoted or moved to another service provider');
+    }
+  }
+  if ('account_sid' in payload) {
+    const [r] = await promisePool.query('SELECT * FROM accounts WHERE account_sid = ?', payload.account_sid);
+    if (r.length === 0) throw new DbErrorBadRequest('Invalid request: account_sid does not exist');
+    const {service_provider_sid} = r[0];
+    if (service_provider_sid !== user.service_provider_sid) {
+      throw new DbErrorBadRequest('Invalid request: user may not be moved to another service provider');
+    }
+  }
+
+  if (initial_password) {
+    await validatePasswordSettings(initial_password);
+  }
+
   if ((old_password && !new_password) || (new_password && !old_password)) {
     throw new DbErrorBadRequest('new_password and old_password both required');
   }
+  if (new_password) {
+    await validatePasswordSettings(new_password);
+  }
   if (new_password && name) throw new DbErrorBadRequest('can not change name and password simultaneously');
   if (new_password && user.provider !== 'local') {
     throw new DbErrorBadRequest('can not change password when using oauth2');
   }
 
-  if ((email && !email_activation_code) || (email_activation_code && !email)) {
+  if (email_activation_code && !email) {
     throw new DbErrorBadRequest('email and email_activation_code both required');
   }
-  if (!name && !new_password && !email) throw new DbErrorBadRequest('no updates requested');
+  if (!name && !new_password && !email && !initial_password && !force_change && !is_active)
+    throw new DbErrorBadRequest('no updates requested');
 
   return user;
 };
 
+const getActiveAdminUsers = (users) => {
+  return users.filter((e) => !e.account_sid && !e.service_provider_sid && e.is_active);
+};
+
+const ensureUserActionIsAllowed = (req, user) => {
+  if (req.user.hasAdminAuth) {
+    return;
+  }
+
+  if (req.user.hasServiceProviderAuth && req.user.service_provider_sid === user.service_provider_sid) {
+    return;
+  }
+
+  if (req.user.hasAccountAuth && req.user.account_sid === user.account_sid) {
+    return;
+  }
+
+  throw new DbErrorForbidden('insufficient permissions');
+};
+
+const ensureUserDeletionIsAllowed = (req, activeAdminUsers, user) => {
+  try {
+    if (req.user.hasAdminAuth && activeAdminUsers.length === 1 && activeAdminUsers[0].user_sid === user[0].user_sid) {
+      throw new BadRequestError('cannot delete this admin user - there are no other active admin users');
+    }
+
+    ensureUserActionIsAllowed(req, user[0]);
+    return;
+  } catch (error) {
+    throw error;
+  }
+};
+
+const ensureUserRetrievalIsAllowed = (req, user) => {
+  try {
+    ensureUserActionIsAllowed(req, user);
+    return;
+  } catch (error) {
+    throw error;
+  }
+};
+
+router.get('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  let usersList;
+  try {
+    let results;
+    if (req.user.hasAdminAuth) {
+      results = await User.retrieveAll();
+    }
+    else if (req.user.hasAccountAuth) {
+      results = await User.retrieveAllForAccount(req.user.account_sid, true);
+    }
+    else if (req.user.hasServiceProviderAuth) {
+      results = await User.retrieveAllForServiceProvider(req.user.service_provider_sid, true);
+    }
+
+    if (results.length === 0) throw new Error('failure retrieving users list');
+
+    usersList = results.map((user) => {
+      const {
+        user_sid,
+        name,
+        email,
+        force_change,
+        is_active,
+        account_sid,
+        service_provider_sid,
+        account_name,
+        service_provider_name
+      } = user;
+      let scope;
+      if (account_sid && service_provider_sid) {
+        scope = 'account';
+      } else if (service_provider_sid) {
+        scope = 'service_provider';
+      } else {
+        scope = 'admin';
+      }
+
+      const obj = {
+        user_sid,
+        name,
+        email,
+        scope,
+        force_change,
+        is_active,
+        ...(account_sid && {account_sid}),
+        ...(account_name && {account_name}),
+        ...(service_provider_sid && {service_provider_sid}),
+        ...(service_provider_name && {service_provider_name})
+      };
+      return obj;
+    });
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+  res.status(200).json(usersList);
+});
+
 router.get('/me', async(req, res) => {
   const logger = req.app.locals.logger;
   const {user_sid} = req.user;
 
   if (!user_sid) return res.sendStatus(403);
 
+  let payload;
   try {
-    const [r] = await promisePool.query({sql: retrieveMyDetails, nestTables: true}, user_sid);
-    logger.debug(r, 'retrieved user details');
-    const payload = r[0];
-    const {user, account, sp} = payload;
-    ['hashed_password', 'salt', 'phone_activation_code', 'email_activation_code', 'account_sid'].forEach((prop) => {
-      delete user[prop];
-    });
-    ['email_validated', 'phone_validated', 'force_change'].forEach((prop) => user[prop] = !!user[prop]);
-    ['is_active'].forEach((prop) => account[prop] = !!account[prop]);
-    account.root_domain = sp.root_domain;
-    delete payload.sp;
-
-    /* get api keys */
-    const [keys] = await promisePool.query('SELECT * from api_keys WHERE account_sid = ?', account.account_sid);
-    payload.api_keys = keys.map((k) => {
-      return {
-        api_key_sid: k.api_key_sid,
-        //token: k.token.replace(/.(?=.{4,}$)/g, '*'),
-        token: k.token,
-        last_used: k.last_used,
-        created_at: k.created_at
-      };
-    });
-
-    /* get products */
-    const [products] = await promisePool.query({sql: retrieveProducts, nestTables: true}, account.account_sid);
-    if (!products.length || !products[0].account_subscriptions) {
-      throw new Error('account is missing a subscription');
-    }
-    const account_subscription = products[0].account_subscriptions;
-    payload.subscription = {
-      status: 'active',
-      account_subscription_sid: account_subscription.account_subscription_sid,
-      start_date: account_subscription.effective_start_date,
-      products: products.map((prd) => {
-        return {
-          name: prd.products.name,
-          units: prd.products.unit_label,
-          quantity: prd.account_products.quantity
-        };
-      })
-    };
-    if (account_subscription.pending) {
-      Object.assign(payload.subscription, {
-        status: 'suspended',
-        suspend_reason: account_subscription.pending_reason
+    if (process.env.JAMBONES_HOSTING) {
+      const [r] = await promisePool.query({sql: retrieveMyDetails, nestTables: true}, user_sid);
+      logger.debug(r, 'retrieved user details');
+      payload = r[0];
+      const {user, account, sp} = payload;
+      ['hashed_password', 'salt', 'phone_activation_code', 'email_activation_code', 'account_sid'].forEach((prop) => {
+        delete user[prop];
       });
-    }
-    const {
-      last4,
-      exp_month,
-      exp_year,
-      card_type,
-      stripe_statement_descriptor
-    } = account_subscription;
-    if (last4) {
-      const real_last4 = decrypt(last4);
-      Object.assign(payload.subscription, {
-        last4: real_last4,
+      ['email_validated', 'phone_validated', 'force_change'].forEach((prop) => user[prop] = !!user[prop]);
+      ['is_active'].forEach((prop) => account[prop] = !!account[prop]);
+      account.root_domain = sp.root_domain;
+      delete payload.sp;
+
+      /* get api keys */
+      const [keys] = await promisePool.query('SELECT * from api_keys WHERE account_sid = ?', account.account_sid);
+      payload.api_keys = keys.map((k) => {
+        return {
+          api_key_sid: k.api_key_sid,
+          //token: k.token.replace(/.(?=.{4,}$)/g, '*'),
+          token: k.token,
+          last_used: k.last_used,
+          created_at: k.created_at
+        };
+      });
+
+      /* get products */
+      const [products] = await promisePool.query({sql: retrieveProducts, nestTables: true}, account.account_sid);
+      if (!products.length || !products[0].account_subscriptions) {
+        throw new Error('account is missing a subscription');
+      }
+      const account_subscription = products[0].account_subscriptions;
+      payload.subscription = {
+        status: 'active',
+        account_subscription_sid: account_subscription.account_subscription_sid,
+        start_date: account_subscription.effective_start_date,
+        products: products.map((prd) => {
+          return {
+            name: prd.products.name,
+            units: prd.products.unit_label,
+            quantity: prd.account_products.quantity
+          };
+        })
+      };
+      if (account_subscription.pending) {
+        Object.assign(payload.subscription, {
+          status: 'suspended',
+          suspend_reason: account_subscription.pending_reason
+        });
+      }
+      const {
+        last4,
         exp_month,
         exp_year,
         card_type,
-        statement_descriptor: stripe_statement_descriptor
+        stripe_statement_descriptor
+      } = account_subscription;
+      if (last4) {
+        const real_last4 = decrypt(last4);
+        Object.assign(payload.subscription, {
+          last4: real_last4,
+          exp_month,
+          exp_year,
+          card_type,
+          statement_descriptor: stripe_statement_descriptor
+        });
+      }
+
+      /* get static ips */
+      const [static_ips] = await promisePool.query(retrieveStaticIps, account.account_sid);
+      payload.static_ips = static_ips.map((r) => r.public_ipv4);
+    }
+    else {
+      const [r] = await promisePool.query({sql: retrieveMyDetails2, nestTables: true}, user_sid);
+      logger.debug(r, 'retrieved user details');
+      payload = r[0];
+      const {user} = payload;
+      ['hashed_password', 'salt', 'phone_activation_code', 'email_activation_code'].forEach((prop) => {
+        delete user[prop];
       });
+      ['email_validated', 'phone_validated', 'force_change'].forEach((prop) => user[prop] = !!user[prop]);
+    }
+    logger.debug({payload}, 'returning user details');
+    res.json(payload);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.get('/:user_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const user_sid = parseUserSid(req);
+    const [user] = await User.retrieve(user_sid);
+
+    if (!user) {
+      throw new Error('failure retrieving user');
     }
 
-    /* get static ips */
-    const [static_ips] = await promisePool.query(retrieveStaticIps, account.account_sid);
-    payload.static_ips = static_ips.map((r) => r.public_ipv4);
+    ensureUserRetrievalIsAllowed(req, user);
 
-    logger.debug({payload}, 'returning user details');
-
-    res.json(payload);
+    // eslint-disable-next-line no-unused-vars
+    const { hashed_password, ...rest } = user;
+    return res.status(200).json(rest);
   } catch (err) {
     sysError(logger, res, err);
   }
@@ -133,12 +320,32 @@ router.get('/me', async(req, res) => {
 router.put('/:user_sid', async(req, res) => {
   const logger = req.app.locals.logger;
   const {user_sid} = req.params;
-  const {old_password, new_password, name, email, email_activation_code} = req.body;
+  const user = await User.retrieve(user_sid);
+  const {hasAccountAuth, hasServiceProviderAuth, hasAdminAuth} = req.user;
+  const {
+    old_password,
+    new_password,
+    initial_password,
+    email_activation_code,
+    email,
+    name,
+    is_active,
+    force_change,
+    account_sid,
+    service_provider_sid
+  } = req.body;
 
-  if (req.user.user_sid && req.user.user_sid !== user_sid) return res.sendStatus(403);
+  //if (req.user.user_sid && req.user.user_sid !== user_sid) return res.sendStatus(403);
+
+  if (!hasAdminAuth &&
+  !(hasAccountAuth && user[0] && req.user.account_sid === user[0].account_sid) &&
+  !(hasServiceProviderAuth && user[0] && req.user.service_provider_sid === user[0].service_provider_sid) &&
+  (req.user.user_sid && req.user.user_sid !== user_sid)) {
+    return res.sendStatus(403);
+  }
 
   try {
-    const user = await validateRequest(user_sid, req.body);
+    const user = await validateRequest(user_sid, req);
     if (!user) return res.sendStatus(404);
 
     if (new_password) {
@@ -149,6 +356,11 @@ router.put('/:user_sid', async(req, res) => {
         //debug(`PUT /Users/:sid pwd ${old_password} does not match hash ${old_hashed_password}`);
         return res.sendStatus(403);
       }
+
+      if (old_password === new_password) {
+        throw new Error('new password cannot be your old password');
+      }
+
       const passwordHash = await generateHashedPassword(new_password);
       //debug(`updating hashed_password to ${passwordHash}`);
       const r = await promisePool.execute(updateSql, [passwordHash, user_sid]);
@@ -160,10 +372,55 @@ router.put('/:user_sid', async(req, res) => {
       if (0 === r.changedRows) throw new Error('database update failed');
     }
 
-    if (email) {
+    if (initial_password) {
+      const passwordHash = await generateHashedPassword(initial_password);
       const r = await promisePool.execute(
-        'UPDATE users SET email = ?, email_activation_code = ?, email_validated = 0 WHERE user_sid = ?',
-        [email, email_activation_code, user_sid]);
+        'UPDATE users SET hashed_password = ? WHERE user_sid = ?',
+        [passwordHash, user_sid]
+      );
+      if (0 === r.changedRows) throw new Error('database update failed');
+    }
+
+    if (typeof is_active !== 'undefined') {
+      const r = await promisePool.execute('UPDATE users SET is_active = ? WHERE user_sid = ?', [is_active, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+    }
+
+    if (typeof force_change !== 'undefined') {
+      const r = await promisePool.execute(
+        'UPDATE users SET force_change = ? WHERE user_sid = ?',
+        [force_change, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+    }
+
+    if (account_sid || account_sid === null) {
+      const r = await promisePool.execute(
+        'UPDATE users SET account_sid = ? WHERE user_sid = ?',
+        [account_sid, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+      const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+      await cacheClient.delete(redisKey);
+    }
+
+    if (service_provider_sid || service_provider_sid === null) {
+      const r = await promisePool.execute(
+        'UPDATE users SET service_provider_sid = ? WHERE user_sid = ?',
+        [service_provider_sid, user_sid]);
+      if (0 === r.changedRows) throw new Error('database update failed');
+      const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+      await cacheClient.delete(redisKey);
+    }
+
+    if (email) {
+      if (email_activation_code) {
+        const r = await promisePool.execute(
+          'UPDATE users SET email = ?, email_activation_code = ?, email_validated = 0 WHERE user_sid = ?',
+          [email, email_activation_code, user_sid]);
+        if (0 === r.changedRows) throw new Error('database update failed');
+      }
+      const r = await promisePool.execute(
+        'UPDATE users SET email = ? WHERE user_sid = ?',
+        [email, user_sid]);
       if (0 === r.changedRows) throw new Error('database update failed');
 
       if (process.env.NODE_ENV !== 'test') {
@@ -176,5 +433,80 @@ router.put('/:user_sid', async(req, res) => {
   }
 });
 
+router.post('/', async(req, res) => {
+  const logger = req.app.locals.logger;
+  const passwordHash = await generateHashedPassword(req.body.initial_password);
+  const payload = {
+    ...req.body,
+    provider: 'local',
+    hashed_password: passwordHash,
+  };
+  const allUsers = await User.retrieveAll();
+  delete payload.initial_password;
+
+  try {
+    if (req.body.initial_password) {
+      await validatePasswordSettings(req.body.initial_password);
+    }
+    const email = allUsers.find((e) => e.email === payload.email);
+    const name = allUsers.find((e) => e.name === payload.name);
+
+    if (name) {
+      logger.debug({payload}, 'user with this username already exists');
+      return res.status(422).json({msg: 'invalid username or email'});
+    }
+
+    if (email) {
+      logger.debug({payload}, 'user with this email already exists');
+      return res.status(422).json({msg: 'invalid username or email'});
+    }
+
+    if (req.user.hasAdminAuth) {
+      logger.debug({payload}, 'POST /users');
+      const uuid = await User.make(payload);
+      res.status(201).json({user_sid: uuid});
+    }
+    else if (req.user.hasAccountAuth) {
+      logger.debug({payload}, 'POST /users');
+      const uuid = await User.make({
+        ...payload,
+        account_sid: req.user.account_sid,
+      });
+      res.status(201).json({user_sid: uuid});
+    }
+    else if (req.user.hasServiceProviderAuth) {
+      logger.debug({payload}, 'POST /users');
+      const uuid = await User.make({
+        ...payload,
+        service_provider_sid: req.user.service_provider_sid,
+      });
+      res.status(201).json({user_sid: uuid});
+    }
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
+
+router.delete('/:user_sid', async(req, res) => {
+  const logger = req.app.locals.logger;
+
+  try {
+    const user_sid = parseUserSid(req);
+    const allUsers = await User.retrieveAll();
+    const activeAdminUsers = getActiveAdminUsers(allUsers);
+    const user = allUsers.filter((user) => user.user_sid === user_sid);
+
+    ensureUserDeletionIsAllowed(req, activeAdminUsers, user);
+    await User.remove(user_sid);
+
+    /* invalidate the jwt of the deleted user */
+    const redisKey = cacheClient.generateRedisKey('jwt', user_sid, 'v2');
+    await cacheClient.delete(redisKey);
+
+    return res.sendStatus(204);
+  } catch (err) {
+    sysError(logger, res, err);
+  }
+});
 
 module.exports = router;

lib/routes/api/utils.js

@@ -1,14 +1,18 @@
-const uuid = require('uuid').v4;
+const { v4: uuid, validate } = require('uuid');
+const bent = require('bent');
 const Account = require('../../models/account');
 const {promisePool} = require('../../db');
 const {cancelSubscription, detachPaymentMethod} = require('../../utils/stripe-utils');
 const freePlans = require('../../utils/free_plans');
+const { BadRequestError, DbErrorBadRequest } = require('../../utils/errors');
 const insertAccountSubscriptionSql = `INSERT INTO account_subscriptions 
 (account_subscription_sid, account_sid) 
 values (?, ?)`;
 const replaceOldSubscriptionSql = `UPDATE account_subscriptions 
 SET effective_end_date = CURRENT_TIMESTAMP, change_reason = ?  
 WHERE account_subscription_sid = ?`;
+//const request = require('request');
+//require('request-debug')(request);
 
 const setupFreeTrial = async(logger, account_sid, isReturningUser) => {
   const sid = uuid();
@@ -123,7 +127,7 @@ const createTestAlerts = async(writeAlerts, AlertType, account_sid) => {
         data.push({timestamp, account_sid, alert_type: AlertType.CARRIER_NOT_PROVISIONED});
         break;
       case 4:
-        data.push({timestamp, account_sid, alert_type: AlertType.CALL_LIMIT, count: 50});
+        data.push({timestamp, account_sid, alert_type: AlertType.ACCOUNT_CALL_LIMIT, count: 50});
         break;
       default:
         break;
@@ -134,38 +138,179 @@ const createTestAlerts = async(writeAlerts, AlertType, account_sid) => {
 
 };
 
+const validateSid = (model, req) => {
+  const arr = new RegExp(`${model}\/([^\/]*)`).exec(req.originalUrl);
+
+  if (arr) {
+    const sid = arr[1];
+    const sid_validation = validate(sid);
+    if (!sid_validation) {
+      throw new BadRequestError(`invalid ${model}Sid format`);
+    }
+
+    return arr[1];
+  }
+
+  return;
+};
+
 const parseServiceProviderSid = (req) => {
-  const arr = /ServiceProviders\/([^\/]*)/.exec(req.originalUrl);
-  if (arr) return arr[1];
+  try {
+    return validateSid('ServiceProviders', req);
+  } catch (error) {
+    throw error;
+  }
 };
 
 const parseAccountSid = (req) => {
-  const arr = /Accounts\/([^\/]*)/.exec(req.originalUrl);
-  if (arr) return arr[1];
+  try {
+    return validateSid('Accounts', req);
+  } catch (error) {
+    throw error;
+  }
 };
 
-const hasAccountPermissions = (req, res, next) => {
-  if (req.user.hasScope('admin')) return next();
-  if (req.user.hasScope('account')) {
-    const account_sid = parseAccountSid(req);
-    if (account_sid === req.user.account_sid) return next();
+const parseApplicationSid = (req) => {
+  try {
+    return validateSid('Applications', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseCallSid = (req) => {
+  try {
+    return validateSid('Calls', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parsePhoneNumberSid = (req) => {
+  try {
+    return validateSid('PhoneNumbers', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseSpeechCredentialSid = (req) => {
+  try {
+    return validateSid('SpeechCredentials', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseVoipCarrierSid = (req) => {
+  try {
+    return validateSid('VoipCarriers', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseWebhookSid = (req) => {
+  try {
+    return validateSid('Webhooks', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseSipGatewaySid = (req) => {
+  try {
+    return validateSid('SipGateways', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseUserSid = (req) => {
+  try {
+    return validateSid('Users', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const parseLcrSid = (req) => {
+  try {
+    return validateSid('Lcrs', req);
+  } catch (error) {
+    throw error;
+  }
+};
+
+const hasAccountPermissions = async(req, res, next) => {
+  try {
+    if (req.user.hasScope('admin')) {
+      return next();
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const service_provider_sid = parseServiceProviderSid(req);
+      const account_sid = parseAccountSid(req);
+      if (service_provider_sid) {
+        if (service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+      if (account_sid) {
+        const [r] = await Account.retrieve(account_sid);
+        if (r && r.service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+    }
+
+    if (req.user.hasScope('account')) {
+      const account_sid = parseAccountSid(req);
+      const service_provider_sid = parseServiceProviderSid(req);
+      const [r] = await Account.retrieve(account_sid);
+
+      if (account_sid) {
+        if (r && r.account_sid === req.user.account_sid) {
+          return next();
+        }
+      }
+
+      if (service_provider_sid) {
+        if (r && r.service_provider_sid === req.user.service_provider_sid) {
+          return next();
+        }
+      }
+    }
+
+    res.status(403).json({
+      status: 'fail',
+      message: 'insufficient privileges'
+    });
+  } catch (error) {
+    throw error;
   }
-  res.status(403).json({
-    status: 'fail',
-    message: 'insufficient privileges'
-  });
 };
 
 const hasServiceProviderPermissions = (req, res, next) => {
-  if (req.user.hasScope('admin')) return next();
-  if (req.user.hasScope('service_provider')) {
-    const service_provider_sid = parseServiceProviderSid(req);
-    if (service_provider_sid === req.user.service_provider_sid) return next();
+  try {
+    if (req.user.hasScope('admin')) {
+      return next();
+    }
+
+    if (req.user.hasScope('service_provider')) {
+      const service_provider_sid = parseServiceProviderSid(req);
+      if (service_provider_sid === req.user.service_provider_sid) {
+        return next();
+      }
+    }
+
+    res.status(403).json({
+      status: 'fail',
+      message: 'insufficient privileges'
+    });
+  } catch (error) {
+    throw error;
   }
-  res.status(403).json({
-    status: 'fail',
-    message: 'insufficient privileges'
-  });
 };
 
 const checkLimits = async(req, res, next) => {
@@ -224,13 +369,96 @@ const checkLimits = async(req, res, next) => {
   next();
 };
 
+const getSubspaceJWT = async(id, secret) => {
+  const postJwt = bent('https://id.subspace.com', 'POST', 'json', 200);
+  const jwt = await postJwt('/oauth/token',
+    {
+      client_id: id,
+      client_secret: secret,
+      audience: 'https://api.subspace.com/',
+      grant_type: 'client_credentials',
+    }
+  );
+  return jwt.access_token;
+};
+
+const enableSubspace = async(opts) => {
+  const {subspace_client_id, subspace_client_secret, destination} = opts;
+  const accessToken = await getSubspaceJWT(subspace_client_id, subspace_client_secret);
+  const postTeleport = bent('https://api.subspace.com', 'POST', 'json', 200);
+
+  const teleport = await postTeleport('/v1/sipteleport',
+    {
+      name: 'Jambonz',
+      destination,
+      status: 'ENABLED'
+    },
+    {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${accessToken}`
+    }
+  );
+
+  return teleport;
+};
+
+const disableSubspace = async(opts) => {
+  const {subspace_client_id, subspace_client_secret, subspace_sip_teleport_id} = opts;
+  const accessToken = await getSubspaceJWT(subspace_client_id, subspace_client_secret);
+  const relativeUrl = `/v1/sipteleport/${subspace_sip_teleport_id}`;
+  const deleteTeleport = bent('https://api.subspace.com', 'DELETE', 'json', 200);
+  await deleteTeleport(relativeUrl, {},
+    {
+      Authorization: `Bearer ${accessToken}`
+    }
+  );
+  return;
+};
+
+const validatePasswordSettings = async(password) => {
+  const sql = 'SELECT * from password_settings';
+  const [rows] = await promisePool.execute(sql);
+  const specialChars = /[!@#$%^&*()_+\-=\[\]{};':"\\|,.<>\/?]+/;
+  const numbers = /[0-9]+/;
+  if (rows.length === 0) {
+    if (password.length < 8 || password.length > 20) {
+      throw new DbErrorBadRequest('password length must be between 8 and 20');
+    }
+  } else {
+    if (rows[0].min_password_length && password.length < rows[0].min_password_length) {
+      throw new DbErrorBadRequest(`password must be at least ${rows[0].min_password_length} characters long`);
+    }
+
+    if (rows[0].require_digit === 1 && !numbers.test(password)) {
+      throw new DbErrorBadRequest('password must contain at least one digit');
+    }
+
+    if (rows[0].require_special_character === 1 && !specialChars.test(password)) {
+      throw new DbErrorBadRequest('password must contain at least one special character');
+    }
+  }
+  return;
+};
+
 module.exports = {
   setupFreeTrial,
   createTestCdrs,
   createTestAlerts,
   parseAccountSid,
+  parseApplicationSid,
+  parseCallSid,
+  parsePhoneNumberSid,
   parseServiceProviderSid,
+  parseSpeechCredentialSid,
+  parseVoipCarrierSid,
+  parseWebhookSid,
+  parseSipGatewaySid,
+  parseUserSid,
+  parseLcrSid,
   hasAccountPermissions,
   hasServiceProviderPermissions,
-  checkLimits
+  checkLimits,
+  enableSubspace,
+  disableSubspace,
+  validatePasswordSettings
 };

lib/routes/api/voip-carriers.js

@@ -4,6 +4,7 @@ const VoipCarrier = require('../../models/voip-carrier');
 const {promisePool} = require('../../db');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const { parseVoipCarrierSid } = require('./utils');
 
 const validate = async(req) => {
   const {lookupAppBySid, lookupAccountBySid} = req.app.locals;
@@ -73,7 +74,14 @@ decorate(router, VoipCarrier, ['add', 'update', 'delete'], preconditions);
 router.get('/', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
-    const results = await VoipCarrier.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null);
+    const results = req.user.hasAdminAuth ?
+      await VoipCarrier.retrieveAll(req.user.hasAccountAuth ? req.user.account_sid : null) :
+      await VoipCarrier.retrieveAllForSP(req.user.service_provider_sid);
+
+    if (req.user.hasScope('account')) {
+      return res.status(200).json(results.filter((c) => c.account_sid === req.user.account_sid || !c.account_sid));
+    }
+
     res.status(200).json(results);
   } catch (err) {
     sysError(logger, res, err);
@@ -84,9 +92,24 @@ router.get('/', async(req, res) => {
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
   try {
+    const sid = parseVoipCarrierSid(req);
     const account_sid = req.user.hasAccountAuth ? req.user.account_sid : null;
-    const results = await VoipCarrier.retrieve(req.params.sid, account_sid);
+    const results = await VoipCarrier.retrieve(sid, account_sid);
     if (results.length === 0) return res.status(404).end();
+    const ret = results[0];
+    ret.register_status = JSON.parse(ret.register_status || '{}');
+
+    if (req.user.hasServiceProviderAuth && results.length === 1) {
+      if (results.length === 1 && results[0].service_provider_sid !== req.user.service_provider_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+    if (req.user.hasAccountAuth && results.length === 1) {
+      if (results.length === 1 && results[0].account_sid !== req.user.account_sid) {
+        throw new DbErrorBadRequest('insufficient privileges');
+      }
+    }
+
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/api/webhooks.js

@@ -2,15 +2,37 @@ const router = require('express').Router();
 const Webhook = require('../../models/webhook');
 const decorate = require('./decorate');
 const sysError = require('../error');
+const {DbErrorForbidden} = require('../../utils/errors');
+const { parseWebhookSid } = require('./utils');
+const {promisePool} = require('../../db');
 
 decorate(router, Webhook, ['add']);
 
 /* retrieve */
 router.get('/:sid', async(req, res) => {
   const logger = req.app.locals.logger;
+
   try {
-    const results = await Webhook.retrieve(req.params.sid);
+    const sid = parseWebhookSid(req);
+    const results = await Webhook.retrieve(sid);
+
     if (results.length === 0) return res.status(404).end();
+
+    if (req.user.hasAccountAuth) {
+      /* can only update carriers for the user's account */
+      if (results[0].account_sid !== req.user.account_sid) {
+        throw new DbErrorForbidden('insufficient privileges');
+      }
+    }
+    if (req.user.hasServiceProviderAuth) {
+      const [r] = await promisePool.execute(
+        'SELECT service_provider_sid from accounts WHERE account_sid = ?', [results[0].account_sid]
+      );
+      if (r.length === 1 && r[0].service_provider_sid === req.user.service_provider_sid) {
+        return;
+      }
+      throw new DbErrorForbidden('insufficient permissions');
+    }
     return res.status(200).json(results[0]);
   }
   catch (err) {

lib/routes/error.js

@@ -1,6 +1,15 @@
-const {DbErrorBadRequest, DbErrorUnprocessableRequest, DbErrorForbidden} = require('../utils/errors');
+const {
+  BadRequestError,
+  DbErrorBadRequest,
+  DbErrorUnprocessableRequest,
+  DbErrorForbidden
+} = require('../utils/errors');
 
 function sysError(logger, res, err) {
+  if (err instanceof BadRequestError) {
+    logger.info(err, err.message);
+    return res.status(400).json({msg: 'Bad request'});
+  }
   if (err instanceof DbErrorBadRequest) {
     logger.info(err, 'invalid client request');
     return res.status(400).json({msg: err.message});

lib/routes/stripe/webhook.js

@@ -61,8 +61,7 @@ router.post('/', express.raw({type: 'application/json'}), async(req, res) => {
   }
 
   /* process event */
-  logger.info(`received webhook: ${evt.type}`);
-  if (evt.type.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
+  if (evt?.type?.startsWith('invoice.')) handleInvoiceEvents(logger, evt);
   else {
     logger.debug(evt, 'unhandled stripe webook');
   }

lib/utils/email-utils.js

@@ -1,6 +1,7 @@
 const formData = require('form-data');
 const Mailgun = require('mailgun.js');
 const mailgun = new Mailgun(formData);
+const bent = require('bent');
 const validateEmail = (email) => {
   // eslint-disable-next-line max-len
   const re = /^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
@@ -8,23 +9,67 @@ const validateEmail = (email) => {
 };
 
 const emailSimpleText = async(logger, to, subject, text) => {
-  const mg = mailgun.client({
-    username: 'api',
-    key: process.env.MAILGUN_API_KEY
-  });
-  if (!process.env.MAILGUN_API_KEY) throw new Error('MAILGUN_API_KEY env variable is not defined!');
-  if (!process.env.MAILGUN_DOMAIN) throw new Error('MAILGUN_DOMAIN env variable is not defined!');
+  const from = 'jambonz Support <support@jambonz.org>';
+  if (process.env.CUSTOM_EMAIL_VENDOR_URL) {
+    await sendEmailByCustomVendor(logger, from, to, subject, text);
+  } else {
+    await sendEmailByMailgun(logger, from, to, subject, text);
+  }
 
+};
+
+const sendEmailByCustomVendor = async(logger, from, to, subject, text) => {
   try {
-    const res = await mg.messages.create(process.env.MAILGUN_DOMAIN, {
-      from: 'jambonz Support <support@jambonz.org>',
+    const post = bent('POST', {
+      'Content-Type': 'application/json',
+      ...((process.env.CUSTOM_EMAIL_VENDOR_USERNAME && process.env.CUSTOM_EMAIL_VENDOR_PASSWORD) &&
+        ({
+          'Authorization':`Basic ${Buffer.from(
+            `${process.env.CUSTOM_EMAIL_VENDOR_USERNAME}:${process.env.CUSTOM_EMAIL_VENDOR_PASSWORD}`
+          ).toString('base64')}`
+        }))
+    });
+
+    const res = await post(process.env.CUSTOM_EMAIL_VENDOR_URL, {
+      from,
       to,
       subject,
       text
     });
-    logger.debug({res}, 'sent email');
+    logger.debug({
+      res
+    }, 'sent email to custom vendor.');
   } catch (err) {
-    logger.info({err}, 'Error sending email');
+    logger.info({
+      err
+    }, 'Error sending email From Custom email vendor');
+  }
+};
+
+const sendEmailByMailgun = async(logger, from, to, subject, text) => {
+  if (!process.env.MAILGUN_API_KEY) throw new Error('MAILGUN_API_KEY env variable is not defined!');
+  if (!process.env.MAILGUN_DOMAIN) throw new Error('MAILGUN_DOMAIN env variable is not defined!');
+
+  const mg = mailgun.client({
+    username: 'api',
+    key: process.env.MAILGUN_API_KEY,
+    ...(process.env.MAILGUN_URL && {url: process.env.MAILGUN_URL})
+  });
+
+  try {
+    const res = await mg.messages.create(process.env.MAILGUN_DOMAIN, {
+      from,
+      to,
+      subject,
+      text
+    });
+    logger.debug({
+      res
+    }, 'sent email');
+  } catch (err) {
+    logger.info({
+      err
+    }, 'Error sending email From mailgun');
   }
 };
 

lib/utils/encrypt-decrypt.js

@@ -1,10 +1,10 @@
 const crypto = require('crypto');
-const algorithm = 'aes-256-cbc';
+const algorithm = process.env.LEGACY_CRYPTO ? 'aes-256-ctr' : 'aes-256-cbc';
 const iv = crypto.randomBytes(16);
 const secretKey = crypto.createHash('sha256')
-  .update(String(process.env.JWT_SECRET))
+  .update(process.env.ENCRYPTION_SECRET || process.env.JWT_SECRET)
   .digest('base64')
-  .substr(0, 32);
+  .substring(0, 32);
 
 const encrypt = (text) => {
   const cipher = crypto.createCipheriv(algorithm, secretKey, iv);
@@ -23,7 +23,18 @@ const decrypt = (data) => {
   return decrpyted.toString();
 };
 
+const obscureKey = (key, key_spoiler_length = 6) => {
+  const key_spoiler_char = 'X';
+
+  if (!key || key.length <= key_spoiler_length) {
+    return key;
+  }
+
+  return `${key.slice(0, key_spoiler_length)}${key_spoiler_char.repeat(key.length - key_spoiler_length)}`;
+};
+
 module.exports = {
   encrypt,
-  decrypt
+  decrypt,
+  obscureKey
 };

lib/utils/errors.js

@@ -1,3 +1,9 @@
+class BadRequestError extends Error {
+  constructor(msg) {
+    super(msg);
+  }
+}
+
 class DbError extends Error {
   constructor(msg) {
     super(msg);
@@ -23,6 +29,7 @@ class DbErrorForbidden extends DbError {
 }
 
 module.exports = {
+  BadRequestError,
   DbError,
   DbErrorBadRequest,
   DbErrorUnprocessableRequest,

lib/utils/free_plans.json

@@ -2,7 +2,7 @@
   "trial": [
     {
       "category": "voice_call_session",
-      "quantity": 20
+      "quantity": 5
     },
     {
       "category": "device",

lib/utils/homer-utils.js

@@ -39,11 +39,17 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
     const obj = await postJSON('/api/v3/call/transaction', {
       param: {
         transaction: {
-          call: true
+          call: true,
+          registration: true,
+          rest: false
         },
+        orlogic: true,
         search: {
           '1_call': {
             callid: [callId]
+          },
+          '1_registration': {
+            callid: [callId]
           }
         },
       },
@@ -58,7 +64,7 @@ const getHomerSipTrace = async(logger, apiKey, callId) => {
   }
 };
 
-const getHomerPcap = async(logger, apiKey, callIds) => {
+const getHomerPcap = async(logger, apiKey, callIds, method) => {
   if (!process.env.HOMER_BASE_URL || !process.env.HOMER_USERNAME || !process.env.HOMER_PASSWORD) {
     logger.debug('getHomerPcap: Homer integration not installed');
   }
@@ -67,12 +73,23 @@ const getHomerPcap = async(logger, apiKey, callIds) => {
     const stream = await postPcap('/api/v3/export/call/messages/pcap', {
       param: {
         transaction: {
-          call: true
+          call: method === 'invite',
+          registration: method === 'register',
+          rest: false
         },
+        orlogic: true,
         search: {
-          '1_call': {
-            callid: callIds
-          }
+          ...(method === 'invite' && {
+            '1_call': {
+              callid: callIds
+            }
+          })
+          ,
+          ...(method === 'register' && {
+            '1_registration': {
+              callid: callIds
+            }
+          })
         },
       },
       timestamp: {

lib/utils/jaeger-utils.js

@@ -0,0 +1,19 @@
+const bent = require('bent');
+const getJSON = bent(process.env.JAEGER_BASE_URL || 'http://127.0.0.1', 'GET', 'json', 200);
+
+const getJaegerTrace = async(logger, traceId) => {
+  if (!process.env.JAEGER_BASE_URL) {
+    logger.debug('getJaegerTrace: jaeger integration not installed');
+    return null;
+  }
+  try {
+    return await getJSON(`/api/v3/traces/${traceId}`);
+  } catch (err) {
+    const url = `${process.env.JAEGER_BASE_URL}/api/traces/${traceId}`;
+    logger.error({err, traceId}, `getJaegerTrace: Error retrieving spans from ${url}`);
+  }
+};
+
+module.exports = {
+  getJaegerTrace
+};

lib/utils/jambonz-sample.text

@@ -0,0 +1 @@
+Hello From Jambonz. This file was created because Record all call bucket credential test.

lib/utils/password-utils.js

@@ -1,18 +1,19 @@
 const crypto = require('crypto');
-const { argon2i } = require('argon2-ffi');
+const argon2 = require('argon2');
 const util = require('util');
 
+const { argon2i } = argon2;
+
 const getRandomBytes = util.promisify(crypto.randomBytes);
 
 const generateHashedPassword = async(password) => {
   const salt = await getRandomBytes(32);
-  const passwordHash =  await argon2i.hash(password, salt);
+  const passwordHash = await argon2.hash(password, { type: argon2i, salt });
   return passwordHash;
 };
 
-const verifyPassword = async(passwordHash, password) => {
-  const isCorrect = await argon2i.verify(passwordHash, password);
-  return isCorrect;
+const verifyPassword = (passwordHash, password) => {
+  return argon2.verify(passwordHash, password);
 };
 
 const hashString = (s) => crypto.createHash('md5').update(s).digest('hex');

lib/utils/speech-utils.js

@@ -1,12 +1,41 @@
-const ttsGoogle = require('@google-cloud/text-to-speech');
 const sttGoogle = require('@google-cloud/speech').v1p1beta1;
-const Polly = require('aws-sdk/clients/polly');
-const AWS = require('aws-sdk');
+const { TranscribeClient, ListVocabulariesCommand } = require('@aws-sdk/client-transcribe');
+const { Deepgram } = require('@deepgram/sdk');
+const sdk = require('microsoft-cognitiveservices-speech-sdk');
+const { SpeechClient } = require('@soniox/soniox-node');
+const bent = require('bent');
 const fs = require('fs');
 
-const testGoogleTts = async(logger, credentials) => {
-  const client = new ttsGoogle.TextToSpeechClient({credentials});
-  await client.listVoices();
+
+const testSonioxStt = async(logger, credentials) => {
+  const api_key = credentials;
+  const soniox = new SpeechClient(api_key);
+
+  return new Promise(async(resolve, reject) => {
+    try {
+      const result = await soniox.transcribeFileShort('data/test_audio.wav');
+      if (result.words.length > 0) resolve(result);
+      else reject(new Error('no transcript returned'));
+    } catch (error) {
+      logger.info({error}, 'failed to get soniox transcript');
+      reject(error);
+    }
+  });
+};
+
+const testNuanceTts = async(logger, getTtsVoices, credentials) => {
+  const voices = await getTtsVoices({vendor: 'nuance', credentials});
+  return voices;
+};
+const testNuanceStt = async(logger, credentials) => {
+  //TODO
+  return true;
+};
+
+const testGoogleTts = async(logger, getTtsVoices, credentials) => {
+  const voices = await getTtsVoices({vendor: 'google', credentials});
+  return voices;
+
 };
 
 const testGoogleStt = async(logger, credentials) => {
@@ -31,30 +60,196 @@ const testGoogleStt = async(logger, credentials) => {
   }
 };
 
-const testAwsTts = (logger, credentials) => {
-  const polly = new Polly(credentials);
+const testDeepgramStt = async(logger, credentials) => {
+  const {api_key} = credentials;
+  const deepgram = new Deepgram(api_key);
+
+  const mimetype = 'audio/wav';
+  const source = {
+    buffer: fs.readFileSync(`${__dirname}/../../data/test_audio.wav`),
+    mimetype: mimetype
+  };
+
   return new Promise((resolve, reject) => {
-    polly.describeVoices({LanguageCode: 'en-US'}, (err, data) => {
-      if (err) return reject(err);
-      resolve();
+    // Send the audio to Deepgram and get the response
+    deepgram.transcription
+      .preRecorded(source, {punctuate: true})
+      .then((response) => {
+        //logger.debug({response}, 'got transcript');
+        if (response?.results?.channels[0]?.alternatives?.length > 0) resolve(response);
+        else reject(new Error('no transcript returned'));
+        return;
+      })
+      .catch((err) => {
+        logger.info({err}, 'failed to get deepgram transcript');
+        reject(err);
+      });
+  });
+};
+
+const testMicrosoftStt = async(logger, credentials) => {
+  const {api_key, region} = credentials;
+
+  const speechConfig = sdk.SpeechConfig.fromSubscription(api_key, region);
+  const audioConfig = sdk.AudioConfig.fromWavFileInput(fs.readFileSync(`${__dirname}/../../data/test_audio.wav`));
+  speechConfig.speechRecognitionLanguage = 'en-US';
+  const speechRecognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);
+
+  return new Promise((resolve, reject) => {
+    speechRecognizer.recognizeOnceAsync((result) => {
+      switch (result.reason) {
+        case sdk.ResultReason.RecognizedSpeech:
+          resolve();
+          break;
+        case sdk.ResultReason.NoMatch:
+          reject('Speech could not be recognized.');
+          break;
+        case sdk.ResultReason.Canceled:
+          const cancellation = sdk.CancellationDetails.fromResult(result);
+          logger.info(`CANCELED: Reason=${cancellation.reason}`);
+          if (cancellation.reason == sdk.CancellationReason.Error) {
+            logger.info(`CANCELED: ErrorCode=${cancellation.ErrorCode}`);
+            logger.info(`CANCELED: ErrorDetails=${cancellation.errorDetails}`);
+          }
+          reject(cancellation.reason);
+          break;
+      }
+      speechRecognizer.close();
     });
   });
 };
 
-const testAwsStt = (logger, credentials) => {
-  const transcribeservice = new AWS.TranscribeService(credentials);
-  return new Promise((resolve, reject) => {
-    transcribeservice.listVocabularies((err, data) => {
-      if (err) return reject(err);
-      logger.info({data}, 'retrieved language models');
-      resolve();
+const testAwsTts = async(logger, getTtsVoices, credentials) => {
+  try {
+    const voices = await getTtsVoices({vendor: 'aws', credentials});
+    return voices;
+  } catch (err) {
+    logger.info({err}, 'testMicrosoftTts - failed to list voices for region ${region}');
+    throw err;
+  }
+};
+
+const testAwsStt = async(logger, credentials) => {
+  try {
+    const {region, accessKeyId, secretAccessKey} = credentials;
+    const client = new TranscribeClient({
+      region,
+      credentials: {
+        accessKeyId,
+        secretAccessKey
+      }
     });
+    const command = new ListVocabulariesCommand({});
+    const response =  await client.send(command);
+    return response;
+  } catch (err) {
+    logger.info({err}, 'testMicrosoftTts - failed to list voices for region ${region}');
+    throw err;
+  }
+};
+
+const testMicrosoftTts = async(logger, credentials) => {
+  const {
+    api_key,
+    region,
+    // eslint-disable-next-line no-unused-vars
+    use_custom_tts,
+    // eslint-disable-next-line no-unused-vars
+    custom_tts_endpoint,
+    // eslint-disable-next-line no-unused-vars
+    use_custom_stt,
+    // eslint-disable-next-line no-unused-vars
+    custom_stt_endpoint
+  } = credentials;
+
+  logger.info({
+    api_key,
+    region,
+    use_custom_tts,
+    custom_tts_endpoint,
+    use_custom_stt,
+    custom_stt_endpoint
+  }, 'testing microsoft tts');
+  if (!api_key) throw new Error('testMicrosoftTts: credentials are missing api_key');
+  if (!region) throw new Error('testMicrosoftTts: credentials are missing region');
+  try {
+    const getJSON = bent('json', {
+      'Ocp-Apim-Subscription-Key': api_key
+    });
+    const response = await getJSON(`https://${region}.tts.speech.microsoft.com/cognitiveservices/voices/list`);
+    return response;
+  } catch (err) {
+    logger.info({err}, `testMicrosoftTts - failed to list voices for region ${region}`);
+    throw err;
+  }
+};
+
+const testWellSaidTts = async(logger, credentials) => {
+  const {api_key} = credentials;
+  try {
+    const post = bent('https://api.wellsaidlabs.com', 'POST', 'buffer', {
+      'X-Api-Key': api_key,
+      'Accept': 'audio/mpeg',
+      'Content-Type': 'application/json'
+    });
+    const mp3 = await post('/v1/tts/stream', {
+      text: 'Hello, world',
+      speaker_id: '3'
+    });
+    return mp3;
+  } catch (err) {
+    logger.info({err}, 'testWellSaidTts returned error');
+    throw err;
+  }
+};
+
+const testIbmTts = async(logger, getTtsVoices, credentials) => {
+  const {tts_api_key, tts_region} = credentials;
+  const voices = await getTtsVoices({vendor: 'ibm', credentials: {tts_api_key, tts_region}});
+  return voices;
+};
+
+const testIbmStt = async(logger, credentials) => {
+  const {stt_api_key, stt_region} = credentials;
+  const SpeechToTextV1 = require('ibm-watson/speech-to-text/v1');
+  const { IamAuthenticator } = require('ibm-watson/auth');
+  const speechToText = new SpeechToTextV1({
+    authenticator: new IamAuthenticator({
+      apikey: stt_api_key
+    }),
+    serviceUrl: `https://api.${stt_region}.speech-to-text.watson.cloud.ibm.com`
   });
+  return new Promise((resolve, reject) => {
+    speechToText.listModels()
+      .then((speechModels) => {
+        logger.debug({speechModels}, 'got IBM speech models');
+        return resolve();
+      })
+      .catch((err) => {
+        logger.info({err}, 'failed to get speech models');
+        reject(err);
+      });
+  });
+};
+
+const testWellSaidStt = async(logger, credentials) => {
+  //TODO
+  return true;
 };
 
 module.exports = {
   testGoogleTts,
   testGoogleStt,
   testAwsTts,
-  testAwsStt
+  testWellSaidTts,
+  testAwsStt,
+  testMicrosoftTts,
+  testMicrosoftStt,
+  testWellSaidStt,
+  testNuanceTts,
+  testNuanceStt,
+  testDeepgramStt,
+  testIbmTts,
+  testIbmStt,
+  testSonioxStt
 };

lib/utils/storage-utils.js

@@ -0,0 +1,134 @@
+const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3');
+const {Storage} = require('@google-cloud/storage');
+const fs = require('fs');
+const { BlobServiceClient } = require('@azure/storage-blob');
+
+// Azure
+
+async function testAzureStorage(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient('jambonz-sample.text');
+
+  await blockBlobClient.uploadFile(`${__dirname}/jambonz-sample.text`);
+}
+
+async function getAzureStorageObject(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient(opts.key);
+  const response = await blockBlobClient.download(0);
+  return response.readableStreamBody;
+}
+
+async function deleteAzureStorageObject(logger, opts) {
+  const blobServiceClient = BlobServiceClient.fromConnectionString(opts.connection_string);
+  const containerClient = blobServiceClient.getContainerClient(opts.name);
+  const blockBlobClient = containerClient.getBlockBlobClient(opts.key);
+  await blockBlobClient.delete();
+}
+
+// Google
+
+function _initGoogleClient(opts) {
+  const serviceKey = JSON.parse(opts.service_key);
+  return new Storage({
+    projectId: serviceKey.project_id,
+    credentials: {
+      client_email: serviceKey.client_email,
+      private_key: serviceKey.private_key
+    },
+  });
+}
+
+async function testGoogleStorage(logger, opts) {
+  return new Promise((resolve, reject) => {
+    const storage = _initGoogleClient(opts);
+
+    const blob = storage.bucket(opts.name).file('jambonz-sample.text');
+
+    fs.createReadStream(`${__dirname}/jambonz-sample.text`)
+      .pipe(blob.createWriteStream())
+      .on('error', (err) => reject(err))
+      .on('finish', () => resolve());
+  });
+}
+
+async function getGoogleStorageObject(logger, opts) {
+  const storage = _initGoogleClient(opts);
+
+  const bucket = storage.bucket(opts.name);
+  const file = bucket.file(opts.key);
+  const [exists] = await file.exists();
+  if (exists) {
+    return file.createReadStream();
+  }
+}
+
+async function deleteGoogleStorageObject(logger, opts) {
+  const storage = _initGoogleClient(opts);
+
+  const bucket = storage.bucket(opts.name);
+  const file = bucket.file(opts.key);
+
+  await file.delete();
+}
+
+// S3
+
+function _initS3Client(opts) {
+  return new S3Client({
+    credentials: {
+      accessKeyId: opts.access_key_id,
+      secretAccessKey: opts.secret_access_key,
+    },
+    region: opts.region || 'us-east-1',
+    ...(opts.vendor === 's3_compatible' && { endpoint: opts.endpoint, forcePathStyle: true })
+  });
+}
+
+async function testS3Storage(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const input = {
+    'Body': 'Hello From Jambonz',
+    'Bucket': opts.name,
+    'Key': 'jambonz-sample.text'
+  };
+  const command = new PutObjectCommand(input);
+  await s3.send(command);
+}
+
+async function getS3Object(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const command = new GetObjectCommand(
+    {
+      Bucket: opts.name,
+      Key: opts.key
+    }
+  );
+  const res = await s3.send(command);
+  return res.Body;
+}
+
+async function deleteS3Object(logger, opts) {
+  const s3 = _initS3Client(opts);
+  const command = new DeleteObjectCommand(
+    {
+      Bucket: opts.name,
+      Key: opts.key
+    }
+  );
+  await s3.send(command);
+}
+
+module.exports = {
+  testS3Storage,
+  getS3Object,
+  deleteS3Object,
+  testGoogleStorage,
+  getGoogleStorageObject,
+  deleteGoogleStorageObject,
+  testAzureStorage,
+  getAzureStorageObject,
+  deleteAzureStorageObject
+};